diff --git a/.env b/.env index 9d604630073..6d99d85b3a7 100644 --- a/.env +++ b/.env @@ -1,5 +1,5 @@ APP_IMAGE=gdcc/dataverse:unstable POSTGRES_VERSION=17 DATAVERSE_DB_USER=dataverse -SOLR_VERSION=9.3.0 -SKIP_DEPLOY=0 +SOLR_VERSION=9.8.0 +SKIP_DEPLOY=0 \ No newline at end of file diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 00000000000..6325029dac1 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,11 @@ +# Set update schedule for GitHub Actions +# https://docs.github.com/en/code-security/supply-chain-security/keeping-your-dependencies-updated-automatically/keeping-your-actions-up-to-date-with-dependabot + +version: 2 +updates: + + - package-ecosystem: "github-actions" + directory: "/" + schedule: + # Check for updates to GitHub Actions daily + interval: "daily" diff --git a/.github/workflows/container_app_pr.yml b/.github/workflows/container_app_pr.yml index c86d284e74b..4a06cb567b0 100644 --- a/.github/workflows/container_app_pr.yml +++ b/.github/workflows/container_app_pr.yml @@ -20,14 +20,14 @@ jobs: if: ${{ github.repository_owner == 'IQSS' }} steps: # Checkout the pull request code as when merged - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: ref: 'refs/pull/${{ github.event.client_payload.pull_request.number }}/merge' - - uses: actions/setup-java@v3 + - uses: actions/setup-java@v4 with: java-version: "17" distribution: 'adopt' - - uses: actions/cache@v3 + - uses: actions/cache@v4 with: path: ~/.m2 key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} @@ -35,14 +35,14 @@ jobs: # Note: Accessing, pushing tags etc. to GHCR will only succeed in upstream because secrets. - name: Login to Github Container Registry - uses: docker/login-action@v2 + uses: docker/login-action@v3 with: registry: ghcr.io username: ${{ secrets.GHCR_USERNAME }} password: ${{ secrets.GHCR_TOKEN }} - name: Set up QEMU for multi-arch builds - uses: docker/setup-qemu-action@v2 + uses: docker/setup-qemu-action@v3 # Get the image tag from either the command or default to branch name (Not used for now) #- name: Get the target tag name @@ -87,7 +87,7 @@ jobs: :ship: [See on GHCR](https://github.com/orgs/gdcc/packages/container). Use by referencing with full name as printed above, mind the registry name. # Leave a note when things have gone sideways - - uses: peter-evans/create-or-update-comment@v3 + - uses: peter-evans/create-or-update-comment@v4 if: ${{ failure() }} with: issue-number: ${{ github.event.client_payload.pull_request.number }} diff --git a/.github/workflows/container_app_push.yml b/.github/workflows/container_app_push.yml index 3b7ce066d73..71ffffb5f48 100644 --- a/.github/workflows/container_app_push.yml +++ b/.github/workflows/container_app_push.yml @@ -68,15 +68,15 @@ jobs: if: ${{ github.event_name != 'pull_request' && github.ref_name == 'develop' && github.repository_owner == 'IQSS' }} runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - uses: peter-evans/dockerhub-description@v3 + - uses: actions/checkout@v4 + - uses: peter-evans/dockerhub-description@v4 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} repository: gdcc/dataverse short-description: "Dataverse Application Container Image providing the executable" readme-filepath: ./src/main/docker/README.md - - uses: peter-evans/dockerhub-description@v3 + - uses: peter-evans/dockerhub-description@v4 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} @@ -126,20 +126,20 @@ jobs: # Depending on context, we push to different targets. Login accordingly. - if: github.event_name != 'pull_request' name: Log in to Docker Hub registry - uses: docker/login-action@v2 + uses: docker/login-action@v3 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - if: ${{ github.event_name == 'pull_request' }} name: Login to Github Container Registry - uses: docker/login-action@v2 + uses: docker/login-action@v3 with: registry: ghcr.io username: ${{ secrets.GHCR_USERNAME }} password: ${{ secrets.GHCR_TOKEN }} - name: Set up QEMU for multi-arch builds - uses: docker/setup-qemu-action@v2 + uses: docker/setup-qemu-action@v3 - name: Re-set image tag based on branch (if master) if: ${{ github.ref_name == 'master' }} diff --git a/.github/workflows/copy_labels.yml b/.github/workflows/copy_labels.yml new file mode 100644 index 00000000000..8e9061c6655 --- /dev/null +++ b/.github/workflows/copy_labels.yml @@ -0,0 +1,15 @@ +name: Copy labels from issue to pull request + +on: + pull_request: + types: [opened] + +jobs: + copy-labels: + runs-on: ubuntu-latest + name: Copy labels from linked issues + steps: + - name: copy-labels + uses: michalvankodev/copy-issue-labels@v1.3.0 + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/deploy_beta_testing.yml b/.github/workflows/deploy_beta_testing.yml index eca8416732a..7d3a45c6235 100644 --- a/.github/workflows/deploy_beta_testing.yml +++ b/.github/workflows/deploy_beta_testing.yml @@ -68,7 +68,7 @@ jobs: overwrite: true - name: Execute payara war deployment remotely - uses: appleboy/ssh-action@v1.0.0 + uses: appleboy/ssh-action@v1.2.1 env: INPUT_WAR_FILE: ${{ env.war_file }} with: diff --git a/.github/workflows/guides_build_sphinx.yml b/.github/workflows/guides_build_sphinx.yml index 86b59b11d35..fa3a876c418 100644 --- a/.github/workflows/guides_build_sphinx.yml +++ b/.github/workflows/guides_build_sphinx.yml @@ -10,7 +10,7 @@ jobs: docs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - uses: uncch-rdmc/sphinx-action@master with: docs-folder: "doc/sphinx-guides/" diff --git a/.github/workflows/pr_comment_commands.yml b/.github/workflows/pr_comment_commands.yml index 5ff75def623..06b11b1ac5b 100644 --- a/.github/workflows/pr_comment_commands.yml +++ b/.github/workflows/pr_comment_commands.yml @@ -9,7 +9,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Dispatch - uses: peter-evans/slash-command-dispatch@v3 + uses: peter-evans/slash-command-dispatch@v4 with: # This token belongs to @dataversebot and has sufficient scope. token: ${{ secrets.GHCR_TOKEN }} diff --git a/.github/workflows/reviewdog_checkstyle.yml b/.github/workflows/reviewdog_checkstyle.yml index 90a0dd7d06b..804b04f696a 100644 --- a/.github/workflows/reviewdog_checkstyle.yml +++ b/.github/workflows/reviewdog_checkstyle.yml @@ -10,7 +10,7 @@ jobs: name: Checkstyle job steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v4 - name: Run check style uses: nikitasavinov/checkstyle-action@master with: diff --git a/.github/workflows/shellcheck.yml b/.github/workflows/shellcheck.yml index 56f7d648dc4..fb9cf5a0a1f 100644 --- a/.github/workflows/shellcheck.yml +++ b/.github/workflows/shellcheck.yml @@ -21,7 +21,7 @@ jobs: permissions: pull-requests: write steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: shellcheck uses: reviewdog/action-shellcheck@v1 with: diff --git a/.github/workflows/shellspec.yml b/.github/workflows/shellspec.yml index 3320d9d08a4..cc09992edac 100644 --- a/.github/workflows/shellspec.yml +++ b/.github/workflows/shellspec.yml @@ -19,7 +19,7 @@ jobs: steps: - name: Install shellspec run: curl -fsSL https://git.io/shellspec | sh -s ${{ env.SHELLSPEC_VERSION }} --yes - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Run Shellspec run: | cd tests/shell @@ -30,7 +30,7 @@ jobs: container: image: rockylinux/rockylinux:9 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Install shellspec run: | curl -fsSL https://github.com/shellspec/shellspec/releases/download/${{ env.SHELLSPEC_VERSION }}/shellspec-dist.tar.gz | tar -xz -C /usr/share @@ -47,7 +47,7 @@ jobs: steps: - name: Install shellspec run: curl -fsSL https://git.io/shellspec | sh -s 0.28.1 --yes - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Run Shellspec run: | cd tests/shell diff --git a/.github/workflows/spi_release.yml b/.github/workflows/spi_release.yml index 8ad74b3e4bb..6398edca412 100644 --- a/.github/workflows/spi_release.yml +++ b/.github/workflows/spi_release.yml @@ -37,15 +37,15 @@ jobs: runs-on: ubuntu-latest if: github.event_name == 'pull_request' && needs.check-secrets.outputs.available == 'true' steps: - - uses: actions/checkout@v3 - - uses: actions/setup-java@v3 + - uses: actions/checkout@v4 + - uses: actions/setup-java@v4 with: java-version: '17' distribution: 'adopt' server-id: ossrh server-username: MAVEN_USERNAME server-password: MAVEN_PASSWORD - - uses: actions/cache@v2 + - uses: actions/cache@v4 with: path: ~/.m2 key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} @@ -63,12 +63,12 @@ jobs: runs-on: ubuntu-latest if: github.event_name == 'push' && needs.check-secrets.outputs.available == 'true' steps: - - uses: actions/checkout@v3 - - uses: actions/setup-java@v3 + - uses: actions/checkout@v4 + - uses: actions/setup-java@v4 with: java-version: '17' distribution: 'adopt' - - uses: actions/cache@v2 + - uses: actions/cache@v4 with: path: ~/.m2 key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} @@ -76,7 +76,7 @@ jobs: # Running setup-java again overwrites the settings.xml - IT'S MANDATORY TO DO THIS SECOND SETUP!!! - name: Set up Maven Central Repository - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: java-version: '17' distribution: 'adopt' diff --git a/README.md b/README.md index 77720453d5f..2303c001d2c 100644 --- a/README.md +++ b/README.md @@ -1,20 +1,81 @@ Dataverse® =============== -Dataverse is an [open source][] software platform for sharing, finding, citing, and preserving research data (developed by the [Dataverse team](https://dataverse.org/about) at the [Institute for Quantitative Social Science](https://iq.harvard.edu/) and the [Dataverse community][]). +![Dataverse-logo](https://github.com/IQSS/dataverse-frontend/assets/7512607/6c4d79e4-7be5-4102-88bd-dfa167dc79d3) -[dataverse.org][] is our home on the web and shows a map of Dataverse installations around the world, a list of [features][], [integrations][] that have been made possible through [REST APIs][], our [project board][], our development [roadmap][], and more. +## Table of Contents -We maintain a demo site at [demo.dataverse.org][] which you are welcome to use for testing and evaluating Dataverse. +1. [โ“ What is Dataverse?](#what-is-dataverse) +2. [โœ” Try Dataverse](#try-dataverse) +3. [๐ŸŒ Features, Integrations, Roadmaps, and More](#website) +4. [๐Ÿ“ฅ Installation](#installation) +5. [๐Ÿ˜ Community and Support](#community-and-support) +6. [๐Ÿง‘โ€๐Ÿ’ป๏ธ Contributing](#contributing) +7. [โš–๏ธ Legal Information](#legal-informations) -To install Dataverse, please see our [Installation Guide][] which will prompt you to download our [latest release][]. Docker users should consult the [Container Guide][]. + -To discuss Dataverse with the community, please join our [mailing list][], participate in a [community call][], chat with us at [chat.dataverse.org][], or attend our annual [Dataverse Community Meeting][]. +## โ“ What is Dataverse? -We love contributors! Please see our [Contributing Guide][] for ways you can help. +Welcome to Dataverseยฎ, the [open source][] software platform designed for sharing, finding, citing, and preserving research data. Developed by the Dataverse team at the [Institute for Quantitative Social Science](https://iq.harvard.edu/) and the [Dataverse community][], our platform makes it easy for research organizations to host, manage, and share their data with the world. + + + +## โœ” Try Dataverse + +We invite you to explore our demo site at [demo.dataverse.org][]. This site is ideal for testing and evaluating Dataverse in a risk-free environment. + + + +## ๐ŸŒ Features, Integrations, Roadmaps, and More + +Visit [dataverse.org][], our home on the web, for a comprehensive overview of Dataverse. Here, you will find: + +- An interactive map showcasing Dataverse installations worldwide. +- A detailed list of [features][]. +- Information on [integrations][] that have been made possible through our [REST APIs][]. +- Our [project board][] and development [roadmap][]. +- News, events, and more. + + + +## ๐Ÿ“ฅ Installation + +Ready to get started? Follow our [Installation Guide][] to download and install the latest release of Dataverse. + +If you are using Docker, please refer to our [Container Guide][] for detailed instructions. + + + +## ๐Ÿ˜ Community and Support + +Engage with the vibrant Dataverse community through various channels: + +- **[Mailing List][]**: Join the conversation on our [mailing list][]. +- **[Community Calls][]**: Participate in our regular [community calls][] to discuss new features, ask questions, and share your experiences. +- **[Chat][]**: Connect with us and other users in real-time at [dataverse.zulipchat.com][]. +- **[Dataverse Community Meeting][]**: Attend our annual [Dataverse Community Meeting][] to network, learn, and collaborate with peers and experts. +- **[DataverseTV][]**: Watch the video content from the Dataverse community on [DataverseTV][] and on [Harvard's IQSS YouTube channel][]. + + +## ๐Ÿง‘โ€๐Ÿ’ป๏ธ Contribute to Dataverse + +We love contributors! Whether you are a developer, researcher, or enthusiast, there are many ways you can help. + +Visit our [Contributing Guide][] to learn how you can get involved. + +Join us in building and enhancing Dataverse to make research data more accessible and impactful. Your support and participation are crucial to our success! + + +## โš–๏ธ Legal Information Dataverse is a trademark of President and Fellows of Harvard College and is registered in the United States. +--- +For more detailed information, visit our website at [dataverse.org][]. + +Feel free to [reach out] with any questions or feedback. Happy researching! + [![Dataverse Project logo](src/main/webapp/resources/images/dataverseproject_logo.jpg "Dataverse Project")](http://dataverse.org) [![API Test Status](https://jenkins.dataverse.org/buildStatus/icon?job=IQSS-dataverse-develop&subject=API%20Test%20Status)](https://jenkins.dataverse.org/job/IQSS-dataverse-develop/) @@ -37,6 +98,11 @@ Dataverse is a trademark of President and Fellows of Harvard College and is regi [Contributing Guide]: CONTRIBUTING.md [mailing list]: https://groups.google.com/group/dataverse-community [community call]: https://dataverse.org/community-calls -[chat.dataverse.org]: https://chat.dataverse.org +[Chat]: https://dataverse.zulipchat.com +[dataverse.zulipchat.com]: https://dataverse.zulipchat.com [Dataverse Community Meeting]: https://dataverse.org/events [open source]: LICENSE.md +[community calls]: https://dataverse.org/community-calls +[DataverseTV]: https://dataverse.org/dataversetv +[Harvard's IQSS YouTube channel]: https://www.youtube.com/@iqssatharvarduniversity8672 +[reach out]: https://dataverse.org/contact diff --git a/conf/keycloak/test-realm.json b/conf/keycloak/test-realm.json index efe71cc5d29..2e5ed1c4d69 100644 --- a/conf/keycloak/test-realm.json +++ b/conf/keycloak/test-realm.json @@ -45,287 +45,411 @@ "quickLoginCheckMilliSeconds" : 1000, "maxDeltaTimeSeconds" : 43200, "failureFactor" : 30, - "roles" : { - "realm" : [ { - "id" : "075daee1-5ab2-44b5-adbf-fa49a3da8305", - "name" : "uma_authorization", - "description" : "${role_uma_authorization}", - "composite" : false, - "clientRole" : false, - "containerId" : "80a7e04b-a2b5-4891-a2d1-5ad4e915f983", - "attributes" : { } - }, { - "id" : "b4ff9091-ddf9-4536-b175-8cfa3e331d71", - "name" : "default-roles-test", - "description" : "${role_default-roles}", - "composite" : true, - "composites" : { - "realm" : [ "offline_access", "uma_authorization" ], - "client" : { - "account" : [ "view-profile", "manage-account" ] - } + "roles": { + "realm": [ + { + "id": "075daee1-5ab2-44b5-adbf-fa49a3da8305", + "name": "uma_authorization", + "description": "${role_uma_authorization}", + "composite": false, + "clientRole": false, + "containerId": "80a7e04b-a2b5-4891-a2d1-5ad4e915f983", + "attributes": {} }, - "clientRole" : false, - "containerId" : "80a7e04b-a2b5-4891-a2d1-5ad4e915f983", - "attributes" : { } - }, { - "id" : "e6d31555-6be6-4dee-bc6a-40a53108e4c2", - "name" : "offline_access", - "description" : "${role_offline-access}", - "composite" : false, - "clientRole" : false, - "containerId" : "80a7e04b-a2b5-4891-a2d1-5ad4e915f983", - "attributes" : { } - } ], - "client" : { - "realm-management" : [ { - "id" : "1955bd12-5f86-4a74-b130-d68a8ef6f0ee", - "name" : "impersonation", - "description" : "${role_impersonation}", - "composite" : false, - "clientRole" : true, - "containerId" : "dada0ae8-ee9f-415a-9685-42da7c563660", - "attributes" : { } - }, { - "id" : "1109c350-9ab1-426c-9876-ef67d4310f35", - "name" : "view-authorization", - "description" : "${role_view-authorization}", - "composite" : false, - "clientRole" : true, - "containerId" : "dada0ae8-ee9f-415a-9685-42da7c563660", - "attributes" : { } - }, { - "id" : "980c3fd3-1ae3-4b8f-9a00-d764c939035f", - "name" : "query-users", - "description" : "${role_query-users}", - "composite" : false, - "clientRole" : true, - "containerId" : "dada0ae8-ee9f-415a-9685-42da7c563660", - "attributes" : { } - }, { - "id" : "5363e601-0f9d-4633-a8c8-28cb0f859b7b", - "name" : "query-groups", - "description" : "${role_query-groups}", - "composite" : false, - "clientRole" : true, - "containerId" : "dada0ae8-ee9f-415a-9685-42da7c563660", - "attributes" : { } - }, { - "id" : "59aa7992-ad78-48db-868a-25d6e1d7db50", - "name" : "realm-admin", - "description" : "${role_realm-admin}", - "composite" : true, - "composites" : { - "client" : { - "realm-management" : [ "impersonation", "view-authorization", "query-users", "query-groups", "manage-clients", "manage-realm", "view-identity-providers", "query-realms", "manage-authorization", "manage-identity-providers", "manage-users", "view-users", "view-realm", "create-client", "view-clients", "manage-events", "query-clients", "view-events" ] + { + "id": "b4ff9091-ddf9-4536-b175-8cfa3e331d71", + "name": "default-roles-test", + "description": "${role_default-roles}", + "composite": true, + "composites": { + "realm": [ + "offline_access", + "uma_authorization" + ], + "client": { + "account": [ + "view-profile", + "manage-account" + ] } }, - "clientRole" : true, - "containerId" : "dada0ae8-ee9f-415a-9685-42da7c563660", - "attributes" : { } - }, { - "id" : "112f53c2-897d-4c01-81db-b8dc10c5b995", - "name" : "manage-clients", - "description" : "${role_manage-clients}", - "composite" : false, - "clientRole" : true, - "containerId" : "dada0ae8-ee9f-415a-9685-42da7c563660", - "attributes" : { } - }, { - "id" : "c7f57bbd-ef32-4a64-9888-7b8abd90777a", - "name" : "manage-realm", - "description" : "${role_manage-realm}", - "composite" : false, - "clientRole" : true, - "containerId" : "dada0ae8-ee9f-415a-9685-42da7c563660", - "attributes" : { } - }, { - "id" : "8885dac8-0af3-45af-94ce-eff5e801bb80", - "name" : "view-identity-providers", - "description" : "${role_view-identity-providers}", - "composite" : false, - "clientRole" : true, - "containerId" : "dada0ae8-ee9f-415a-9685-42da7c563660", - "attributes" : { } - }, { - "id" : "2673346c-b0ef-4e01-8a90-be03866093af", - "name" : "manage-authorization", - "description" : "${role_manage-authorization}", - "composite" : false, - "clientRole" : true, - "containerId" : "dada0ae8-ee9f-415a-9685-42da7c563660", - "attributes" : { } - }, { - "id" : "b7182885-9e57-445f-8dae-17c16eb31b5d", - "name" : "manage-identity-providers", - "description" : "${role_manage-identity-providers}", - "composite" : false, - "clientRole" : true, - "containerId" : "dada0ae8-ee9f-415a-9685-42da7c563660", - "attributes" : { } - }, { - "id" : "ba7bfe0c-cb07-4a47-b92c-b8132b57e181", - "name" : "manage-users", - "description" : "${role_manage-users}", - "composite" : false, - "clientRole" : true, - "containerId" : "dada0ae8-ee9f-415a-9685-42da7c563660", - "attributes" : { } - }, { - "id" : "13a8f0fc-647d-4bfe-b525-73956898e550", - "name" : "query-realms", - "description" : "${role_query-realms}", - "composite" : false, - "clientRole" : true, - "containerId" : "dada0ae8-ee9f-415a-9685-42da7c563660", - "attributes" : { } - }, { - "id" : "ef4c57dc-78c2-4f9a-8d2b-0e97d46fc842", - "name" : "view-realm", - "description" : "${role_view-realm}", - "composite" : false, - "clientRole" : true, - "containerId" : "dada0ae8-ee9f-415a-9685-42da7c563660", - "attributes" : { } - }, { - "id" : "2875da34-006c-4b7f-bfc8-9ae8e46af3a2", - "name" : "view-users", - "description" : "${role_view-users}", - "composite" : true, - "composites" : { - "client" : { - "realm-management" : [ "query-users", "query-groups" ] + "clientRole": false, + "containerId": "80a7e04b-a2b5-4891-a2d1-5ad4e915f983", + "attributes": {} + }, + { + "id": "131ff85b-0c25-491b-8e13-dde779ec0854", + "name": "admin", + "description": "", + "composite": true, + "composites": { + "client": { + "realm-management": [ + "impersonation", + "view-authorization", + "query-users", + "manage-realm", + "view-identity-providers", + "manage-authorization", + "view-clients", + "manage-events", + "query-clients", + "view-events", + "query-groups", + "realm-admin", + "manage-clients", + "query-realms", + "manage-identity-providers", + "manage-users", + "view-users", + "view-realm", + "create-client" + ], + "broker": [ + "read-token" + ], + "account": [ + "delete-account", + "manage-consent", + "view-consent", + "view-applications", + "view-groups", + "manage-account-links", + "view-profile", + "manage-account" + ] } }, - "clientRole" : true, - "containerId" : "dada0ae8-ee9f-415a-9685-42da7c563660", - "attributes" : { } - }, { - "id" : "c8c8f7dc-876b-4263-806f-3329f7cd5fd3", - "name" : "create-client", - "description" : "${role_create-client}", - "composite" : false, - "clientRole" : true, - "containerId" : "dada0ae8-ee9f-415a-9685-42da7c563660", - "attributes" : { } - }, { - "id" : "21b84f90-5a9a-4845-a7ba-bbd98ac0fcc4", - "name" : "view-clients", - "description" : "${role_view-clients}", - "composite" : true, - "composites" : { - "client" : { - "realm-management" : [ "query-clients" ] - } + "clientRole": false, + "containerId": "80a7e04b-a2b5-4891-a2d1-5ad4e915f983", + "attributes": {} + }, + { + "id": "e6d31555-6be6-4dee-bc6a-40a53108e4c2", + "name": "offline_access", + "description": "${role_offline-access}", + "composite": false, + "clientRole": false, + "containerId": "80a7e04b-a2b5-4891-a2d1-5ad4e915f983", + "attributes": {} + } + ], + "client": { + "realm-management": [ + { + "id": "1955bd12-5f86-4a74-b130-d68a8ef6f0ee", + "name": "impersonation", + "description": "${role_impersonation}", + "composite": false, + "clientRole": true, + "containerId": "dada0ae8-ee9f-415a-9685-42da7c563660", + "attributes": {} }, - "clientRole" : true, - "containerId" : "dada0ae8-ee9f-415a-9685-42da7c563660", - "attributes" : { } - }, { - "id" : "6fd64c94-d663-4501-ad77-0dcf8887d434", - "name" : "manage-events", - "description" : "${role_manage-events}", - "composite" : false, - "clientRole" : true, - "containerId" : "dada0ae8-ee9f-415a-9685-42da7c563660", - "attributes" : { } - }, { - "id" : "b321927a-023c-4d2a-99ad-24baf7ff6d83", - "name" : "query-clients", - "description" : "${role_query-clients}", - "composite" : false, - "clientRole" : true, - "containerId" : "dada0ae8-ee9f-415a-9685-42da7c563660", - "attributes" : { } - }, { - "id" : "2fc21160-78de-457b-8594-e5c76cde1d5e", - "name" : "view-events", - "description" : "${role_view-events}", - "composite" : false, - "clientRole" : true, - "containerId" : "dada0ae8-ee9f-415a-9685-42da7c563660", - "attributes" : { } - } ], - "test" : [ ], - "security-admin-console" : [ ], - "admin-cli" : [ ], - "account-console" : [ ], - "broker" : [ { - "id" : "07ee59b5-dca6-48fb-83d4-2994ef02850e", - "name" : "read-token", - "description" : "${role_read-token}", - "composite" : false, - "clientRole" : true, - "containerId" : "b57d62bb-77ff-42bd-b8ff-381c7288f327", - "attributes" : { } - } ], - "account" : [ { - "id" : "17d2f811-7bdf-4c73-83b4-1037001797b8", - "name" : "view-applications", - "description" : "${role_view-applications}", - "composite" : false, - "clientRole" : true, - "containerId" : "77f8127a-261e-4cd8-a77d-b74a389f7fd4", - "attributes" : { } - }, { - "id" : "d1ff44f9-419e-42fd-98e8-1add1169a972", - "name" : "delete-account", - "description" : "${role_delete-account}", - "composite" : false, - "clientRole" : true, - "containerId" : "77f8127a-261e-4cd8-a77d-b74a389f7fd4", - "attributes" : { } - }, { - "id" : "14c23a18-ae2d-43c9-b0c0-aaf6e0c7f5b0", - "name" : "manage-account-links", - "description" : "${role_manage-account-links}", - "composite" : false, - "clientRole" : true, - "containerId" : "77f8127a-261e-4cd8-a77d-b74a389f7fd4", - "attributes" : { } - }, { - "id" : "6fbe58af-d2fe-4d66-95fe-a2e8a818cb55", - "name" : "view-profile", - "description" : "${role_view-profile}", - "composite" : false, - "clientRole" : true, - "containerId" : "77f8127a-261e-4cd8-a77d-b74a389f7fd4", - "attributes" : { } - }, { - "id" : "bdfd02bc-6f6a-47d2-82bc-0ca52d78ff48", - "name" : "manage-consent", - "description" : "${role_manage-consent}", - "composite" : true, - "composites" : { - "client" : { - "account" : [ "view-consent" ] - } + { + "id": "1109c350-9ab1-426c-9876-ef67d4310f35", + "name": "view-authorization", + "description": "${role_view-authorization}", + "composite": false, + "clientRole": true, + "containerId": "dada0ae8-ee9f-415a-9685-42da7c563660", + "attributes": {} }, - "clientRole" : true, - "containerId" : "77f8127a-261e-4cd8-a77d-b74a389f7fd4", - "attributes" : { } - }, { - "id" : "782f3b0c-a17b-4a87-988b-1a711401f3b0", - "name" : "manage-account", - "description" : "${role_manage-account}", - "composite" : true, - "composites" : { - "client" : { - "account" : [ "manage-account-links" ] - } + { + "id": "980c3fd3-1ae3-4b8f-9a00-d764c939035f", + "name": "query-users", + "description": "${role_query-users}", + "composite": false, + "clientRole": true, + "containerId": "dada0ae8-ee9f-415a-9685-42da7c563660", + "attributes": {} + }, + { + "id": "5363e601-0f9d-4633-a8c8-28cb0f859b7b", + "name": "query-groups", + "description": "${role_query-groups}", + "composite": false, + "clientRole": true, + "containerId": "dada0ae8-ee9f-415a-9685-42da7c563660", + "attributes": {} + }, + { + "id": "59aa7992-ad78-48db-868a-25d6e1d7db50", + "name": "realm-admin", + "description": "${role_realm-admin}", + "composite": true, + "composites": { + "client": { + "realm-management": [ + "impersonation", + "view-authorization", + "query-users", + "query-groups", + "manage-clients", + "manage-realm", + "view-identity-providers", + "query-realms", + "manage-authorization", + "manage-identity-providers", + "manage-users", + "view-users", + "view-realm", + "create-client", + "view-clients", + "manage-events", + "query-clients", + "view-events" + ] + } + }, + "clientRole": true, + "containerId": "dada0ae8-ee9f-415a-9685-42da7c563660", + "attributes": {} + }, + { + "id": "112f53c2-897d-4c01-81db-b8dc10c5b995", + "name": "manage-clients", + "description": "${role_manage-clients}", + "composite": false, + "clientRole": true, + "containerId": "dada0ae8-ee9f-415a-9685-42da7c563660", + "attributes": {} + }, + { + "id": "c7f57bbd-ef32-4a64-9888-7b8abd90777a", + "name": "manage-realm", + "description": "${role_manage-realm}", + "composite": false, + "clientRole": true, + "containerId": "dada0ae8-ee9f-415a-9685-42da7c563660", + "attributes": {} + }, + { + "id": "8885dac8-0af3-45af-94ce-eff5e801bb80", + "name": "view-identity-providers", + "description": "${role_view-identity-providers}", + "composite": false, + "clientRole": true, + "containerId": "dada0ae8-ee9f-415a-9685-42da7c563660", + "attributes": {} + }, + { + "id": "2673346c-b0ef-4e01-8a90-be03866093af", + "name": "manage-authorization", + "description": "${role_manage-authorization}", + "composite": false, + "clientRole": true, + "containerId": "dada0ae8-ee9f-415a-9685-42da7c563660", + "attributes": {} + }, + { + "id": "b7182885-9e57-445f-8dae-17c16eb31b5d", + "name": "manage-identity-providers", + "description": "${role_manage-identity-providers}", + "composite": false, + "clientRole": true, + "containerId": "dada0ae8-ee9f-415a-9685-42da7c563660", + "attributes": {} + }, + { + "id": "ba7bfe0c-cb07-4a47-b92c-b8132b57e181", + "name": "manage-users", + "description": "${role_manage-users}", + "composite": false, + "clientRole": true, + "containerId": "dada0ae8-ee9f-415a-9685-42da7c563660", + "attributes": {} }, - "clientRole" : true, - "containerId" : "77f8127a-261e-4cd8-a77d-b74a389f7fd4", - "attributes" : { } - }, { - "id" : "8a3bfe15-66d9-4f3d-83ac-801d682d42b0", - "name" : "view-consent", - "description" : "${role_view-consent}", - "composite" : false, - "clientRole" : true, - "containerId" : "77f8127a-261e-4cd8-a77d-b74a389f7fd4", - "attributes" : { } - } ] + { + "id": "13a8f0fc-647d-4bfe-b525-73956898e550", + "name": "query-realms", + "description": "${role_query-realms}", + "composite": false, + "clientRole": true, + "containerId": "dada0ae8-ee9f-415a-9685-42da7c563660", + "attributes": {} + }, + { + "id": "ef4c57dc-78c2-4f9a-8d2b-0e97d46fc842", + "name": "view-realm", + "description": "${role_view-realm}", + "composite": false, + "clientRole": true, + "containerId": "dada0ae8-ee9f-415a-9685-42da7c563660", + "attributes": {} + }, + { + "id": "2875da34-006c-4b7f-bfc8-9ae8e46af3a2", + "name": "view-users", + "description": "${role_view-users}", + "composite": true, + "composites": { + "client": { + "realm-management": [ + "query-users", + "query-groups" + ] + } + }, + "clientRole": true, + "containerId": "dada0ae8-ee9f-415a-9685-42da7c563660", + "attributes": {} + }, + { + "id": "c8c8f7dc-876b-4263-806f-3329f7cd5fd3", + "name": "create-client", + "description": "${role_create-client}", + "composite": false, + "clientRole": true, + "containerId": "dada0ae8-ee9f-415a-9685-42da7c563660", + "attributes": {} + }, + { + "id": "21b84f90-5a9a-4845-a7ba-bbd98ac0fcc4", + "name": "view-clients", + "description": "${role_view-clients}", + "composite": true, + "composites": { + "client": { + "realm-management": [ + "query-clients" + ] + } + }, + "clientRole": true, + "containerId": "dada0ae8-ee9f-415a-9685-42da7c563660", + "attributes": {} + }, + { + "id": "6fd64c94-d663-4501-ad77-0dcf8887d434", + "name": "manage-events", + "description": "${role_manage-events}", + "composite": false, + "clientRole": true, + "containerId": "dada0ae8-ee9f-415a-9685-42da7c563660", + "attributes": {} + }, + { + "id": "b321927a-023c-4d2a-99ad-24baf7ff6d83", + "name": "query-clients", + "description": "${role_query-clients}", + "composite": false, + "clientRole": true, + "containerId": "dada0ae8-ee9f-415a-9685-42da7c563660", + "attributes": {} + }, + { + "id": "2fc21160-78de-457b-8594-e5c76cde1d5e", + "name": "view-events", + "description": "${role_view-events}", + "composite": false, + "clientRole": true, + "containerId": "dada0ae8-ee9f-415a-9685-42da7c563660", + "attributes": {} + } + ], + "test": [], + "security-admin-console": [], + "admin-cli": [], + "account-console": [], + "broker": [ + { + "id": "07ee59b5-dca6-48fb-83d4-2994ef02850e", + "name": "read-token", + "description": "${role_read-token}", + "composite": false, + "clientRole": true, + "containerId": "b57d62bb-77ff-42bd-b8ff-381c7288f327", + "attributes": {} + } + ], + "account": [ + { + "id": "17d2f811-7bdf-4c73-83b4-1037001797b8", + "name": "view-applications", + "description": "${role_view-applications}", + "composite": false, + "clientRole": true, + "containerId": "77f8127a-261e-4cd8-a77d-b74a389f7fd4", + "attributes": {} + }, + { + "id": "f5918d56-bd4d-4035-8fa7-8622075ed690", + "name": "view-groups", + "description": "${role_view-groups}", + "composite": false, + "clientRole": true, + "containerId": "77f8127a-261e-4cd8-a77d-b74a389f7fd4", + "attributes": {} + }, + { + "id": "d1ff44f9-419e-42fd-98e8-1add1169a972", + "name": "delete-account", + "description": "${role_delete-account}", + "composite": false, + "clientRole": true, + "containerId": "77f8127a-261e-4cd8-a77d-b74a389f7fd4", + "attributes": {} + }, + { + "id": "14c23a18-ae2d-43c9-b0c0-aaf6e0c7f5b0", + "name": "manage-account-links", + "description": "${role_manage-account-links}", + "composite": false, + "clientRole": true, + "containerId": "77f8127a-261e-4cd8-a77d-b74a389f7fd4", + "attributes": {} + }, + { + "id": "6fbe58af-d2fe-4d66-95fe-a2e8a818cb55", + "name": "view-profile", + "description": "${role_view-profile}", + "composite": false, + "clientRole": true, + "containerId": "77f8127a-261e-4cd8-a77d-b74a389f7fd4", + "attributes": {} + }, + { + "id": "bdfd02bc-6f6a-47d2-82bc-0ca52d78ff48", + "name": "manage-consent", + "description": "${role_manage-consent}", + "composite": true, + "composites": { + "client": { + "account": [ + "view-consent" + ] + } + }, + "clientRole": true, + "containerId": "77f8127a-261e-4cd8-a77d-b74a389f7fd4", + "attributes": {} + }, + { + "id": "782f3b0c-a17b-4a87-988b-1a711401f3b0", + "name": "manage-account", + "description": "${role_manage-account}", + "composite": true, + "composites": { + "client": { + "account": [ + "manage-account-links" + ] + } + }, + "clientRole": true, + "containerId": "77f8127a-261e-4cd8-a77d-b74a389f7fd4", + "attributes": {} + }, + { + "id": "8a3bfe15-66d9-4f3d-83ac-801d682d42b0", + "name": "view-consent", + "description": "${role_view-consent}", + "composite": false, + "clientRole": true, + "containerId": "77f8127a-261e-4cd8-a77d-b74a389f7fd4", + "attributes": {} + } + ] } }, "groups" : [ { @@ -409,7 +533,7 @@ } ], "disableableCredentialTypes" : [ ], "requiredActions" : [ ], - "realmRoles" : [ "default-roles-test" ], + "realmRoles" : [ "default-roles-test", "admin" ], "notBefore" : 0, "groups" : [ "/admins" ] }, { diff --git a/conf/solr/schema.xml b/conf/solr/schema.xml index d5c789c7189..50835957b04 100644 --- a/conf/solr/schema.xml +++ b/conf/solr/schema.xml @@ -38,36 +38,37 @@ catchall "text" field, and use that for searching. --> - + @@ -115,12 +116,12 @@ - - - - - - + + + + + + @@ -167,6 +168,8 @@ + + @@ -201,7 +204,10 @@ - + + + + @@ -212,7 +218,7 @@ - + @@ -250,6 +256,21 @@ WARNING: Do not remove the following include guards if you intend to use the neat helper scripts we provide. --> + + + + + + + + + + + + + + + @@ -273,38 +294,38 @@ - - - + + + - - - - + + + + - - - + + + - - + + - + - - - + + + - + @@ -312,7 +333,7 @@ - + @@ -324,12 +345,12 @@ - + - + @@ -349,19 +370,19 @@ - + - + - + @@ -385,28 +406,28 @@ - - + + - + - + - - + + + - @@ -492,6 +513,21 @@ WARNING: Do not remove the following include guards if you intend to use the neat helper scripts we provide. --> + + + + + + + + + + + + + + + @@ -534,12 +570,12 @@ - + @@ -570,8 +606,8 @@ - + @@ -595,9 +631,9 @@ - + @@ -627,13 +663,13 @@ - - + + - + @@ -645,10 +681,10 @@ + - @@ -751,7 +787,7 @@ - - + + @@ -815,7 +851,9 @@ - + + + diff --git a/conf/solr/solrconfig.xml b/conf/solr/solrconfig.xml index 34386375fe1..97965bd77d7 100644 --- a/conf/solr/solrconfig.xml +++ b/conf/solr/solrconfig.xml @@ -35,52 +35,7 @@ that you fully re-index after changing this setting as it can affect both how text is indexed and queried. --> - 9.7 - - - - - - - - + 9.11 ${solr.ulog.dir:} - ${solr.ulog.numVersionBuckets:65536} ${solr.max.booleanClauses:1024} + + ${solr.query.minPrefixLength:-1} + @@ -494,23 +457,6 @@ --> 200 - - - + processor="uuid,remove-blank,field-name-mutating,max-fields,parse-boolean,parse-long,parse-double,parse-date,add-schema-fields"> diff --git a/doc/release-notes/6.6-release-notes.md b/doc/release-notes/6.6-release-notes.md new file mode 100644 index 00000000000..751c471a0a3 --- /dev/null +++ b/doc/release-notes/6.6-release-notes.md @@ -0,0 +1,549 @@ +# Dataverse 6.6 + +Please note: To read these instructions in full, please go to https://github.com/IQSS/dataverse/releases/tag/v6.6 rather than the [list of releases](https://github.com/IQSS/dataverse/releases), which will cut them off. + +This release brings new features, enhancements, and bug fixes to Dataverse. Thank you to all of the community members who contributed code, suggestions, bug reports, and other assistance across the project! + +## Release Highlights + +Highlights for Dataverse 6.6 include: + +- metadata fields can be "display on create" per collection +- ORCIDs linked to accounts +- version notes +- harvesting from DataCite +- citations using Citation Style Language (CSL) +- license metadata enhancements +- metadata fields now support range searches (dates, integers, etc.) +- more accurate search highlighting +- collections can be moved by using the superuser dashboard +- new 3D Objects metadata block +- new Archival metadata block (experimental) +- optionally prevent publishing of datasets without files +- Signposting output now contains links to all dataset metadata export formats +- infrastructure updates (Payara and Solr) + +In a recent community call, we talked about many of these highlights if you'd like to watch the [video](https://harvard.zoom.us/rec/share/Ir5CkFHkzoya9b5Nk69rLFUpTyGics3-KGLl9WITSLMy4ezHRsB8CnY22cUNg2g.JPpxrjzHMeCii_zO) (around 22:30). + +## Features Added + +### Metadata Fields Can Be "Display on Create" Per Collection + +Collection administrators can now configure which metadata fields appear during dataset creation through the `displayOnCreate` property, even when fields are not required. This provides greater control over metadata visibility and can help improve metadata completeness. + +Currently this feature can only be configured [via API](https://guides.dataverse.org/en/6.6/api/native-api.html#update-collection-input-levels), but a UI implementation is planned in #11221. See #10476, #11224, and #11312. + +### ORCIDs Linked to Accounts + +Dataverse now includes improved integration with ORCID, supported through a grant to GDCC from the ([ORCID Global Participation Fund](https://info.orcid.org/global-participation-fund-announces-fourth-round-of-awardees/)). + +Specifically, Dataverse users can now link their Dataverse account with their ORCID profile. Previously, this was only available to users who logged in with ORCID. Once linked, Dataverse will automatically prepopulate their ORCID to their author metadata when they create a dataset. + +This functionality leverages Dataverse's existing support for login via ORCID, but can be turned on independently of it. If ORCID login is enabled, the user's ORCID will automatically be added to their profile. If the user has logged in via some other mechanism, they are able to click a button to initiate a similar authentication process in which the user must login to their ORCID account and approve the connection. + +Feedback from installations that enable this functionality is requested and we expect that updates can be made in the next Dataverse release. + +See the [User Guide](http://guides.dataverse.org/en/6.6/user/account.html#linking-orcid-with-your-account-profile), [Installation Guide](http://guides.dataverse.org/en/6.6/installation/orcid.html), #7284, and #11222. + +### Version Notes + +Dataverse now supports the option of adding a version note before or during the publication of a dataset. These notes can be used, for example, to indicate why a version was created or how it differs from the prior version. Whether this feature is enabled is controlled by the flag `dataverse.feature.enable-version-note`. Version notes are shown in the user interface (in the dataset page version table), indexed (as `versionNote`), available via the API, and have been added to the JSON, DDI, DataCite, and OAI-ORE exports. + +With the addition of this feature, work has been done to clean-up and rename fields that have been used for specifying the reason for deaccessioning a dataset and providing an optional link to a non-Dataverse location where the dataset still can be found. The former was listed in some JSON-based API calls and exports as "versionNote" and is now "deaccessionNote", while the latter was referred to as "archiveNote" and is now "deaccessionLink". + +Further, some database consolidation has been done to combine the deaccessionlink and archivenote fields, which appear to have both been used for the same purpose. The deaccessionlink database field is older and also was not displayed in the current UI. Going forward, only the deaccessionlink column exists. + +See the [User Guide](https://guides.dataverse.org/en/6.6/user/dataset-management.html#data-provenance), [API Guide](https://guides.dataverse.org/en/6.6/api/native-api.html#dataset-version-notes) #8431, and #11068. + +### OAI-PMH Harvesting from DataCite + +DataCite maintains an OAI server () that serves records for every DOI they have registered. There's been a lot of interest in the community in being able to harvest from them. This way, it will be possible to harvest metadata from institution X even if the institution X does not maintain an OAI server of their own, if they happen to register their DOIs with DataCite. One extra element of this harvesting model that makes it especially powerful and flexible is the DataCite's concept of a "dynamic OAI set": a harvester is not limited to harvesting the pre-defined set of ALL the records registered by the institution X, but can instead harvest virtually any arbitrary subset thereof; any query that the DataCite search API understands can be used as an OAI set. The feature is already in use at Harvard Dataverse, as a beta version patch. + +For various reasons, in order to take advantage of this feature harvesting clients must be created using the `/api/harvest/clients` API. Once configured however, harvests can be run from the Harvesting Clients control panel in the UI. + +DataCite-harvesting clients must be configured with 2 new feature flags, `useListRecords` and `useOaiIdentifiersAsPids` (added in Dataverse 6.5). Note that these features may be of use when harvesting from other sources, not just from DataCite. + +See the [Admin Guide](http://guides.dataverse.org/en/6.6/admin/harvestclients.html#harvesting-from-datacite), [API Guide](http://guides.dataverse.org/en/6.6/api/native-api.html#harvesting-from-datacite), #10909, and #11011. + +### Citations Using Citation Style Language (CSL) + +This release adds support for generating citations in any of the standard independent formats specified using the [Citation Style Language](https://citationstyles.org). + +The CSL formats are available to copy/paste if you click "Cite Dataset" and then "View Styled Citations" on the dataset page. An API call to retrieve a dataset citation in EndNote, RIS, BibTeX, and CSLJson format has also been added. The first three have been available as downloads from the UI (CSLJson is not) but have not been directly accessible via API until now. The CSLJson format is new to Dataverse and can be used with open source libraries to generate all of the other CSL-style citations. + +Admins can use a new `dataverse.csl.common-styles` setting to highlight commonly used styles. Common styles are listed in the pop-up, while others can be found by type-ahead search in a list of 1000+ options. + +See the [User Guide](http://guides.dataverse.org/en/6.6/user/find-use-data.html#cite-data), [Settings](http://guides.dataverse.org/en/6.6/installation/config.html#dataverse-csl-common-styles), [API Guide](http://guides.dataverse.org/en/6.6/api/native-api.html#get-citation-in-other-formats), and #11163. + +### License Metadata Enhancements + +- Added new fields to licenses: rightsIdentifier, rightsIdentifierScheme, schemeUri, languageCode. See JSON files under [Adding Licenses](https://guides.dataverse.org/en/6.6/installation/config.html#adding-licenses) in the guides +- Updated DataCite metadata export to include rightsIdentifier, rightsIdentifierScheme, and schemeUri consistent with the DataCite 4.5 schema and examples +- Enhanced metadata exports to include all new license fields +- Existing licenses from the example set included with Dataverse will be automatically updated with new fields +- Existing API calls support the new optional fields + +See below for upgrade instructions. See also #10883 and #11232. + +### Range Search + +This release enhances how numerical and date fields are indexed in Solr. Previously, all fields were indexed as English text (`text_en`), but with this update: + +* Integer fields are indexed as `plong` +* Float fields are indexed as `pdouble` +* Date fields are indexed as `date_range` (`solr.DateRangeField`) + +This change enables range queries when searching from both the UI and the API, such as `dateOfDeposit:[2000-01-01 TO 2014-12-31]` or `targetSampleActualSize:[25 TO 50]`. See below for a full list of fields that now support range search. + +Additionally, search result highlighting is now more accurate, ensuring that only fields relevant to the query are highlighted in search results. If the query is specifically limited to certain fields, the highlighting is now limited to those fields as well. See #10887. + +Specifically, the following fields were updated: + +- coverage.Depth +- coverage.ObjectCount +- coverage.ObjectDensity +- coverage.Redshift.MaximumValue +- coverage.Redshift.MinimumValue +- coverage.RedshiftValue +- coverage.SkyFraction +- coverage.Spectral.CentralWavelength +- coverage.Spectral.MaximumWavelength +- coverage.Spectral.MinimumWavelength +- coverage.Temporal.StartTime +- coverage.Temporal.StopTime +- dateOfCollectionEnd +- dateOfCollectionStart +- dateOfDeposit +- distributionDate +- dsDescriptionDate +- journalPubDate +- productionDate +- resolution.Redshift +- targetSampleActualSize +- timePeriodCoveredEnd +- timePeriodCoveredStart + +### New 3D Objects Metadata Block + +A new metadata block has been added for describing 3D object data. You can download it from the [guides](https://guides.dataverse.org/en/6.6/user/appendix.html). See also #11120 and #11167. + +All new Dataverse installations will receive this metadata block by default. We recommend adding it by following the upgrade instructions below. + +### New Archival Metadata Block (Experimental) + +An experimental "Archival" metadata block has been added, [downloadable](https://guides.dataverse.org/en/6.6/user/appendix.html) from the User Guide. The purpose of the metadata block is to enable repositories to register metadata relating to the potential archiving of the dataset at a depositor archive, whether that being your own institutional archive or an external archive, i.e. a historical archive. Feedback is welcome! See also #10626. + +### Prevent Publishing of Datasets Without Files + +Datasets without files can be optionally prevented from being published through a new "requireFilesToPublishDataset" boolean defined at the collection level. This boolean can be set only via API and only by a superuser. See [Change Collection Attributes](https://guides.dataverse.org/en/6.6/api/native-api.html#change-collection-attributes). If the boolean is not set, the parent collection is consulted. If you do not set the boolean, the existing behavior of datasets being able to be published without files will continue. Superusers can still publish datasets whether or not the boolean is set. See #10981 and #10994. + +### Metadata Source Facet Can Now Differentiate Between Harvested Sources + +The behavior of the feature flag `index-harvested-metadata-source` and the "Metadata Source" facet, which were added and updated, respectively, in [Dataverse 6.3](https://github.com/IQSS/dataverse/releases/tag/v6.3) (through pull requests #10464 and #10651), have been updated. A new field called "Source Name" has been added to harvesting clients. + +Before Dataverse 6.3, all harvested content (datasets and files) appeared together under "Harvested" under the "Metadata Source" facet. This is still the behavior of Dataverse out of the box. Since Dataverse 6.3, enabling the `index-harvested-metadata-source` feature flag (and reindexing) resulted in harvested content appearing under the nickname for whatever harvesting client was used to bring in the content. This meant that instead of having all harvested content lumped together under "Harvested", content would appear under "client1", "client2", etc. + +With this release, enabling the `index-harvested-metadata-source` feature flag, populating a new field for harvesting clients called "Source Name" ("sourceName" in the [API](https://dataverse-guide--11217.org.readthedocs.build/en/11217/api/native-api.html#create-a-harvesting-client)), and reindexing (see upgrade instructions below) results in the source name appearing under the "Metadata Source" facet rather than the harvesting client nickname. This gives you more control over the name that appears under the "Metadata Source" facet and allows you to reuse the same source name to group harvested content from various harvesting clients under the same name if you wish. + +Previously, `index-harvested-metadata-source` was not documented in the guides, but now you can find information about it under [Feature Flags](https://guides.dataverse.org/en/6.6/installation/config.html#feature-flags). See also #10217 and #11217. + +### Globus Framework Improvements + +The improvements and optimizations in this release build on top of the earlier work (such as #10781). They are based on the experience gained at IQSS as part of the production rollout of the Large Data Storage services that utilizes Globus. + +The changes in this release focus on improving Globus *downloads*, i.e., transfers from Dataverse-linked Globus volumes to users' Globus collections. Most importantly, the mechanism of "Asynchronous Task Monitoring", first introduced in #10781 for *uploads*, has been extended to handle downloads as well. This generally makes downloads more reliable, specifically in how Dataverse manages temporary access rules granted to users, minimizing the risk of consequent downloads failing because of stale access rules left in place. + +Multiple other improvements have been made making the underlying Globus framework more reliable and robust. + +See `globus-use-experimental-async-framework` under [Feature Flags](https://guides.dataverse.org/en/6.6/installation/config.html#feature-flags) and [dataverse.files.globus-monitoring-server](https://guides.dataverse.org/en/6.6/installation/config.html#dataverse-files-globus-monitoring-server) in the Installation Guide, #11057, and #11125. + +### OIDC Bearer Tokens + +The release extends the OIDC API auth mechanism, available through feature flag `api-bearer-auth`, to properly handle cases where ``BearerTokenAuthMechanism`` successfully validates the token but cannot identify any Dataverse user because there is no account associated with the token. + +To register a new user who has authenticated via an OIDC provider, a new endpoint has been implemented (`/users/register`). A feature flag named `api-bearer-auth-provide-missing-claims` has been implemented to allow sending missing user claims in the request JSON. This is useful when the identity provider does not supply the necessary claims. However, this flag will only be considered if the `api-bearer-auth` feature flag is enabled. If the latter is not enabled, the `api-bearer-auth-provide-missing-claims` flag will be ignored. + +A feature flag named `api-bearer-auth-handle-tos-acceptance-in-idp` has been implemented. When enabled, it specifies that Terms of Service acceptance is managed by the identity provider, eliminating the need to explicitly include the acceptance in the user registration request JSON. + +See [the guides](https://guides.dataverse.org/en/6.6/api/auth.html#bearer-tokens), #10959, and #10972. + +### Signposting Output Now Contains Links to All Dataset Metadata Export Formats + +When Signposting was added in Dataverse 5.14 (#8981), it provided links only for the `schema.org` metadata export format. + +The output of HEAD, GET, and the Signposting "linkset" API have all been updated to include links to all available dataset metadata export formats, including any external exporters, such as Croissant, that have been enabled. + +This provides a lightweight machine-readable way to first retrieve a list of links, such as via a HTTP HEAD request, to each available metadata export format and then follow up with a request for the export format of interest. + +In addition, the content type for the `schema.org` dataset metadata export format has been corrected. It was `application/json` and now it is `application/ld+json`. + +See also [the guides](https://guides.dataverse.org/en/6.6/api/native-api.html#retrieve-signposting-information), #10542 and #11045. + +### Dataset Types Can Be Linked to Metadata Blocks + +Metadata blocks, such as (e.g. "CodeMeta") can now be linked to dataset types (e.g. "software") using new superuser APIs. + +This will have the following effects for the APIs used by the [new Dataverse UI](https://github.com/IQSS/dataverse-frontend): + +- The list of fields shown when creating a dataset will include fields marked as "displayoncreate" (in the tsv/database) for metadata blocks (e.g. "CodeMeta") that are linked to the dataset type (e.g. "software") that is passed to the API. +- The metadata blocks shown when editing a dataset will include metadata blocks (e.g. "CodeMeta") that are linked to the dataset type (e.g. "software") that is passed to the API. + +Mostly in order to write automated tests for the above, a [displayOnCreate](https://guides.dataverse.org/en/6.6/api/native-api.html#set-displayoncreate-for-a-dataset-field) API endpoint has been added. + +For more information, see the guides ([overview](https://guides.dataverse.org/en/6.6/user/dataset-management.html#dataset-types), [new APIs](https://guides.dataverse.org/en/6.6/api/native-api.html#link-dataset-type-with-metadata-blocks)), #10519 and #11001. + +### Other Features + +- In addition to the API [Move a Dataverse Collection](https://guides.dataverse.org/en/6.6/admin/dataverses-datasets.html#move-a-dataverse-collection), it is now possible for a Dataverse administrator to move a collection using the Dataverse dashboard. See #10304 and #11150. +- The Preview URL popup and [related documentation](https://guides.dataverse.org/en/6.6/user/dataset-management.html#preview-url-to-review-unpublished-dataset) have been updated to give more information about anonymous access, including the names of the dataset fields that will be withheld from the Anonymous Preview URL user and to suggest how to review the URL before releasing it. See also #11159 and #11164. +- [ROR](https://ror.org) (Research Organization Registry) has been added as an Author Identifier Type for when the author is an organization rather than a person. Like ORCID, ROR will appear in the "Datacite" metadata export format. See #11075 and #11118. +- The publisher value of harvested datasets is now attributed to the dataset's distributor instead of its producer. This improves the citation associated with these datasets, but the change affects only newly harvested datasets. See "Upgrade Instructions" below on how to re-harvest. For more information, see [the guides](http://guides.dataverse.org/en/6.6/admin/harvestclients.html#harvesting-client-changelog), #8739, and #9013. +- A new harvest status differentiates between a complete harvest with errors ("completed with failures") and without errors ("completed"). Also, harvest status labels are now internationalized. See #9294 and #11017. +- The OAI-ORE exporter can now export metadata containing nested compound fields or compound fields within compound fields. See #10809 and #11190. +- It is now possible to edit a custom role with the same alias. See #8808 and #10612. +- The [Metadata Customization](https://guides.dataverse.org/en/6.6/admin/metadatacustomization.html#controlledvocabulary-enumerated-properties) documentation has been updated to explain how to implement a boolean fieldtype (look for "boolean"). See #7961 and #11064. +- The version of Stata files is now detected during S3 direct upload (as it was for normal uploads), allowing ingest of Stata 14 and 15 files that have been uploaded directly. See [the guides](https://guides.dataverse.org/en/6.6/developers/big-data-support.html#features-that-are-disabled-if-s3-direct-upload-is-enabled) #10108, and #11054. +- It is now possible to populate the "Keyword" metadata field from an [OntoPortal](https://ontoportal.org) service. The code has been shared to the GDCC [dataverse-external-vocab-support](https://github.com/gdcc/dataverse-external-vocab-support#scripts-in-production) GitHub repository. See #11258. +- Support for legacy configuration of a PermaLink PID provider, such as using the :Protocol,:Authority, and :Shoulder settings, has been fixed. See #10516 and #10521. +- On the home page for each guide (User Guide, etc.) there was an overwhelming amount of information in the form of a deeply nested table of contents. The depth of the table of contents has been reduced to two levels, making the home page for each guide more readable. Compare the User Guide for [6.5](https://guides.dataverse.org/en/6.5/user/index.html) vs. [6.6](https://guides.dataverse.org/en/6.6/user/index.html) and see #11166. +- For compliance with GDPR and other privacy regulations, advice on adding a cookie consent popup has been added to the guides. See the new [cookie consent](https://guides.dataverse.org/en/6.6/installation/config.html#adding-cookie-consent-for-gdpr-etc) section and #10320. +- A new file has been added to import the French Open License to Dataverse: licenseEtalab-2.0.json. You can download it from [the guides](http://guides.dataverse.org/en/6.6/installation/config.html#adding-licenses). This license, which is compatible with the Creative Commons license, is recommended by the French government for open documents. See #9301, #9302, and #11302. +- The API that lists versions of a dataset now features an optional `excludeMetadataBlocks` parameter, which defaults to "false" for backward compatibility. For a dataset with a large number of versions and/or metadataBlocks, having the metadata blocks included can dramatically increase the volume of the output. See also [the guides](https://guides.dataverse.org/en/6.6/api/native-api.html#list-versions-of-a-dataset), #10171, and #10778. +- Deeply nested metadata fields are not supported but the code used to generate the Solr schema has been adjusted to support them. See #11136. +- The [tutorial](https://guides.dataverse.org/en/6.6/container/running/demo.html) on running Dataverse in Docker has been updated to explain how to configure the root collection using a JSON file (#10541 and #11201) and now uses the Permalink PID provider instead of the FAKE DOI Provider (#11107 and #11108). +- Payara application server has been upgraded to version 6.2025.2. See #11126 and #11128. +- Solr has been upgraded to version 9.8.0. See #10713. +- For testing purposes, the FAKE PID provider can now be used with [file PIDs enabled](https://guides.dataverse.org/en/6.6/installation/config.html#filepidsenabled). (The FAKE provider is not recommended for any production use.) See #10979. + +## Bugs Fixed + +- A bug which causes users of the Anonymous Review URL to have some metadata of published datasets withheld has been fixed. See #11202 and #11164. +- A bug that caused ORCIDs starting with "https://orcid.org/" entered as author identifier to be ignored when creating the DataCite metadata has been fixed. This primarily affected users of the ORCID external vocabulary script; for the manual entry form, we used to recommend not using the URL form. The display of authorIdentifier, when not using any external vocabulary scripts, has been improved so that either the plain identifier (e.g. "0000-0002-1825-0097") or its URL form (e.g. "https://orcid.org/0000-0002-1825-0097") will result in valid links in the display (for identifier types that have a URL form). The URL form is now [recommended](http://guides.dataverse.org/en/6.6/user/dataset-management.html#adding-a-new-dataset) when doing manual entry. See #11242 and #11242. +- Multiple small issues with the formatting of PIDs in the DDI exporters, and EndNote and BibTeX citation formats have been addressed. These should improve the ability to import Dataverse citations into reference managers and fix potential issues harvesting datasets using PermaLinks. See #10768, #10769, #11165, and #10790. +- On the Advanced Search page, the metadata fields are now displayed in the correct order as defined in the TSV file via the displayOrder value, making the order the same as when you view or edit metadata. Note that fields that are not defined in the TSV file, like the "Persistent ID" and "Publication Date", will be displayed at the end. See #11272 and #11279. +- Bugs that caused 1) guestbook questions to appear along with terms of use/terms of access in the request access dialog when no guestbook was configured, and 2) terms of access to not be shown when using the per-file request access/download menu items have been fixed. Text related to configuring the choice to have guestbooks appear when file access is requested or when files are downloaded has been updated to make it clearer that this affects only datasets where guestbooks have been configured. See #11203. +- The file page version table now shows whether a file has been replaced. See #11142 and #11145. +- We fixed an issue where draft versions of datasets were sorted using the release timestamp of their most recent major version. This caused newer drafts to appear incorrectly alongside their corresponding major version, instead of at the top, when sorted by "newest first". Sorting now uses the last update timestamp when sorting draft datasets. The sorting behavior of published major and minor dataset versions is unchanged. There is no need to reindex datasets because Solr is being upgraded (see "Upgrade Instructions"), which will result in an empty database that will be reindexed. See #11178. +- Some external controlled vocabulary scripts/configurations, when used on a metadata field that is single-valued, could result in indexing failure for the dataset, e.g. when the script tried to index both the identifier and name of the identified entity for indexing. Dataverse has been updated to correctly indicate the need for a multi-valued Solr field in these cases in the call to `/api/admin/index/solr/schema`. Configuring the Solr schema and running the update-fields.sh script as usually recommended when using custom metadata blocks (see "Upgrade Instructions") will resolve the issue. See [the guides](https://guides.dataverse.org/en/6.6/admin/metadatacustomization.html#using-external-vocabulary-services), #11095, and #11096. +- The OpenAIRE metadata export format can now correctly process one or multiple productionPlaces as geolocation. See #9546 and #11194 +- We fixed a bug that caused adding free-form provenance to a file to fail. See #11145. +- A bug has been fixed which could cause publication of datasets to fail in cases where they were not assigned a DOI at creation. See #11234 and #11236. +- When users request access to files, the people who have permission to grant access received an email with a link that didn't work due to a trailing period (full stop) right next to the link, e.g. `https://demo.dataverse.org/permissions-manage-files.xhtml?id=9.` A space has been added to fix this. See #10384 and #11115. +- Harvesting clients now use the correct granularity while re-running a partial harvest, using the `from` parameter. The correct granularity comes from the `Identify` verb request. See #11020 and #11038. +- Access requests were missing on the File Permission page after upgrading from Dataverse 6.0. This has been corrected with a database update script. See #10714 and #11061. +- When a dataset has a long running lock, including when it is "in review", Dataverse will now slow the page refresh rate over time. See #11264 and #11269. +- The `/api/info/metrics/files/monthly` API call had a bug that resulted in files being counted each time they were published in a new version if those publication events occurred in different months. This resulted in an over-count. The `/api/info/metrics/files` and `/api/info/metrics/files/toMonth` API calls had a bug that resulted in files that were published but no longer in the latest published version as of the specified date (now, or the date entered in the `/toMonth` variant). This resulted in an under-count. See #11189. +- DatasetFieldTypes in MetadataBlock response that are also a child of another DatasetFieldType were being returned twice. The child DatasetFieldType was included in the "fields" object as well as in the "childFields" of its parent DatasetFieldType. This fix suppresses the standalone object so only one instance of the DatasetFieldType is returned (in the "childFields" of its parent). This fix changes the JSON output of the API `/api/dataverses/{dataverseAlias}/metadatablocks` (see "Backward Incompatible Changes", below). See #10472 and #11066. +- A bug that caused replacing files via API when file PIDs were enabled to fail has been fixed. See #10975 and #10979. +- The [:CustomDatasetSummaryFields](https://guides.dataverse.org/en/6.6/installation/config.html#customdatasetsummaryfields) setting now allows spaces along with a comma separating field names. In addition, a bug that caused license information to be hidden if there are no values for any of the custom fields specified has been fixed. See #11228 and #11229. +- Dataverse 6.5 introduced a bug which causes search to fail for non-superusers in multiple groups when the `AVOID_EXPENSIVE_SOLR_JOIN` feature flag is set to true. This release fixes the bug. See #11133 and #11134. +- We fixed a bug with My Data where listing collections for a user with only rights on harvested collections would result in a server error response. See #11083. +- Minor styling fixes for the Related Publication field and fields using ORCID or ROR have been made. See #11053, #10964, and #11106. +- In the Search API, files were displaying DRAFT version instead of latest released version under `dataset_citation`. See #10735 and #11051. +- Unnecessary Solr documents were being created when a file was added or deleted from a draft dataset. These documents could accumulate and potentially impact performance. There is no action to take because this release includes a new Solr version, which will start with an empty database. See #11113 and #11114. +- When using the API to update a collection, omitting optional fields such as `inputLevels`, `facetIds`, or `metadataBlockNames` caused data to be deleted. The fix no longer deletes data for these fields. Two new flags have been added to the `metadataBlocks` JSON object to signal the deletion of the data: `inheritMetadataBlocksFromParent: true` and `inheritFacetsFromParent: true`. See [the guides](https://guides.dataverse.org/en/6.6/api/native-api.html#update-a-dataverse-collection), #11130, and #11144. + +## API Updates + +### Search API Returns Additional Fields for Files + +Added new fields to search results type=files + +For Files: + +- restricted: boolean +- canDownloadFile: boolean (from file user permission) +- categories: array of string "categories" would be similar to what it is in metadata api. + +For tabular files: + +- tabularTags: array of string for example, `{"tabularTags" : ["Event", "Genomics", "Geospatial"]}` +- variables: number/int shows how many variables we have for the tabular file +- observations: number/int shows how many observations for the tabular file + +See #11027 and #11097. + +### Backend Support for Collection Featured Items + +CRUD endpoints for Collection Featured Items have been implemented. In particular, the following endpoints have been implemented: + +- Create a feature item (POST `/api/dataverses//featuredItems`) +- Update a feature item (PUT `/api/dataverseFeaturedItems/`) +- Delete a feature item (DELETE `/api/dataverseFeaturedItems/`) +- List all featured items in a collection (GET `/api/dataverses//featuredItems`) +- Delete all featured items in a collection (DELETE `/api/dataverses//featuredItems`) +- Update all featured items in a collection (PUT `/api/dataverses//featuredItems`) + +See also the "Settings Added" section, #10943 and #11124. + +### Other API Updates + +- Multiple files can be deleted from a dataset at once. See the [the guides](https://guides.dataverse.org/en/6.6/api/native-api.html#delete-files-from-a-dataset) and #11230. +- An API has been added to get the "classic" download count from a dataset with an optional `includeMDC` parameter (for Make Data Count). See [the guides](http://guides.dataverse.org/en/6.6/api/native-api.html#get-the-download-count-of-a-dataset), #11244 and #11282. +- An API has been added that lists the collections that the user has access to via the permission passed. See [the guides](http://guides.dataverse.org/en/6.6/api/native-api.html#list-dataverse-collections-a-user-can-act-on-based-on-their-permissions), #6467, and #10906. +- An API has been added to get dataset versions including a summary of differences between consecutive versions where available. See [the docs](https://guides.dataverse.org/en/6.6/api/native-api.html#get-versions-of-a-dataset-with-summary-of-changes ), #10888, and #10945. +- An API has been added to list of versions of a data file showing any changes that affected the file with each version. See [the guides](https://guides.dataverse.org/en/6.6/api/native-api.html#get-json-representation-of-a-file-s-versions), #11198 and #11237. +- The Search API has a new [parameter](https://guides.dataverse.org/en/6.6/api/search.html#parameters) called `show_type_counts`. If you set it to true, it will return `total_count_per_object_type` for the types dataverse, dataset, and files (#11065 and #11082) even if the search result for any given type is 0 (#11127 and #11138). +- CRUD operations for external tools are now available for superusers from non-localhost. See [the guides](https://guides.dataverse.org/en/6.6/admin/external-tools.html#managing-external-tools), #10930 and #11079. +- A new API endpoint has been added that allows a global role to be updated. See [the guides](https://guides.dataverse.org/en/6.6/api/native-api.html#update-global-role) and #10612. +- An API has been added to send feedback to the collection, dataset, or data file's contacts. If necessary, you can [rate limit](https://guides.dataverse.org/en/6.6/installation/config.html#rate-limiting) the `CheckRateLimitForDatasetFeedbackCommand` and configure the new [:ContactFeedbackMessageSizeLimit](https://guides.dataverse.org/en/6.6/installation/config.html#contactfeedbackmessagesizelimit) database setting. See [the guides](http://guides.dataverse.org/en/6.6/api/native-api.html#send-feedback-to-contact-s), #11129, and #11162. +- /api/metadatablocks is no longer returning duplicated metadata properties and does not omit metadata properties when called. See "Backward Incompatible Changes" below and #10764. +- A new query param, `returnChildCount`, has been added to the getDataverse endpoint (`/api/dataverses/{id}`) for optionally retrieving the child count, which represents the number of collections, datasets, or files within the collection (direct children only). See also #11255 and #11259. + +## End-Of-Life (EOL) Announcements + +### PostgreSQL 13 reaches EOL on 13 November 2025 + +Per PostgreSQL 13 reaches EOL on 13 November 2025. Our first step toward moving off version 13 was to [switch](https://github.com/gdcc/dataverse-ansible/commit/8ebbd84ad2cf3903b8f995f0d34578250f4223ff) our testing to version 16, as we've [noted](https://guides.dataverse.org/en/6.6/installation/prerequisites.html#postgresql) in the guides. You are encouraged to start planning your upgrade and may want to review the [Dataverse 5.4 release notes](https://github.com/IQSS/dataverse/releases/tag/v5.4) as the upgrade process (e.g. `pg_dumpall`, etc.) will likely be similar. If you notice any bumps along the way, please let us know! + +Dataverse developers [using Docker](https://guides.dataverse.org/en/6.6/container/dev-usage.html) have been using PostgreSQL 17 since Dataverse 6.5 (#10912). (Developers not using Docker who are still on PostgreSQL 13 are encouraged to upgrade.) Older or newer versions should work, within reason. + +See also #11212 and #11215. + +## Security + +### SameSite Cookie Attribute + +The SameSite cookie attribute is defined in an upcoming revision to [RFC 6265](https://datatracker.ietf.org/doc/html/rfc6265) (HTTP State Management Mechanism) called [6265bis](https://datatracker.ietf.org/doc/html/draft-ietf-httpbis-rfc6265bis-19>) ("bis" meaning "repeated"). The possible values are "None", "Lax", and "Strict". + +"If no SameSite attribute is set, the cookie is treated as Lax by default" by browsers according to [MDN](https://developer.mozilla.org/en-US/docs/Web/HTTP/Cookies#controlling_third-party_cookies_with_samesite). This was the previous behavior of Dataverse, to not set the SameSite attribute. + +New Dataverse installations now explicitly set to the SameSite cookie attribute to "Lax" out of the box through the installer (in the case of a "classic" installation) or through an updated base image (in the case of a Docker installation). Classic installations should follow the upgrade instructions below to bring their installation up to date with the behavior for new installations. Docker installations will automatically get the updated base image. + +While you are welcome to experiment with "Strict", which is intended to help prevent Cross-Site Request Forgery (CSRF) attacks, as described in the RFC proposal and an OWASP [cheatsheet](https://cheatsheetseries.owasp.org/cheatsheets/Cross-Site_Request_Forgery_Prevention_Cheat_Sheet.html#samesite-cookie-attribute), our testing so far indicates that some functionality, such as OIDC login, seems to be incompatible with "Strict". + +You should avoid the use of "None" as it is less secure than "Lax". See also [the guides](https://guides.dataverse.org/en/6.6/installation/config.html#samesite-cookie-attribute), https://github.com/IQSS/dataverse-security/issues/27, #11210, and the upgrade instructions below. + +## Settings Added + +- dataverse.feature.enable-version-note +- dataverse.csl.common-styles +- dataverse.files.featured-items.image-maxsize - It sets the maximum allowed size of the image that can be added to a featured item. +- dataverse.files.featured-items.image-uploads - It specifies the name of the subdirectory for saving featured item images within the docroot directory. +- dataverse.feature.api-bearer-auth-provide-missing-claims +- dataverse.feature.api-bearer-auth-handle-tos-acceptance-in-idp +- :ContactFeedbackMessageSizeLimit + +## Backward Incompatible Changes + +Generally speaking, see the [API Changelog](https://guides.dataverse.org/en/latest/api/changelog.html) for a list of backward-incompatible API changes. + +- /api/metadatablocks is no longer returning duplicated metadata properties and does not omit metadata properties when called. See #10764. +- The JSON response of API call `/api/dataverses/{dataverseAlias}/metadatablocks` will no longer include the DatasetFieldTypes in "fields" if they are children of another DatasetFieldType. The child DatasetFieldType will only be included in the "childFields" of its parent DatasetFieldType. See #10472 and #11066. +- `versionNote` has been renamed to `deaccessionNote`. `archiveNote` has been renamed to `deaccessionLink`. See #11068. +- The [Show Role](https://guides.dataverse.org/en/6.6/api/native-api.html#show-role) API endpoint was returning 401 Unauthorized when a permission check failed. This has been corrected to return 403 Forbidden instead. That is, the API token is known to be good (401 otherwise) but the user lacks permission (403 is now sent). See also the [API Changelog](https://guides.dataverse.org/en/6.6/11116/api/changelog.html), #10340, and #11116. +- Changes to PID formatting occur in the DDI/DDI Html export formats and the EndNote and BibTex citation formats. These changes correct errors and improve conformance with best practices but could break parsing of these formats. See #10768, #10769, #11165, and #10790. + +## Complete List of Changes + +For the complete list of code changes in this release, see the [6.6 milestone](https://github.com/IQSS/dataverse/issues?q=milestone%3A6.6+is%3Aclosed) in GitHub. + +## Getting Help + +For help with upgrading, installing, or general questions please post to the [Dataverse Community Google Group](https://groups.google.com/g/dataverse-community) or email support@dataverse.org. + +## Installation + +If this is a new installation, please follow our [Installation Guide](https://guides.dataverse.org/en/latest/installation/). Please don't be shy about [asking for help](https://guides.dataverse.org/en/latest/installation/intro.html#getting-help) if you need it! + +Once you are in production, we would be delighted to update our [map of Dataverse installations](https://dataverse.org/installations) around the world to include yours! Please [create an issue](https://github.com/IQSS/dataverse-installations/issues) or email us at support@dataverse.org to join the club! + +You are also very welcome to join the [Global Dataverse Community Consortium](https://www.gdcc.io/) (GDCC). + +## Upgrade Instructions + +Upgrading requires a maintenance window and downtime. Please plan accordingly, create backups of your database, etc. + +These instructions assume that you've already upgraded through all the 5.x releases and are now running Dataverse 6.5. + +0\. These instructions assume that you are upgrading from the immediate previous version. If you are running an earlier version, the only supported way to upgrade is to progress through the upgrades to all the releases in between before attempting the upgrade to this version. + +If you are running Payara as a non-root user (and you should be!), **remember not to execute the commands below as root**. By default, Payara runs as the `dataverse` user. In the commands below, we use sudo to run the commands as a non-root user. + +Also, we assume that Payara 6 is installed in `/usr/local/payara6`. If not, adjust as needed. + +```shell +export PAYARA=/usr/local/payara6 +``` + +(or `setenv PAYARA /usr/local/payara6` if you are using a `csh`-like shell) + +1\. List deployed applications + +```shell +$PAYARA/bin/asadmin list-applications +``` + +2\. Undeploy the previous version (should match "list-applications" above) + +```shell +$PAYARA/bin/asadmin undeploy dataverse-6.5 +``` + +3\. Stop Payara + +```shell +sudo service payara stop +``` + +4\. Upgrade to Payara 6.2025.2 + +The steps below reuse your existing domain directory with the new distribution of Payara. You may also want to review the Payara upgrade instructions as it could be helpful during any troubleshooting: +[Payara Release Notes](https://docs.payara.fish/community/docs/6.2025.2/Release%20Notes/Release%20Notes%206.2025.2.html). +We also recommend you ensure you followed all update instructions from the past releases regarding Payara. +(The most recent Payara update was for [v6.3](https://github.com/IQSS/dataverse/releases/tag/v6.3).) + +Move the current Payara directory out of the way: + +```shell +mv $PAYARA $PAYARA.6.2024.6 +``` + +Download the new Payara version 6.2025.2 (from https://www.payara.fish/downloads/payara-platform-community-edition/ or https://nexus.payara.fish/repository/payara-community/fish/payara/distributions/payara/6.2025.2/payara-6.2025.2.zip), and unzip it in its place: + +```shell +cd /usr/local +unzip payara-6.2025.2.zip +``` + +Replace the brand new `payara/glassfish/domains/domain1` with your old, preserved domain1: + +```shell +mv payara6/glassfish/domains/domain1 payara6/glassfish/domains/domain1_DIST +mv payara6.6.2024.6/glassfish/domains/domain1 payara6/glassfish/domains/ +``` + +5\. Download and deploy this version + +```shell +wget https://github.com/IQSS/dataverse/releases/download/v6.6/dataverse-6.6.war +$PAYARA/bin/asadmin deploy dataverse-6.6.war +``` + +Note: if you have any trouble deploying, stop Payara, remove the following directories, start Payara, and try to deploy again. + +```shell +sudo service payara stop +sudo rm -rf $PAYARA/glassfish/domains/domain1/generated +sudo rm -rf $PAYARA/glassfish/domains/domain1/osgi-cache +sudo rm -rf $PAYARA/glassfish/domains/domain1/lib/databases +sudo service payara start +``` + +6\. For installations with internationalization or text customizations: + +Please remember to update translations via [Dataverse language packs](https://github.com/GlobalDataverseCommunityConsortium/dataverse-language-packs). + +If you have text customizations you can get the latest English files from . + +7\. Decide to enable (or not) the `index-harvested-metadata-source` feature flag + +Decide whether or not to enable the `dataverse.feature.index-harvested-metadata-source` feature flag described above, in [the guides](https://guides.dataverse.org/en/6.6/installation/config.html#feature-flags), #10217 and #11217. The reason to decide now is that reindexing is required and the next steps involve restarting Payara and upgrading Solr, which will result in a fresh index. + +8\. Configure SameSite + +To bring your Dataverse installation in line with new installations, as described above and in [the guides](https://guides.dataverse.org/en/6.6/installation/config.html#samesite-cookie-attribute), we recommend running the following commands: + +``` +./asadmin set server-config.network-config.protocols.protocol.http-listener-1.http.cookie-same-site-value=Lax + +./asadmin set server-config.network-config.protocols.protocol.http-listener-1.http.cookie-same-site-enabled=true +``` + +Please note that "None" is less secure than "Lax" and should be avoided. You can test the setting by inspecting headers with curl, looking at the JSESSIONID cookie for "SameSite=Lax" (yes, it's expected to be repeated, probably due to a bug in Payara) like this: + +``` +% curl -s -I http://localhost:8080 | grep JSESSIONID +Set-Cookie: JSESSIONID=6574324d75aebeb86dc96ecb3bb0; Path=/;SameSite=Lax;SameSite=Lax +``` + +Before making the changes above, SameSite attribute should be absent, like this: + +``` +% curl -s -I http://localhost:8080 | grep JSESSIONID +Set-Cookie: JSESSIONID=6574324d75aebeb86dc96ecb3bb0; Path=/ +``` + +8\. Restart Payara + +```shell +sudo service payara stop +sudo service payara start +``` + +9\. Update metadata blocks + +These changes reflect incremental improvements made to the handling of core metadata fields. + +Expect the loading of the citation block to take several seconds because of its size (especially due to the number of languages). + +```shell +wget https://raw.githubusercontent.com/IQSS/dataverse/v6.6/scripts/api/data/metadatablocks/citation.tsv + +curl http://localhost:8080/api/admin/datasetfield/load -H "Content-type: text/tab-separated-values" -X POST --upload-file citation.tsv +``` + +The 3D Objects metadata block is included in all new installations of Dataverse so we recommend adding it. + +```shell +wget https://raw.githubusercontent.com/IQSS/dataverse/v6.6/scripts/api/data/metadatablocks/3d_objects.tsv + +curl http://localhost:8080/api/admin/datasetfield/load -H "Content-type: text/tab-separated-values" -X POST --upload-file 3d_objects.tsv +``` + +10\. Upgrade Solr + +Solr 9.8.0 is now the version recommended in our Installation Guide and used with automated testing. Additionally, due to the new range search support feature and the addition of fields (e.g. versionNote, fileRestricted, canDownloadFile, variableCount, and observations), the default `schema.xml` files has changed so you must upgrade. + +Install Solr 9.8.0 following the [instructions](https://guides.dataverse.org/en/6.6/installation/prerequisites.html#solr) from the Installation Guide. + +The instructions in the guide suggest to use the config files from the installer zip bundle. When upgrading an existing instance, it may be easier to download them from the source tree: + +```shell +wget https://raw.githubusercontent.com/IQSS/dataverse/v6.6/conf/solr/solrconfig.xml +wget https://raw.githubusercontent.com/IQSS/dataverse/v6.6/conf/solr/schema.xml +cp solrconfig.xml schema.xml /usr/local/solr/solr-9.8.0/server/solr/collection1/conf +``` + +10a\. For installations with additional metadata blocks or external controlled vocabulary scripts, update fields + +- Stop Solr instance (usually `service solr stop`, depending on Solr installation/OS, see the [Installation Guide](https://guides.dataverse.org/en/6.6/installation/prerequisites.html#solr-init-script)). + +- Run the `update-fields.sh` script that we supply, as in the example below (modify the command lines as needed to reflect the correct path of your Solr installation): + +```shell +wget https://raw.githubusercontent.com/IQSS/dataverse/v6.6/conf/solr/update-fields.sh +chmod +x update-fields.sh +curl "http://localhost:8080/api/admin/index/solr/schema" | ./update-fields.sh /usr/local/solr/solr-9.8.0/server/solr/collection1/conf/schema.xml +``` + +- Start Solr instance (usually `service solr start` depending on Solr/OS). + +11\. Reindex Solr + +```shell +curl http://localhost:8080/api/admin/index +``` + +12\. Run reExportAll to update dataset metadata exports + +For existing published datasets, additional license metadata will not be available from DataCite or in metadata exports until + +- the dataset is republished or +- the /api/admin/metadata/{id}/reExportDataset is run for the dataset or +- the /api/datasets/{id}/modifyRegistrationMetadata API is run for the dataset or +- the global version of these API calls (/api/admin/metadata/reExportAll, /api/datasets/modifyRegistrationPIDMetadataAll) are used. + +For this reason, we recommend reexporting all dataset metadata. For more advanced usage, please see [the guides](http://guides.dataverse.org/en/6.6/admin/metadataexport.html#batch-exports-through-the-api). + +```shell +curl http://localhost:8080/api/admin/metadata/reExportAll +``` + +13\. (Optional) Re-harvest datasets + +The publisher value of harvested datasets is now attributed to the dataset's distributor instead of its producer. For more information, see [the guides](http://guides.dataverse.org/en/6.6/admin/harvestclients.html#harvesting-client-changelog), #8739, and #9013. + +This improves the citation associated with these datasets, but the change only affects newly harvested datasets. + +If you would like to pick up this change for existing harvested datasets, you should re-harvest them. This can be accomplished by deleting and re-adding each harvesting client, followed by a harvesting run. You may want to use [harvesting client APIs](https://guides.dataverse.org/en/6.6/api/native-api.html#managing-harvesting-clients) to save (serialize), add, and remove clients. diff --git a/doc/sphinx-guides/source/_static/admin/counter-processor-config.yaml b/doc/sphinx-guides/source/_static/admin/counter-processor-config.yaml index 26144544d9e..f3501ead7b3 100644 --- a/doc/sphinx-guides/source/_static/admin/counter-processor-config.yaml +++ b/doc/sphinx-guides/source/_static/admin/counter-processor-config.yaml @@ -33,8 +33,10 @@ path_types: # Robots and machines urls are urls where the script can download a list of regular expressions to determine # if something is a robot or machine user-agent. The text file has one regular expression per line -robots_url: https://raw.githubusercontent.com/CDLUC3/Make-Data-Count/master/user-agents/lists/robot.txt -machines_url: https://raw.githubusercontent.com/CDLUC3/Make-Data-Count/master/user-agents/lists/machine.txt +#robots_url: https://raw.githubusercontent.com/CDLUC3/Make-Data-Count/master/user-agents/lists/robot.txt +#machines_url: https://raw.githubusercontent.com/CDLUC3/Make-Data-Count/master/user-agents/lists/machine.txt +robots_url: https://raw.githubusercontent.com/IQSS/counter-processor/refs/heads/goto-gdcc/user-agents/lists/robots.txt +machines_url: https://raw.githubusercontent.com/IQSS/counter-processor/refs/heads/goto-gdcc/user-agents/lists/machine.txt # the year and month for the report you are creating. year_month: 2019-01 diff --git a/doc/sphinx-guides/source/_static/admin/dataverse-external-tools.tsv b/doc/sphinx-guides/source/_static/admin/dataverse-external-tools.tsv index 3df5ed5d24f..9a3d2a89acb 100644 --- a/doc/sphinx-guides/source/_static/admin/dataverse-external-tools.tsv +++ b/doc/sphinx-guides/source/_static/admin/dataverse-external-tools.tsv @@ -1,9 +1,8 @@ Tool Type Scope Description -Data Explorer explore file "A GUI which lists the variables in a tabular data file allowing searching, charting and cross tabulation analysis. See the README.md file at https://github.com/scholarsportal/dataverse-data-explorer-v2 for the instructions on adding Data Explorer to your Dataverse." +Data Explorer explore file "A GUI which lists the variables in a tabular data file allowing searching, charting and cross tabulation analysis. The latest version incorporates the Data Curation Tool, a GUI for curating data by adding labels, groups, weights and other details to assist with informed reuse. See the README.md file at https://github.com/scholarsportal/Dataverse-Data-Explorer for the instructions on adding Data Explorer to your Dataverse." Whole Tale explore dataset "A platform for the creation of reproducible research packages that allows users to launch containerized interactive analysis environments based on popular tools such as Jupyter and RStudio. Using this integration, Dataverse users can launch Jupyter and RStudio environments to analyze published datasets. For more information, see the `Whole Tale User Guide `_." Binder explore dataset Binder allows you to spin up custom computing environments in the cloud (including Jupyter notebooks) with the files from your dataset. See https://github.com/IQSS/dataverse-binder-redirect for installation instructions. File Previewers explore file "A set of tools that display the content of files - including audio, html, `Hypothes.is `_ annotations, images, PDF, Markdown, text, video, tabular data, spreadsheets, GeoJSON, zip, and NcML files - allowing them to be viewed without downloading the file. The previewers can be run directly from github.io, so the only required step is using the Dataverse API to register the ones you want to use. Documentation, including how to optionally brand the previewers, and an invitation to contribute through github are in the README.md file. Initial development was led by the Qualitative Data Repository and the spreasdheet previewer was added by the Social Sciences and Humanities Open Cloud (SSHOC) project. https://github.com/gdcc/dataverse-previewers" -Data Curation Tool configure file "A GUI for curating data by adding labels, groups, weights and other details to assist with informed reuse. See the README.md file at https://github.com/scholarsportal/Dataverse-Data-Curation-Tool for the installation instructions." Ask the Data query file Ask the Data is an experimental tool that allows you ask natural language questions about the data contained in Dataverse tables (tabular data). See the README.md file at https://github.com/IQSS/askdataverse/tree/main/askthedata for the instructions on adding Ask the Data to your Dataverse installation. TurboCurator by ICPSR configure dataset TurboCurator generates metadata improvements for title, description, and keywords. It relies on open AI's ChatGPT & ICPSR best practices. See the `TurboCurator Dataverse Administrator `_ page for more details on how it works and adding TurboCurator to your Dataverse installation. JupyterHub explore file The `Dataverse-to-JupyterHub Data Transfer Connector `_ is a tool that simplifies the transfer of data between Dataverse repositories and the cloud-based platform JupyterHub. It is designed for researchers, scientists, and data analysts, facilitating collaboration on projects by seamlessly moving datasets and files. The tool is a lightweight client-side web application built using React and relies on the Dataverse External Tool feature, allowing for easy deployment on modern integration systems. Currently optimized for small to medium-sized files, future plans include extending support for larger files and signed Dataverse endpoints. For more details, you can refer to the external tool manifest: https://forgemia.inra.fr/dipso/eosc-pillar/dataverse-jupyterhub-connector/-/blob/master/externalTools.json diff --git a/doc/sphinx-guides/source/_static/api/add-license.json b/doc/sphinx-guides/source/_static/api/add-license.json index a9d5dd34093..9a5c478dc36 100644 --- a/doc/sphinx-guides/source/_static/api/add-license.json +++ b/doc/sphinx-guides/source/_static/api/add-license.json @@ -4,5 +4,9 @@ "shortDescription": "Creative Commons Attribution 4.0 International License.", "iconUrl": "https://i.creativecommons.org/l/by/4.0/88x31.png", "active": true, - "sortOrder": 2 -} + "sortOrder": 2, + "rightsIdentifier": "CC-BY-4.0", + "rightsIdentifierScheme": "SPDX", + "schemeUri": "https://spdx.org/licenses/", + "languageCode": "en" +} \ No newline at end of file diff --git a/doc/sphinx-guides/source/_static/api/dataset-add-single-compound-field-metadata.json b/doc/sphinx-guides/source/_static/api/dataset-add-single-compound-field-metadata.json new file mode 100644 index 00000000000..f49a9e47d5b --- /dev/null +++ b/doc/sphinx-guides/source/_static/api/dataset-add-single-compound-field-metadata.json @@ -0,0 +1,13 @@ +{ + "fields": [ + { + "typeName": "targetSampleSize", + "value": { + "targetSampleActualSize": { + "typeName": "targetSampleSizeFormula", + "value": "n = N*X / (X + N โ€“ 1)" + } + } + } + ] +} \ No newline at end of file diff --git a/doc/sphinx-guides/source/_static/api/dataset-add-single-cvoc-field-metadata.json b/doc/sphinx-guides/source/_static/api/dataset-add-single-cvoc-field-metadata.json new file mode 100644 index 00000000000..620f3df10d1 --- /dev/null +++ b/doc/sphinx-guides/source/_static/api/dataset-add-single-cvoc-field-metadata.json @@ -0,0 +1,4 @@ +{ + "typeName": "journalArticleType", + "value": "abstract" +} \ No newline at end of file diff --git a/doc/sphinx-guides/source/_static/api/dataset-migrate.jsonld b/doc/sphinx-guides/source/_static/api/dataset-migrate.jsonld index 8f43d1dd6e9..bd882846da1 100644 --- a/doc/sphinx-guides/source/_static/api/dataset-migrate.jsonld +++ b/doc/sphinx-guides/source/_static/api/dataset-migrate.jsonld @@ -1,11 +1,31 @@ { "citation:depositor": "Admin, Dataverse", "title": "Test Dataset", +"socialscience:collectionMode": [ + "demonstration" +], "subject": "Computer and Information Science", +"geospatial:geographicCoverage": [ + { + "geospatial:otherGeographicCoverage": "Cambridge" + }, + { + "geospatial:otherGeographicCoverage": "Massachusetts" + } +], "author": { "citation:authorName": "Admin, Dataverse", "citation:authorAffiliation": "GDCC" }, +"kindOfData": "demonstration data", +"citation:keyword": [ + { + "citation:keywordValue": "first keyword" + }, + { + "citation:keywordValue": "second keyword" + } +], "dateOfDeposit": "2020-10-08", "citation:distributor": { "citation:distributorName": "Demo Dataverse Repository", @@ -35,5 +55,9 @@ "title": "http://purl.org/dc/terms/title", "citation": "https://dataverse.org/schema/citation/", "dvcore": "https://dataverse.org/schema/core#", - "schema": "http://schema.org/" -}} + "schema": "http://schema.org/", + "geospatial": "dataverse.siteUrl/schema/geospatial#", + "socialscience": "dataverse.siteUrl/schema/socialscience#", + "kindOfData": "http://rdf-vocabulary.ddialliance.org/discovery#kindOfData" + } +} diff --git a/doc/sphinx-guides/source/_static/api/dataset-schema.json b/doc/sphinx-guides/source/_static/api/dataset-schema.json index 34b8a1eeedb..85ea5a0d773 100644 --- a/doc/sphinx-guides/source/_static/api/dataset-schema.json +++ b/doc/sphinx-guides/source/_static/api/dataset-schema.json @@ -26,6 +26,9 @@ }, "typeName": { "type": "string" + }, + "displayOnCreate": { + "type": "boolean" } } } diff --git a/doc/sphinx-guides/source/_static/api/harvesting-client.json b/doc/sphinx-guides/source/_static/api/harvesting-client.json new file mode 100644 index 00000000000..82a817fc38f --- /dev/null +++ b/doc/sphinx-guides/source/_static/api/harvesting-client.json @@ -0,0 +1,11 @@ +{ + "nickName": "zenodo", + "dataverseAlias": "zenodoHarvested", + "harvestUrl": "https://zenodo.org/oai2d", + "archiveUrl": "https://zenodo.org", + "archiveDescription": "Moissonnรฉ depuis la collection LMOPS de l'entrepรดt Zenodo. En cliquant sur ce jeu de donnรฉes, vous serez redirigรฉ vers Zenodo.", + "metadataFormat": "oai_dc", + "customHeaders": "x-oai-api-key: xxxyyyzzz", + "set": "user-lmops", + "allowHarvestingMissingCVV":true +} diff --git a/doc/sphinx-guides/source/_static/api/transform-oai-ore-jsonld.xq b/doc/sphinx-guides/source/_static/api/transform-oai-ore-jsonld.xq new file mode 100644 index 00000000000..6292f39fbde --- /dev/null +++ b/doc/sphinx-guides/source/_static/api/transform-oai-ore-jsonld.xq @@ -0,0 +1,16 @@ +declare option output:method "json"; + +let $parameters:={ 'method': 'json' } +for $record in /json + let $metadata:=$record/ore_003adescribes + + + let $json:= + + {$metadata/*} + {$record/_0040context} + + + + return if ($metadata) then + file:write("converted.json",$json, $parameters) \ No newline at end of file diff --git a/doc/sphinx-guides/source/_static/installation/files/etc/init.d/solr b/doc/sphinx-guides/source/_static/installation/files/etc/init.d/solr index 14df734cca7..303cd3c7534 100755 --- a/doc/sphinx-guides/source/_static/installation/files/etc/init.d/solr +++ b/doc/sphinx-guides/source/_static/installation/files/etc/init.d/solr @@ -5,7 +5,7 @@ # chkconfig: 35 92 08 # description: Starts and stops Apache Solr -SOLR_DIR="/usr/local/solr/solr-9.4.1" +SOLR_DIR="/usr/local/solr/solr-9.8.0" SOLR_COMMAND="bin/solr" SOLR_ARGS="-m 1g" SOLR_USER=solr diff --git a/doc/sphinx-guides/source/_static/installation/files/etc/systemd/solr.service b/doc/sphinx-guides/source/_static/installation/files/etc/systemd/solr.service index 8ccf7652a49..f3eed1479bc 100644 --- a/doc/sphinx-guides/source/_static/installation/files/etc/systemd/solr.service +++ b/doc/sphinx-guides/source/_static/installation/files/etc/systemd/solr.service @@ -5,9 +5,9 @@ After = syslog.target network.target remote-fs.target nss-lookup.target [Service] User = solr Type = forking -WorkingDirectory = /usr/local/solr/solr-9.4.1 -ExecStart = /usr/local/solr/solr-9.4.1/bin/solr start -m 1g -ExecStop = /usr/local/solr/solr-9.4.1/bin/solr stop +WorkingDirectory = /usr/local/solr/solr-9.8.0 +ExecStart = /usr/local/solr/solr-9.8.0/bin/solr start -m 1g +ExecStop = /usr/local/solr/solr-9.8.0/bin/solr stop LimitNOFILE=65000 LimitNPROC=65000 Restart=on-failure diff --git a/doc/sphinx-guides/source/_static/util/counter_daily.sh b/doc/sphinx-guides/source/_static/util/counter_daily.sh index 5095a83b7e2..ef10db26895 100644 --- a/doc/sphinx-guides/source/_static/util/counter_daily.sh +++ b/doc/sphinx-guides/source/_static/util/counter_daily.sh @@ -1,6 +1,6 @@ #! /bin/bash -COUNTER_PROCESSOR_DIRECTORY="/usr/local/counter-processor-1.05" +COUNTER_PROCESSOR_DIRECTORY="/usr/local/counter-processor-1.06" MDC_LOG_DIRECTORY="/usr/local/payara6/glassfish/domains/domain1/logs/mdc" # counter_daily.sh diff --git a/doc/sphinx-guides/source/admin/discoverability.rst b/doc/sphinx-guides/source/admin/discoverability.rst index 19ef7250a29..22ff66246f0 100644 --- a/doc/sphinx-guides/source/admin/discoverability.rst +++ b/doc/sphinx-guides/source/admin/discoverability.rst @@ -51,7 +51,7 @@ The Dataverse team has been working with Google on both formats. Google has `ind Signposting +++++++++++ -The Dataverse software supports `Signposting `_. This allows machines to request more information about a dataset through the `Link `_ HTTP header. +The Dataverse software supports `Signposting `_. This allows machines to request more information about a dataset through the `Link `_ HTTP header. Links to all enabled metadata export formats are given. See :ref:`metadata-export-formats` for a list. There are 2 Signposting profile levels, level 1 and level 2. In this implementation, * Level 1 links are shown `as recommended `_ in the "Link" diff --git a/doc/sphinx-guides/source/admin/external-tools.rst b/doc/sphinx-guides/source/admin/external-tools.rst index 346ca0b15ee..c3e71c13ac6 100644 --- a/doc/sphinx-guides/source/admin/external-tools.rst +++ b/doc/sphinx-guides/source/admin/external-tools.rst @@ -35,7 +35,13 @@ Configure the tool with the curl command below, making sure to replace the ``fab .. code-block:: bash - curl -X POST -H 'Content-type: application/json' http://localhost:8080/api/admin/externalTools --upload-file fabulousFileTool.json + curl -X POST -H 'Content-type: application/json' http://localhost:8080/api/admin/externalTools --upload-file fabulousFileTool.json + +This API is Superuser only. Note the endpoint difference (/api/externalTools instead of /api/admin/externalTools). + +.. code-block:: bash + + curl -s -H "X-Dataverse-key:$API_TOKEN" -X POST -H 'Content-type: application/json' http://localhost:8080/api/externalTools --upload-file fabulousFileTool.json Listing All External Tools in a Dataverse Installation ++++++++++++++++++++++++++++++++++++++++++++++++++++++ @@ -46,6 +52,12 @@ To list all the external tools that are available in a Dataverse installation: curl http://localhost:8080/api/admin/externalTools +This API is open to any user. Note the endpoint difference (/api/externalTools instead of /api/admin/externalTools). + +.. code-block:: bash + + curl http://localhost:8080/api/externalTools + Showing an External Tool in a Dataverse Installation ++++++++++++++++++++++++++++++++++++++++++++++++++++ @@ -56,6 +68,12 @@ To show one of the external tools that are available in a Dataverse installation export TOOL_ID=1 curl http://localhost:8080/api/admin/externalTools/$TOOL_ID +This API is open to any user. Note the endpoint difference (/api/externalTools instead of /api/admin/externalTools). + +.. code-block:: bash + + curl http://localhost:8080/api/externalTools/$TOOL_ID + Removing an External Tool From a Dataverse Installation +++++++++++++++++++++++++++++++++++++++++++++++++++++++ @@ -66,6 +84,12 @@ Assuming the external tool database id is "1", remove it with the following comm export TOOL_ID=1 curl -X DELETE http://localhost:8080/api/admin/externalTools/$TOOL_ID +This API is Superuser only. Note the endpoint difference (/api/externalTools instead of /api/admin/externalTools). + +.. code-block:: bash + + curl -s -H "X-Dataverse-key:$API_TOKEN" -X DELETE http://localhost:8080/api/externalTools/$TOOL_ID + .. _testing-external-tools: Testing External Tools diff --git a/doc/sphinx-guides/source/admin/harvestclients.rst b/doc/sphinx-guides/source/admin/harvestclients.rst index c4c63c80786..6ac8c480745 100644 --- a/doc/sphinx-guides/source/admin/harvestclients.rst +++ b/doc/sphinx-guides/source/admin/harvestclients.rst @@ -12,6 +12,8 @@ Harvesting is a process of exchanging metadata with other repositories. As a har Harvested records can be kept in sync with the original repository through scheduled incremental updates, daily or weekly. Alternatively, harvests can be run on demand, by the Admin. +.. _managing-harvesting-clients: + Managing Harvesting Clients --------------------------- @@ -23,8 +25,16 @@ The process of creating a new, or editing an existing client, is largely self-ex Please note that in some rare cases this GUI may fail to create a client because of some unexpected errors during these real time exchanges with an OAI server that is otherwise known to be valid. For example, in the past we have had issues with servers offering very long lists of sets (*really* long, in the thousands). To allow an admin to still be able to create a client in a situation like that, we provide the REST API that will do so without attempting any validation in real time. This obviously makes it the responsibility of the admin to supply the values that are definitely known to be valid - a working OAI url, the name of a set that does exist on the server, and/or a supported metadata format. See the :ref:`managing-harvesting-clients-api` section of the :doc:`/api/native-api` guide for more information. -Note that as of 5.13, a new entry "Custom HTTP Header" has been added to the Step 1. of Create or Edit form. This optional field can be used to configure this client with a specific HTTP header to be added to every OAI request. This is to accommodate a (rare) use case where the remote server may require a special token of some kind in order to offer some content not available to other clients. Most OAI servers offer the same publicly-available content to all clients, so few admins will have a use for this feature. It is however on the very first, Step 1. screen in case the OAI server requires this token even for the "ListSets" and "ListMetadataFormats" requests, which need to be sent in the Step 2. of creating or editing a client. Multiple headers can be supplied separated by `\\n` - actual "backslash" and "n" characters, not a single "new line" character. +"Custom HTTP Header" is part of step 1 of the Create or Edit form. This optional field can be used to configure this client with a specific HTTP header to be added to every OAI request. This is to accommodate a (rare) use case where the remote server may require a special token of some kind in order to offer some content not available to other clients. Most OAI servers offer the same publicly-available content to all clients, so few admins will have a use for this feature. However, it appears in Step 1 of the form screen in case the OAI server requires this token even for the "ListSets" and "ListMetadataFormats" requests, which need to be sent in Step 2 of creating or editing a client. Multiple headers can be supplied separated by `\\n` - actual "backslash" and "n" characters, not a single "new line" character. + +.. _harvesting-from-datacite: + +Harvesting from Datacite +~~~~~~~~~~~~~~~~~~~~~~~~ + +It is possible to harvest metadata directly from DataCite. Their OAI gateway (https://oai.datacite.org/oai) serves records for every DOI they have registered. Therefore, it is now possible to harvest metadata from any participating institution even if they do not maintain an OAI server of their own. Their OAI implementation offers a concept of a "dynamic set", making it possible to use any query supported by the DataCite search API as though it were a "set". This makes harvesting from them extra flexible, allowing users to harvest virtually any arbitrary subset of metadata records, potentially spanning multiple institutions and registration authorities. +For various reasons, in order to take advantage of this feature harvesting clients must be created via the ``/api/harvest/clients`` API. Once configured however, harvests can be run from the Harvesting Clients control panel in the UI. See the :ref:`managing-harvesting-clients-api` section of the :doc:`/api/native-api` guide for more information. How to Stop a Harvesting Run in Progress ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -48,7 +58,14 @@ Each harvesting client run logs a separate file per run to the app server's defa Note that you'll want to run a minimum of Dataverse Software 4.6, optimally 4.18 or beyond, for the best OAI-PMH interoperability. +Harvesting Client Changelog +--------------------------- + +- As of Dataverse 6.6, it is possible to harvest metadata directly from DataCite. See :ref:`harvesting-from-datacite`. +- As of Dataverse 6.6, the publisher value of harvested datasets is now attributed to the dataset's distributor instead of its producer. This change affects all newly harvested datasets. For more information, see https://github.com/IQSS/dataverse/pull/9013 +- As of Dataverse 5.13, a new entry called "Custom HTTP Header" has been added to the Step 1. of Create or Edit form. For usage see :ref:`managing-harvesting-clients`. + Harvesting Non-OAI-PMH -~~~~~~~~~~~~~~~~~~~~~~ +---------------------- -`DOI2PMH `__ is a community-driven project intended to allow OAI-PMH harvesting from non-OAI-PMH sources. \ No newline at end of file +`DOI2PMH `__ is a community-driven project intended to allow OAI-PMH harvesting from non-OAI-PMH sources. diff --git a/doc/sphinx-guides/source/admin/index.rst b/doc/sphinx-guides/source/admin/index.rst index 633842044b4..bada3ea20b4 100755 --- a/doc/sphinx-guides/source/admin/index.rst +++ b/doc/sphinx-guides/source/admin/index.rst @@ -11,6 +11,7 @@ This guide documents the functionality only available to superusers (such as "da **Contents:** .. toctree:: + :maxdepth: 2 dashboard external-tools diff --git a/doc/sphinx-guides/source/admin/make-data-count.rst b/doc/sphinx-guides/source/admin/make-data-count.rst index 51bc2c4a9fe..2d9f4a94b55 100644 --- a/doc/sphinx-guides/source/admin/make-data-count.rst +++ b/doc/sphinx-guides/source/admin/make-data-count.rst @@ -84,9 +84,9 @@ Configure Counter Processor * Change to the directory where you installed Counter Processor. - * ``cd /usr/local/counter-processor-1.05`` + * ``cd /usr/local/counter-processor-1.06`` -* Download :download:`counter-processor-config.yaml <../_static/admin/counter-processor-config.yaml>` to ``/usr/local/counter-processor-1.05``. +* Download :download:`counter-processor-config.yaml <../_static/admin/counter-processor-config.yaml>` to ``/usr/local/counter-processor-1.06``. * Edit the config file and pay particular attention to the FIXME lines. @@ -99,7 +99,7 @@ Soon we will be setting up a cron job to run nightly but we start with a single * Change to the directory where you installed Counter Processor. - * ``cd /usr/local/counter-processor-1.05`` + * ``cd /usr/local/counter-processor-1.06`` * If you are running Counter Processor for the first time in the middle of a month, you will need create blank log files for the previous days. e.g.: diff --git a/doc/sphinx-guides/source/admin/metadatacustomization.rst b/doc/sphinx-guides/source/admin/metadatacustomization.rst index e5326efebef..df07b65153b 100644 --- a/doc/sphinx-guides/source/admin/metadatacustomization.rst +++ b/doc/sphinx-guides/source/admin/metadatacustomization.rst @@ -244,6 +244,8 @@ Each of the three main sections own sets of properties: | | #metadataBlock) | | | +---------------------------+--------------------------------------------------------+----------------------------------------------------------+-----------------------+ +.. _cvoc-props: + #controlledVocabulary (enumerated) properties ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -259,10 +261,10 @@ Each of the three main sections own sets of properties: | | | an existing #datasetField from | | | | another metadata block.) | +--------------+--------------------------------------------+-----------------------------------------+ -| Value | A short display string, representing | Free text | -| | an enumerated value for this field. If | | -| | the identifier property is empty, | | -| | this value is used as the identifier. | | +| Value | A short display string, representing | Free text. When defining a boolean, the | +| | an enumerated value for this field. If | values "True" and "False" are | +| | the identifier property is empty, | recommended and "Unknown" can be added | +| | this value is used as the identifier. | if needed. | +--------------+--------------------------------------------+-----------------------------------------+ | identifier | A string used to encode the selected | Free text | | | enumerated value of a field. If this | | @@ -270,7 +272,11 @@ Each of the three main sections own sets of properties: | | โ€œValueโ€ field is used as the identifier. | | +--------------+--------------------------------------------+-----------------------------------------+ | displayOrder | Control the order in which the enumerated | Non-negative integer. | -| | values are displayed for selection. | | +| | values are displayed for selection. When | | +| | adding new values, you don't have to add | | +| | them at the end. You can renumber existing | | +| | values to update the order in which they | | +| | appear. | | +--------------+--------------------------------------------+-----------------------------------------+ FieldType definitions @@ -293,6 +299,9 @@ FieldType definitions +---------------+------------------------------------+ | text | Any text other than newlines may | | | be entered into this field. | +| | The text fieldtype may be used to | +| | define a boolean (see "Value" | +| | under :ref:`cvoc-props`). | +---------------+------------------------------------+ | textbox | Any text may be entered. For | | | input, the Dataverse Software | @@ -539,7 +548,7 @@ a necessary re-index, but for your custom metadata you will need to keep track o Please note also that if you are going to make a pull request updating ``conf/solr/schema.xml`` with fields you have added, you should first load all the custom metadata blocks in ``scripts/api/data/metadatablocks`` (including ones you -don't care about) to create a complete list of fields. (This might change in the future.) +don't care about) to create a complete list of fields. (This might change in the future.) Please see :ref:`update-solr-schema-dev` in the Developer Guide. Reloading a Metadata Block -------------------------- @@ -559,8 +568,7 @@ Using External Vocabulary Services The Dataverse software has a mechanism to associate specific fields defined in metadata blocks with a vocabulary(ies) managed by external services. The mechanism relies on trusted third-party Javascripts. The mapping from field type to external vocabulary(ies) is managed via the :ref:`:CVocConf <:CVocConf>` setting. -*This functionality is considered 'experimental'. It may require significant effort to configure and is likely to evolve in subsequent Dataverse software releases.* - +*This functionality may require significant effort to configure and is likely to evolve in subsequent Dataverse software releases.* The effect of configuring this mechanism is similar to that of defining a field in a metadata block with 'allowControlledVocabulary=true': @@ -585,6 +593,9 @@ Configuration involves specifying which fields are to be mapped, to which Solr f These are all defined in the :ref:`:CVocConf <:CVocConf>` setting as a JSON array. Details about the required elements as well as example JSON arrays are available at https://github.com/gdcc/dataverse-external-vocab-support, along with an example metadata block that can be used for testing. The scripts required can be hosted locally or retrieved dynamically from https://gdcc.github.io/ (similar to how dataverse-previewers work). +Since external vocabulary scripts can change how fields are indexed (storing an identifier and name and/or values in different languages), +updating the Solr schema as described in :ref:`update-solr-schema` should be done after adding new scripts to your configuration. + Please note that in addition to the :ref:`:CVocConf` described above, an alternative is the :ref:`:ControlledVocabularyCustomJavaScript` setting. Protecting MetadataBlocks diff --git a/doc/sphinx-guides/source/admin/troubleshooting.rst b/doc/sphinx-guides/source/admin/troubleshooting.rst index acbdcaae17e..178938af6dc 100644 --- a/doc/sphinx-guides/source/admin/troubleshooting.rst +++ b/doc/sphinx-guides/source/admin/troubleshooting.rst @@ -162,7 +162,9 @@ A full backup of the table can be made with pg_dump, for example: ``pg_dump --table=actionlogrecord --data-only > /tmp/actionlogrecord_backup.sql`` -(In the example above, the output will be saved in raw SQL format. It is portable and human-readable, but uses a lot of space. It does, however, compress very well. Add the ``-Fc`` option to save the output in a proprietary, binary format that's already compressed). +In the example above, the output will be saved in raw SQL format. It is portable and human-readable, but uses a lot of space. It does, however, compress very well. + +Add the ``-Fc`` option to save the output in a proprietary, compressed binary format which will dump and restore much more quickly, but note that in this format dead tuples will be copied as well. To reduce the amount of storage consumed by the newly-trimmed ``actionlogrecord`` table, you must issue ``vacuum full actionlogrecord`` before dumping the database in this custom format. Getting Help diff --git a/doc/sphinx-guides/source/api/auth.rst b/doc/sphinx-guides/source/api/auth.rst index eae3bd3c969..8dffb914e29 100644 --- a/doc/sphinx-guides/source/api/auth.rst +++ b/doc/sphinx-guides/source/api/auth.rst @@ -81,6 +81,27 @@ To test if bearer tokens are working, you can try something like the following ( curl -H "Authorization: Bearer $TOKEN" http://localhost:8080/api/users/:me +To register a new user who has authenticated via an OIDC provider, the following endpoint should be used: + +.. code-block:: bash + + curl -H "Authorization: Bearer $TOKEN" -X POST http://localhost:8080/api/users/register --data '{"termsAccepted":true}' + +By default, the Bearer token is expected to include the following claims that will be used to create the user account: + +- ``username`` +- ``firstName`` +- ``lastName`` +- ``emailAddress`` + +The one parameter required by default is ``termsAccepted`` which must be set to true, indicating that the user has seen and accepted the Terms of Use of the installation. + +If the feature flag ``api-bearer-auth-handle-tos-acceptance-in-idp`` is enabled (along with the ``api-bearer-auth`` feature flag), Dataverse assumes that the Terms of Service acceptance was handled by the identity provider, e.g. in the OIDC ``consent`` dialog, and the ``termsAccepted`` parameter is not needed. + +There is another flag called ``api-bearer-auth-provide-missing-claims`` that can be enabled (along with the ``api-bearer-auth`` feature flag) to allow sending missing user claims in the registration JSON. This is useful when the identity provider does not supply the necessary claims listed above. If properties are provided in the JSON, but corresponding claims already exist in the identity provider, an error will be thrown, outlining the conflicting properties. Note that supplying missing claims is configured via a separate feature flag because using it may introduce user impersonation issues, for example if the identity provider does not provide an email field and the user submits an email address they do not own. + +In all cases, the submitted JSON can optionally include the fields ``position`` or ``affiliation``, which will be added to the user's Dataverse account. These fields are optional, and if not provided, they will be persisted as empty in Dataverse. + Signed URLs ----------- diff --git a/doc/sphinx-guides/source/api/changelog.rst b/doc/sphinx-guides/source/api/changelog.rst index 14958095658..73935e6562a 100644 --- a/doc/sphinx-guides/source/api/changelog.rst +++ b/doc/sphinx-guides/source/api/changelog.rst @@ -7,6 +7,19 @@ This API changelog is experimental and we would love feedback on its usefulness. :local: :depth: 1 +v6.6 +---- + +- The JSON representation for a datasetVersion sent or received in API calls has changed such that + + - "versionNote" -> "deaccessionNote" + - "archiveNote" --> "deaccessionLink" + - These may be non-null for deaccessioned versions and an optional new "versionNote" field indicating the reason a version was created may be present on any datasetversion. + +- **/api/metadatablocks** is no longer returning duplicated metadata properties and does not omit metadata properties when called. +- **/api/roles**: :ref:`show-role` now properly returns 403 Forbidden instead of 401 Unauthorized when you pass a working API token that doesn't have the right permission. +- The content type for the ``schema.org`` dataset metadata export format has been corrected. It was ``application/json`` and now it is ``application/ld+json``. See also :ref:`export-dataset-metadata-api`. + v6.5 ---- diff --git a/doc/sphinx-guides/source/api/index.rst b/doc/sphinx-guides/source/api/index.rst index dd195aa9d62..5ff7626271d 100755 --- a/doc/sphinx-guides/source/api/index.rst +++ b/doc/sphinx-guides/source/api/index.rst @@ -9,6 +9,7 @@ API Guide **Contents:** .. toctree:: + :maxdepth: 2 intro getting-started @@ -24,4 +25,4 @@ API Guide linkeddatanotification apps faq - changelog \ No newline at end of file + changelog diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index dabca195e37..0f37bde35b8 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -128,12 +128,23 @@ Note that setting any of these fields overwrites the previous configuration. When it comes to omitting these fields in the JSON: -- Omitting ``facetIds`` or ``metadataBlockNames`` causes the Dataverse collection to inherit the corresponding configuration from its parent. -- Omitting ``inputLevels`` removes any existing custom input levels in the Dataverse collection. -- Omitting the entire ``metadataBlocks`` object in the request JSON would exclude the three sub-objects, resulting in the application of the two changes described above. +- Omitting ``facetIds`` or ``metadataBlockNames`` causes no change to the Dataverse collection. To delete the current configuration and inherit the corresponding configuration from its parent include the flag ``inheritFacetsFromParent`` and/or ``inheritMetadataBlocksFromParent`` respectively. +- Omitting ``inputLevels`` causes no change to the Dataverse collection. Including the flag ``inheritMetadataBlocksFromParent`` will cause the custom ``inputLevels`` to be deleted and inherited from the parent. +- Omitting the entire ``metadataBlocks`` object in the request JSON would cause no change to the ``inputLevels``, ``facetIds`` or ``metadataBlockNames`` of the Dataverse collection. To obtain an example of how these objects are included in the JSON file, download :download:`dataverse-complete-optional-params.json <../_static/api/dataverse-complete-optional-params.json>` file and modify it to suit your needs. +To force the configurations to be deleted and inherited from the parent's configuration include the following ``metadataBlocks`` object in your JSON + +.. code-block:: json + + "metadataBlocks": { + "inheritMetadataBlocksFromParent": true, + "inheritFacetsFromParent": true + } + +.. note:: Including both the list ``metadataBlockNames`` and the flag ``"inheritMetadataBlocksFromParent": true`` will result in an error being returned {"status": "ERROR", "message": "Metadata block can not contain both metadataBlockNames and inheritMetadataBlocksFromParent: true"}. The same is true for ``facetIds`` and ``inheritFacetsFromParent``. + See also :ref:`collection-attributes-api`. .. _view-dataverse: @@ -166,6 +177,15 @@ Usage example: curl "https://demo.dataverse.org/api/dataverses/root?returnOwners=true" +If you want to include the child count of the Dataverse, which represents the number of dataverses, datasets, or files within the dataverse, you must set ``returnChildCount`` query parameter to ``true``. Please note that this count is for direct children only. It doesn't count children of subdataverses. + +Usage example: + +.. code-block:: bash + + curl "https://demo.dataverse.org/api/dataverses/root?returnChildCount=true" + + To view an unpublished Dataverse collection: .. code-block:: bash @@ -424,13 +444,13 @@ Creates a new role under Dataverse collection ``id``. Needs a json file with the export SERVER_URL=https://demo.dataverse.org export ID=root - curl -H "X-Dataverse-key:$API_TOKEN" -X POST "$SERVER_URL/api/dataverses/$ID/roles" --upload-file roles.json + curl -H "X-Dataverse-key:$API_TOKEN" -H "Content-type:application/json" -X POST "$SERVER_URL/api/dataverses/$ID/roles" --upload-file roles.json The fully expanded example above (without environment variables) looks like this: .. code-block:: bash - curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X POST -H "Content-type:application/json" "https://demo.dataverse.org/api/dataverses/root/roles" --upload-file roles.json + curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -H "Content-type:application/json" -X POST "https://demo.dataverse.org/api/dataverses/root/roles" --upload-file roles.json For ``roles.json`` see :ref:`json-representation-of-a-role` @@ -529,6 +549,8 @@ The fully expanded example above (without environment variables) looks like this curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X DELETE "https://demo.dataverse.org/api/dataverses/root/assignments/6" +.. _list-metadata-blocks-for-a-collection: + List Metadata Blocks Defined on a Dataverse Collection ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -556,6 +578,7 @@ This endpoint supports the following optional query parameters: - ``returnDatasetFieldTypes``: Whether or not to return the dataset field types present in each metadata block. If not set, the default value is false. - ``onlyDisplayedOnCreate``: Whether or not to return only the metadata blocks that are displayed on dataset creation. If ``returnDatasetFieldTypes`` is true, only the dataset field types shown on dataset creation will be returned within each metadata block. If not set, the default value is false. +- ``datasetType``: Optionally return additional fields from metadata blocks that are linked with a particular dataset type (see :ref:`dataset-types` in the User Guide). Pass a single dataset type as a string. For a list of dataset types you can pass, see :ref:`api-list-dataset-types`. An example using the optional query parameters is presented below: @@ -564,14 +587,17 @@ An example using the optional query parameters is presented below: export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx export SERVER_URL=https://demo.dataverse.org export ID=root + export DATASET_TYPE=software - curl -H "X-Dataverse-key:$API_TOKEN" "$SERVER_URL/api/dataverses/$ID/metadatablocks?returnDatasetFieldTypes=true&onlyDisplayedOnCreate=true" + curl -H "X-Dataverse-key:$API_TOKEN" "$SERVER_URL/api/dataverses/$ID/metadatablocks?returnDatasetFieldTypes=true&onlyDisplayedOnCreate=true&datasetType=$DATASET_TYPE" The fully expanded example above (without environment variables) looks like this: .. code-block:: bash - curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" "https://demo.dataverse.org/api/dataverses/root/metadatablocks?returnDatasetFieldTypes=true&onlyDisplayedOnCreate=true" + curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" "https://demo.dataverse.org/api/dataverses/root/metadatablocks?returnDatasetFieldTypes=true&onlyDisplayedOnCreate=true&datasetType=software" + +.. _define-metadata-blocks-for-a-dataverse-collection: Define Metadata Blocks for a Dataverse Collection ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -598,6 +624,8 @@ The fully expanded example above (without environment variables) looks like this curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X POST -H "Content-type:application/json" --upload-file define-metadatablocks.json "https://demo.dataverse.org/api/dataverses/root/metadatablocks" +An alternative to defining metadata blocks at a collection level is to create and use a dataset type. See :ref:`api-link-dataset-type`. + Determine if a Dataverse Collection Inherits Its Metadata Blocks from Its Parent ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -1067,6 +1095,7 @@ The following attributes are supported: * ``description`` Description * ``affiliation`` Affiliation * ``filePIDsEnabled`` ("true" or "false") Restricted to use by superusers and only when the :ref:`:AllowEnablingFilePIDsPerCollection <:AllowEnablingFilePIDsPerCollection>` setting is true. Enables or disables registration of file-level PIDs in datasets within the collection (overriding the instance-wide setting). +* ``requireFilesToPublishDataset`` ("true" or "false") Restricted to use by superusers. Defines if Dataset needs files in order to be published. If not set the determination will be made through inheritance by checking the owners of this collection. Publishing by a superusers will not be blocked. See also :ref:`update-dataverse-api`. @@ -1087,15 +1116,28 @@ This endpoint expects a JSON with the following format:: { "datasetFieldTypeName": "datasetFieldTypeName1", "required": true, - "include": true + "include": true, + "displayOnCreate": null }, { "datasetFieldTypeName": "datasetFieldTypeName2", "required": true, - "include": true + "include": true, + "displayOnCreate": true } ] +.. note:: + Required fields will always be displayed regardless of their displayOnCreate setting, as this is necessary for dataset creation. + When displayOnCreate is null, the field's default display behavior is used. + +Parameters: + +- ``datasetFieldTypeName``: Name of the metadata field +- ``required``: Whether the field is required (boolean) +- ``include``: Whether the field is included (boolean) +- ``displayOnCreate`` (optional): Whether the field is displayed during dataset creation, even when not required (boolean) + .. code-block:: bash export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx @@ -1144,6 +1186,209 @@ Use the ``/settings`` API to enable or disable the enforcement of storage quotas curl -X PUT -d 'true' http://localhost:8080/api/admin/settings/:UseStorageQuotas +List All Collection Featured Items +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +List the featured items configured for a given Dataverse collection ``id``: + +.. code-block:: bash + + export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx + export SERVER_URL=https://demo.dataverse.org + export ID=root + + curl -H "X-Dataverse-key:$API_TOKEN" "$SERVER_URL/api/dataverses/$ID/featuredItems" + +The fully expanded example above (without environment variables) looks like this: + +.. code-block:: bash + + curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" "https://demo.dataverse.org/api/dataverses/root/featuredItems" + +Update All Collection Featured Items +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Updates all featured items in the given Dataverse collection ``id``. + +The data sent to the endpoint represents the desired final state of the featured items in the Dataverse collection and overwrites any existing featured items configuration. + +The parameters ``id``, ``content``, ``displayOrder``, and ``fileName`` must be specified as many times as the number of items we want to add or update. The order in which these parameters are repeated must match to ensure they correspond to the same featured item. + +The ``file`` parameter must be specified for each image we want to attach to featured items. Note that images can be shared between featured items, so ``fileName`` can have the same value in different featured items. + +The ``id`` parameter must be ``0`` for new items or set to the item's identifier for updates. The ``fileName`` parameter should be empty to exclude an image or match the name of a file sent in a ``file`` parameter to set a new image. ``keepFile`` must always be set to ``false``, unless it's an update to a featured item where we want to preserve the existing image, if one exists. + +Note that any existing featured item not included in the call with its associated identifier and corresponding properties will be removed from the collection. + +The following example creates two featured items, with an image assigned to the second one: + +.. code-block:: bash + + export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx + export SERVER_URL=https://demo.dataverse.org + export ID=root + + export FIRST_ITEM_CONTENT='Content 1' + export FIRST_ITEM_DISPLAY_ORDER=1 + + export SECOND_ITEM_IMAGE_FILENAME='image.png' + export SECOND_ITEM_CONTENT='Content 2' + export SECOND_ITEM_DISPLAY_ORDER=2 + + curl -H "X-Dataverse-key:$API_TOKEN" \ + -X PUT \ + -F "id=0" -F "id=0" \ + -F "content=$FIRST_ITEM_CONTENT" -F "content=$SECOND_ITEM_CONTENT" \ + -F "displayOrder=$FIRST_ITEM_DISPLAY_ORDER" -F "displayOrder=$SECOND_ITEM_DISPLAY_ORDER" \ + -F "fileName=" -F "fileName=$SECOND_ITEM_IMAGE_FILENAME" \ + -F "keepFile=false" -F "keepFile=false" \ + -F "file=@$SECOND_ITEM_IMAGE_FILENAME" \ + "$SERVER_URL/api/dataverses/$ID/featuredItems" + + +The fully expanded example above (without environment variables) looks like this: + +.. code-block:: bash + + curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" \ + -X PUT \ + -F "id=0" -F "id=0" \ + -F "content=Content 1" -F "content=Content 2" \ + -F "displayOrder=1" -F "displayOrder=2" \ + -F "fileName=" -F "fileName=image.png" \ + -F "keepFile=false" -F "keepFile=false" \ + -F "file=@image.png" \ + "https://demo.dataverse.org/api/dataverses/root/featuredItems" + +The following example creates one featured item and updates a second one, keeping the existing image it may have had: + +.. code-block:: bash + + curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" \ + -X PUT \ + -F "id=0" -F "id=1" \ + -F "content=Content 1" -F "content=Updated content 2" \ + -F "displayOrder=1" -F "displayOrder=2" \ + -F "fileName=" -F "fileName=" \ + -F "keepFile=false" -F "keepFile=true" \ + "https://demo.dataverse.org/api/dataverses/root/featuredItems" + +Delete All Collection Featured Items +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Deletes the featured items configured for a given Dataverse collection ``id``: + +.. code-block:: bash + + export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx + export SERVER_URL=https://demo.dataverse.org + export ID=root + + curl -H "X-Dataverse-key: $API_TOKEN" -X DELETE "$SERVER_URL/api/dataverses/$ID/featuredItems" + +The fully expanded example above (without environment variables) looks like this: + +.. code-block:: bash + + curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X DELETE "https://demo.dataverse.org/api/dataverses/root/featuredItems" + +Create a Collection Featured Item +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Creates a featured item in the given Dataverse collection ``id``: + +.. code-block:: bash + + export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx + export IMAGE_FILENAME='image.png' + export CONTENT='Content for featured item.' + export DISPLAY_ORDER=1 + export SERVER_URL=https://demo.dataverse.org + export ID=root + + curl -H "X-Dataverse-key:$API_TOKEN" -X POST -F "file=@$IMAGE_FILENAME" -F "content=$CONTENT" -F "displayOrder=$DISPLAY_ORDER" "$SERVER_URL/api/dataverses/$ID/featuredItems" + +The fully expanded example above (without environment variables) looks like this: + +.. code-block:: bash + + curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X POST -F "file=@image.png" -F "content=Content for featured item." -F "displayOrder=1" "https://demo.dataverse.org/api/dataverses/root/featuredItems" + +A featured item may or may not contain an image. If you wish to create it without an image, omit the file parameter in the request. + +Update a Collection Featured Item +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Updates a featured item given its ``id``: + +.. code-block:: bash + + export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx + export IMAGE_FILENAME='image.png' + export CONTENT='Content for featured item.' + export DISPLAY_ORDER=1 + export SERVER_URL=https://demo.dataverse.org + export ID=1 + + curl -H "X-Dataverse-key:$API_TOKEN" -X PUT -F "file=@$IMAGE_FILENAME" -F "content=$CONTENT" -F "displayOrder=$DISPLAY_ORDER" "$SERVER_URL/api/dataverseFeaturedItems/$ID" + +The fully expanded example above (without environment variables) looks like this: + +.. code-block:: bash + + curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X PUT -F "file=@image.png" -F "content=Content for featured item." -F "displayOrder=1" "https://demo.dataverse.org/api/dataverseFeaturedItems/1" + +``content`` and ``displayOrder`` must always be provided; otherwise, an error will occur. Use the ``file`` parameter to set a new image for the featured item. To keep the existing image, omit ``file`` and send ``keepFile=true``. To remove the image, omit the file parameter. + +Updating the featured item keeping the existing image: + +.. code-block:: bash + + curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X PUT -F "keepFile=true" -F "content=Content for featured item." -F "displayOrder=1" "https://demo.dataverse.org/api/dataverseFeaturedItems/1" + +Updating the featured item removing the existing image: + +.. code-block:: bash + + curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X PUT -F "content=Content for featured item." -F "displayOrder=1" "https://demo.dataverse.org/api/dataverseFeaturedItems/1" + +Delete a Collection Featured Item +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Deletes a featured item given its ``id``: + +.. code-block:: bash + + export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx + export SERVER_URL=https://demo.dataverse.org + export ID=1 + + curl -H "X-Dataverse-key:$API_TOKEN" -X DELETE "$SERVER_URL/api/dataverseFeaturedItems/$ID" + +The fully expanded example above (without environment variables) looks like this: + +.. code-block:: bash + + curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X DELETE "https://demo.dataverse.org/api/dataverseFeaturedItems/1" + +Get a Collection Featured Item Image +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Returns the image of a featured item if one is assigned, given the featured item ``id``: + +.. code-block:: bash + + export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx + export SERVER_URL=https://demo.dataverse.org + export ID=1 + + curl -H "X-Dataverse-key:$API_TOKEN" -X GET "$SERVER_URL/api/access/dataverseFeaturedItemImage/{ID}" + +The fully expanded example above (without environment variables) looks like this: + +.. code-block:: bash + + curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X GET "https://demo.dataverse.org/api/access/dataverseFeaturedItemImage/1" Datasets -------- @@ -1261,7 +1506,12 @@ It returns a list of versions with their metadata, and file list: "createTime": "2015-04-20T09:57:32Z", "license": { "name": "CC0 1.0", - "uri": "http://creativecommons.org/publicdomain/zero/1.0" + "uri": "http://creativecommons.org/publicdomain/zero/1.0", + "iconUri": "https://licensebuttons.net/p/zero/1.0/88x31.png", + "rightsIdentifier": "CC0", + "rightsIdentifierScheme": "Creative Commons", + "schemeUri": "https://creativecommons.org/", + "languageCode": "en", }, "termsOfAccess": "You need to request for access.", "fileAccessRequest": true, @@ -1282,7 +1532,12 @@ It returns a list of versions with their metadata, and file list: "createTime": "2015-04-20T09:43:45Z", "license": { "name": "CC0 1.0", - "uri": "http://creativecommons.org/publicdomain/zero/1.0" + "uri": "http://creativecommons.org/publicdomain/zero/1.0", + "iconUri": "https://licensebuttons.net/p/zero/1.0/88x31.png", + "rightsIdentifier": "CC0", + "rightsIdentifierScheme": "Creative Commons", + "schemeUri": "https://creativecommons.org/", + "languageCode": "en", }, "termsOfAccess": "You need to request for access.", "fileAccessRequest": true, @@ -1294,6 +1549,8 @@ It returns a list of versions with their metadata, and file list: The optional ``excludeFiles`` parameter specifies whether the files should be listed in the output. It defaults to ``true``, preserving backward compatibility. (Note that for a dataset with a large number of versions and/or files having the files included can dramatically increase the volume of the output). A separate ``/files`` API can be used for listing the files, or a subset thereof in a given version. +The optional ``excludeMetadataBlocks`` parameter specifies whether the metadata blocks should be listed in the output. It defaults to ``false``, preserving backward compatibility. (Note that for a dataset with a large number of versions and/or metadata blocks having the metadata blocks included can dramatically increase the volume of the output). + The optional ``offset`` and ``limit`` parameters can be used to specify the range of the versions list to be shown. This can be used to paginate through the list in a dataset with a large number of versions. @@ -1318,6 +1575,12 @@ The fully expanded example above (without environment variables) looks like this The optional ``excludeFiles`` parameter specifies whether the files should be listed in the output (defaults to ``true``). Note that a separate ``/files`` API can be used for listing the files, or a subset thereof in a given version. +.. code-block:: bash + + curl "https://demo.dataverse.org/api/datasets/24/versions/1.0?excludeMetadataBlocks=false" + +The optional ``excludeMetadataBlocks`` parameter specifies whether the metadata blocks should be listed in the output (defaults to ``false``). + By default, deaccessioned dataset versions are not included in the search when applying the :latest or :latest-published identifiers. Additionally, when filtering by a specific version tag, you will get a "not found" error if the version is deaccessioned and you do not enable the ``includeDeaccessioned`` option described below. @@ -1344,6 +1607,8 @@ Export Metadata of a Dataset in Various Formats |CORS| Export the metadata of the current published version of a dataset in various formats. +To get a list of available formats, see :ref:`available-exporters` and :ref:`get-export-formats`. + See also :ref:`batch-exports-through-the-api` and the note below: .. code-block:: bash @@ -1360,9 +1625,30 @@ The fully expanded example above (without environment variables) looks like this curl "https://demo.dataverse.org/api/datasets/export?exporter=ddi&persistentId=doi:10.5072/FK2/J8SJZB" -.. note:: Supported exporters (export formats) are ``ddi``, ``oai_ddi``, ``dcterms``, ``oai_dc``, ``schema.org`` , ``OAI_ORE`` , ``Datacite``, ``oai_datacite`` and ``dataverse_json``. Descriptive names can be found under :ref:`metadata-export-formats` in the User Guide. +.. _available-exporters: + +Available Dataset Metadata Exporters +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The following dataset metadata exporters ship with Dataverse: + +- ``Datacite`` +- ``dataverse_json`` +- ``dcterms`` +- ``ddi`` +- ``oai_datacite`` +- ``oai_dc`` +- ``oai_ddi`` +- ``OAI_ORE`` +- ``schema.org`` + +These are the strings to pass as ``$METADATA_FORMAT`` in the examples above. Descriptive names for each format can be found under :ref:`metadata-export-formats` in the User Guide. -.. note:: Additional exporters can be enabled, as described under :ref:`external-exporters` in the Installation Guide. To discover the machine-readable name of each exporter (e.g. ``ddi``), check :ref:`inventory-of-external-exporters` or ``getFormatName`` in the exporter's source code. +Additional exporters can be enabled, as described under :ref:`external-exporters` in the Installation Guide. The machine-readable name/identifier for each external exporter can be found under :ref:`inventory-of-external-exporters`. If you are interested in creating your own exporter, see :doc:`/developers/metadataexport`. + +To discover the machine-readable name of exporters (e.g. ``ddi``) that have been enabled on the installation of Dataverse you are using see :ref:`get-export-formats`. Alternatively, you can use the Signposting "linkset" API documented under :ref:`signposting-api`. + +To discover the machine-readable name of exporters generally, check :ref:`inventory-of-external-exporters` or ``getFormatName`` in the exporter's source code. Schema.org JSON-LD ^^^^^^^^^^^^^^^^^^ @@ -1376,6 +1662,8 @@ Both forms are valid according to Google's Structured Data Testing Tool at https The standard has further evolved into a format called Croissant. For details, see :ref:`schema.org-head` in the Admin Guide. +The ``schema.org`` format changed after Dataverse 6.4 as well. Previously its content type was "application/json" but now it is "application/ld+json". + List Files in a Dataset ~~~~~~~~~~~~~~~~~~~~~~~ @@ -1728,6 +2016,27 @@ The fully expanded example above (without environment variables) looks like this curl "https://demo.dataverse.org/api/datasets/24/versions/:latest-published/compare/:draft" +Get Versions of a Dataset with Summary of Changes +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Returns a list of versions for a given dataset including a summary of differences between consecutive versions where available. Draft versions will only +be available to users who have permission to view unpublished drafts. The api token is optional. + +.. code-block:: bash + + export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx + export SERVER_URL=https://demo.dataverse.org + export PERSISTENT_IDENTIFIER=doi:10.5072/FK2/BCCP9Z + + curl -H "X-Dataverse-key: $API_TOKEN" -X PUT "$SERVER_URL/api/datasets/:persistentId/versions/compareSummary?persistentId=$PERSISTENT_IDENTIFIER" + +The fully expanded example above (without environment variables) looks like this: + +.. code-block:: bash + + curl -H "X-Dataverse-key: xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X PUT "https://demo.dataverse.org/api/datasets/:persistentId/versions/compareSummary?persistentId=doi:10.5072/FK2/BCCP9Z" + + Update Metadata For a Dataset ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -2406,6 +2715,29 @@ Usage example: .. note:: Keep in mind that you can combine all of the above query parameters depending on the results you are looking for. +Get the Download count of a Dataset +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Shows the total number of downloads requested for a dataset. If MDC is enabled the count will be limited to the time before MDC start if the optional `includeMDC` parameter is not included or set to False. +Setting `includeMDC` to True will ignore the `:MDCStartDate` setting and return a total count. + +.. code-block:: bash + + API_TOKEN='xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx' + export DATASET_ID=1 + export includeMDC=True + + curl -s -H "X-Dataverse-key:$API_TOKEN" -X GET http://localhost:8080/api/datasets/$DATASET_ID/download/count + curl -s -H "X-Dataverse-key:$API_TOKEN" -X GET http://localhost:8080/api/datasets/$DATASET_ID/download/count?includeMDC=true + +The fully expanded example above (without environment variables) looks like this: + +.. code-block:: bash + + curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" "https://demo.dataverse.org/api/datasets/1/download/count?includeMDC=False" + + + Submit a Dataset for Review ~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -2457,6 +2789,7 @@ The fully expanded example above (without environment variables) looks like this The review process can sometimes resemble a tennis match, with the authors submitting and resubmitting the dataset over and over until the curators are satisfied. Each time the curators send a "reason for return" via API, that reason is sent by email and is persisted into the database, stored at the dataset version level. Note the reason is required, unless the `disable-return-to-author-reason` feature flag has been set (see :ref:`feature-flags`). Reason is a free text field and could be as simple as "The author would like to modify his dataset", "Files are missing", "Nothing to report" or "A curation report with comments and suggestions/instructions will follow in another email" that suits your situation. +The :ref:`send-feedback-admin` Admin only API call may be useful as a way to move the conversation to email. However, note that these emails go to contacts (versus authors) and there is no database record of the email contents. (:ref:`dataverse.mail.cc-support-on-contact-email` will send a copy of these emails to the support email address which would provide a record.) The :ref:`send-feedback` API call may be useful as a way to move the conversation to email. However, note that these emails go to contacts (versus authors) and there is no database record of the email contents. (:ref:`dataverse.mail.cc-support-on-contact-email` will send a copy of these emails to the support email address which would provide a record.) Link a Dataset @@ -2724,7 +3057,7 @@ The fully expanded example above (without environment variables) looks like this .. code-block:: bash curl "https://demo.dataverse.org/api/datasets/:persistentId/makeDataCount/citations?persistentId=10.5072/FK2/J8SJZB" - + Delete Unpublished Dataset ~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -2942,15 +3275,23 @@ Retrieve Signposting Information Dataverse supports :ref:`discovery-sign-posting` as a discovery mechanism. Signposting involves the addition of a `Link `__ HTTP header providing summary information on GET and HEAD requests to retrieve the dataset page and a separate /linkset API call to retrieve additional information. -Here is an example of a "Link" header: +Signposting Link HTTP Header +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -``Link: ;rel="cite-as", ;rel="describedby";type="application/vnd.citationstyles.csl+json",;rel="describedby";type="application/ld+json", ;rel="type",;rel="type", ;rel="license", ; rel="linkset";type="application/linkset+json"`` +Here is an example of a HTTP "Link" header from a GET or HEAD request for a dataset landing page: -The URL for linkset information is discoverable under the ``rel="linkset";type="application/linkset+json`` entry in the "Link" header, such as in the example above. +``Link: ;rel="cite-as", ;rel="describedby";type="application/vnd.citationstyles.csl+json",;rel="describedby";type="application/json",;rel="describedby";type="application/xml",;rel="describedby";type="application/xml",;rel="describedby";type="application/xml",;rel="describedby";type="application/ld+json",;rel="describedby";type="application/xml",;rel="describedby";type="application/xml",;rel="describedby";type="text/html",;rel="describedby";type="application/json",;rel="describedby";type="application/xml", ;rel="type",;rel="type", ;rel="license", ; rel="linkset";type="application/linkset+json"`` + +The URL for linkset information (described below) is discoverable under the ``rel="linkset";type="application/linkset+json`` entry in the "Link" header, such as in the example above. + +Signposting Linkset API Endpoint +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ The reponse includes a JSON object conforming to the `Signposting `__ specification. As part of this conformance, unlike most Dataverse API responses, the output is not wrapped in a ``{"status":"OK","data":{`` object. Signposting is not supported for draft dataset versions. +Like :ref:`get-export-formats`, this API can be used to get URLs to dataset metadata export formats, but with URLs for the dataset in question. + .. code-block:: bash export SERVER_URL=https://demo.dataverse.org @@ -2983,6 +3324,8 @@ Usage example: Get Citation ~~~~~~~~~~~~ +This API call returns the dataset citation as seen on the dataset page, wrapped as a JSON object, with the value in the "data" sub-object's "message" key. + .. code-block:: bash export SERVER_URL=https://demo.dataverse.org @@ -3000,6 +3343,35 @@ Usage example: .. code-block:: bash curl -H "Accept:application/json" "$SERVER_URL/api/datasets/:persistentId/versions/$VERSION/{version}/citation?persistentId=$PERSISTENT_IDENTIFIER&includeDeaccessioned=true" + +Get Citation In Other Formats +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Dataverse can also generate dataset citations in "EndNote", "RIS", "BibTeX", and "CSLJson" formats. +Unlike the call above, which wraps the result in JSON, this API call sends the raw format with the appropriate content-type (EndNote is XML, RIS and BibTeX are plain text, and CSLJson is JSON). ("Internal" is also a valid value, returning the same content as the above call as HTML). +This API call adds a format parameter in the API call which can be any of the values listed above. + +Usage example: + +.. code-block:: bash + + export SERVER_URL=https://demo.dataverse.org + export PERSISTENT_IDENTIFIER=doi:10.5072/FK2/YD5QDG + export VERSION=1.0 + export FORMAT=EndNote + + curl "$SERVER_URL/api/datasets/:persistentId/versions/$VERSION/{version}/citation/$FORMAT?persistentId=$PERSISTENT_IDENTIFIER" + +By default, deaccessioned dataset versions are not included in the search when applying the :latest or :latest-published identifiers. Additionally, when filtering by a specific version tag, you will get a "not found" error if the version is deaccessioned and you do not enable the ``includeDeaccessioned`` option described below. + +If you want to include deaccessioned dataset versions, you must set ``includeDeaccessioned`` query parameter to ``true``. + +Usage example: + +.. code-block:: bash + + curl "$SERVER_URL/api/datasets/:persistentId/versions/$VERSION/{version}/citation/$FORMAT?persistentId=$PERSISTENT_IDENTIFIER&includeDeaccessioned=true" + Get Citation by Preview URL Token ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -3249,6 +3621,130 @@ The fully expanded example above (without environment variables) looks like this curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X DELETE "https://demo.dataverse.org/api/datasets/datasetTypes/3" +.. _api-link-dataset-type: + +Link Dataset Type with Metadata Blocks +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Linking a dataset type with one or more metadata blocks results in additional fields from those blocks appearing in the output from the :ref:`list-metadata-blocks-for-a-collection` API endpoint. The new frontend for Dataverse (https://github.com/IQSS/dataverse-frontend) uses the JSON output from this API endpoint to construct the page that users see when creating or editing a dataset. Once the frontend has been updated to pass in the dataset type (https://github.com/IQSS/dataverse-client-javascript/issues/210), specifying a dataset type in this way can be an alternative way to display additional metadata fields than the traditional method, which is to enable a metadata block at the collection level (see :ref:`define-metadata-blocks-for-a-dataverse-collection`). + +For example, a superuser could create a type called "software" and link it to the "CodeMeta" metadata block (this example is below). Then, once the new frontend allows it, the user can specify that they want to create a dataset of type software and see the additional metadata fields from the CodeMeta block when creating or editing their dataset. + +This API endpoint is for superusers only. + +.. code-block:: bash + + export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx + export SERVER_URL=https://demo.dataverse.org + export TYPE=software + export JSON='["codeMeta20"]' + + curl -H "X-Dataverse-key:$API_TOKEN" -H "Content-Type: application/json" "$SERVER_URL/api/datasets/datasetTypes/$TYPE" -X PUT -d $JSON + +The fully expanded example above (without environment variables) looks like this: + +.. code-block:: bash + + curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -H "Content-Type: application/json" "https://demo.dataverse.org/api/datasets/datasetTypes/software" -X PUT -d '["codeMeta20"]' + +To update the blocks that are linked, send an array with those blocks. + +To remove all links to blocks, send an empty array. + +.. _api-dataset-version-note: + +Dataset Version Notes +~~~~~~~~~~~~~~~~~~~~~ + +Intended as :ref:`provenance` information about why the version was created/how it differs from the prior version + +Depositors who can edit the dataset and curators can add a version note for the draft version. Superusers can add/delete version notes for any version. + +Version notes can be retrieved via the following, with authorization required to see a note on the :draft version + +.. code-block:: bash + + export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx + export SERVER_URL=https://demo.dataverse.org + export ID=3 + export VERSION=:draft + + curl -H "X-Dataverse-key:$API_TOKEN" "$SERVER_URL/api/datasets/$ID/versions/$VERSION/versionNote" + +The fully expanded example above (without environment variables) looks like this: + +.. code-block:: bash + + curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" "https://demo.dataverse.org/api/datasets/3/versions/:draft/versionNote" + +Notes can be set with: + +.. code-block:: bash + + export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx + export SERVER_URL=https://demo.dataverse.org + export ID=3 + export VERSION=:draft + export NOTE=Files updated to correct typos + + curl -H "X-Dataverse-key:$API_TOKEN" -X PUT -d "$NOTE" "$SERVER_URL/api/datasets/$ID/versions/$VERSION/versionNote" + +The fully expanded example above (without environment variables) looks like this: + +.. code-block:: bash + + curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X PUT -d "Files updated to correct typos" "https://demo.dataverse.org/api/datasets/3/versions/:draft/versionNote" + +And deleted via: + +.. code-block:: bash + + export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx + export SERVER_URL=https://demo.dataverse.org + export ID=3 + export VERSION=2.0 + + curl -H "X-Dataverse-key:$API_TOKEN" -X DELETE "$SERVER_URL/api/datasets/$ID/versions/$VERSION/versionNote" + +The fully expanded example above (without environment variables) looks like this: + +.. code-block:: bash + + curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X DELETE "https://demo.dataverse.org/api/datasets/3/versions/2.0/versionNote" + +Delete Files from a Dataset +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Delete files from a dataset. This API call allows you to delete multiple files from a dataset in a single operation. + +.. code-block:: bash + + export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx + export SERVER_URL=https://demo.dataverse.org + export PERSISTENT_IDENTIFIER=doi:10.5072/FK2ABCDEF + + curl -H "X-Dataverse-key:$API_TOKEN" -X PUT "$SERVER_URL/api/datasets/:persistentId/deleteFiles?persistentId=$PERSISTENT_IDENTIFIER" \ + -H "Content-Type: application/json" \ + -d '{"fileIds": [1, 2, 3]}' + +The fully expanded example above (without environment variables) looks like this: + +.. code-block:: bash + + curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X PUT "https://demo.dataverse.org/api/datasets/:persistentId/deleteFiles?persistentId=doi:10.5072/FK2ABCDEF" \ + -H "Content-Type: application/json" \ + -d '{"fileIds": [1, 2, 3]}' + +The ``fileIds`` in the JSON payload should be an array of file IDs that you want to delete from the dataset. + +You must have the appropriate permissions to delete files from the dataset. + +Upon success, the API will return a JSON response with a success message and the number of files deleted. + +The API call will report a 400 (BAD REQUEST) error if any of the files specified do not exist or are not in the latest version of the specified dataset. +The ``fileIds`` in the JSON payload should be an array of file IDs that you want to delete from the dataset. + + Files ----- @@ -3446,7 +3942,29 @@ The fully expanded example above (without environment variables) looks like this curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" "https://demo.dataverse.org/api/files/:persistentId/versions/:draft?persistentId=doi:10.5072/FK2/J8SJZB&returnOwners=true" +Get JSON Representation of a file's versions +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Gets a list of versions of a data file showing any changes that affected the file with each version. +The fileIdOrPersistentId can be either "persistentId": "doi:10.5072/FK2/ADMYJF" or "datafileId": 19. + +Usage example: + +.. code-block:: bash + + export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx + export SERVER_URL=https://demo.dataverse.org + export ID=1234 + export PERSISTENT_ID=doi:10.5072/FK2/J8SJZB + + curl -H "X-Dataverse-key: $API_TOKEN" -X GET "$SERVER_URL/api/files/$ID/versionDifferences" + curl -H "X-Dataverse-key: $API_TOKEN" -X GET "$SERVER_URL/api/files/:persistentId/versionDifferences?persistentId=$PERSISTENT_ID" + +The fully expanded example above (without environment variables) looks like this: + +.. code-block:: bash + curl -X GET "https://demo.dataverse.org/api/files/1234/versionDifferences" + curl -X GET "https://demo.dataverse.org/api/files/:persistentId/versionDifferences?persistentId=doi:10.5072/FK2/J8SJZB" Adding Files ~~~~~~~~~~~~ @@ -3648,7 +4166,7 @@ The fully expanded example above (without environment variables) looks like this Currently the following methods are used to detect file types: - The file type detected by the browser (or sent via API). -- Custom code that reads the first few bytes. As explained at :ref:`s3-direct-upload-features-disabled`, this method of file type detection is not utilized during direct upload to S3, since by nature of direct upload Dataverse never sees the contents of the file. However, this code is utilized when the "redetect" API is used. +- Custom code that reads the first few bytes. As explained at :ref:`s3-direct-upload-features-disabled`, most of these methods are not utilized during direct upload to S3, since by nature of direct upload Dataverse never sees the contents of the file. However, this code is utilized when the "redetect" API is used. - JHOVE: https://jhove.openpreservation.org . Note that the same applies about direct upload to S3 and the "redetect" API. - The file extension (e.g. ".ipybn") is used, defined in a file called ``MimeTypeDetectionByFileExtension.properties``. - The file name (e.g. "Dockerfile") is used, defined in a file called ``MimeTypeDetectionByFileName.properties``. @@ -4549,12 +5067,12 @@ The JSON representation of a role (``roles.json``) looks like this:: { "alias": "sys1", - "name": โ€œRestricted System Roleโ€, - "description": โ€œA person who may only add datasets.โ€, + "name": "Restricted System Role", + "description": "A person who may only add datasets.", "permissions": [ "AddDataset" ] - } + } .. note:: alias is constrained to a length of 16 characters @@ -4563,17 +5081,49 @@ Create Role Roles can be created globally (:ref:`create-global-role`) or for individual Dataverse collections (:ref:`create-role-in-collection`). +.. _show-role: + Show Role ~~~~~~~~~ -Shows the role with ``id``:: +You must have ``ManageDataversePermissions`` to be able to show a role that was created using :ref:`create-role-in-collection`. Global roles (:ref:`create-global-role`) can only be shown with a superuser API token. + +An example using a role alias: + +.. code-block:: bash + + export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx + export SERVER_URL=https://demo.dataverse.org + export ALIAS=sys1 + + curl -H "X-Dataverse-key:$API_TOKEN" "$SERVER_URL/api/roles/:alias?alias=$ALIAS" + +The fully expanded example above (without environment variables) looks like this: + +.. code-block:: bash + + curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" "https://demo.dataverse.org/api/roles/:alias?alias=sys1" - GET http://$SERVER/api/roles/$id +An example using a role id: + +.. code-block:: bash + + export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx + export SERVER_URL=https://demo.dataverse.org + export ID=11 + + curl -H "X-Dataverse-key:$API_TOKEN" "$SERVER_URL/api/roles/$ID" + +The fully expanded example above (without environment variables) looks like this: + +.. code-block:: bash + + curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" "https://demo.dataverse.org/api/roles/11" Delete Role ~~~~~~~~~~~ -A curl example using an ``ID`` +An example using a role id: .. code-block:: bash @@ -4589,13 +5139,13 @@ The fully expanded example above (without environment variables) looks like this curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X DELETE "https://demo.dataverse.org/api/roles/24" -A curl example using a Role alias ``ALIAS`` +An example using a role alias: .. code-block:: bash export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx export SERVER_URL=https://demo.dataverse.org - export ALIAS=roleAlias + export ALIAS=sys1 curl -H "X-Dataverse-key:$API_TOKEN" -X DELETE "$SERVER_URL/api/roles/:alias?alias=$ALIAS" @@ -4603,8 +5153,7 @@ The fully expanded example above (without environment variables) looks like this .. code-block:: bash - curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X DELETE "https://demo.dataverse.org/api/roles/:alias?alias=roleAlias" - + curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X DELETE "https://demo.dataverse.org/api/roles/:alias?alias=sys1" Explicit Groups --------------- @@ -4889,12 +5438,14 @@ The fully expanded example above (without environment variables) looks like this curl "https://demo.dataverse.org/api/info/settings/:MaxEmbargoDurationInMonths" -Get Export Formats -~~~~~~~~~~~~~~~~~~~~~~~~~~~ +.. _get-export-formats: -Get the available export formats, including custom formats. +Get Dataset Metadata Export Formats +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -The response contains an object with available format names as keys, and as values an object with the following properties: +Get the available dataset metadata export formats, including formats from external exporters (see :ref:`available-exporters`). + +The response contains a JSON object with the available format names as keys (these can be passed to :ref:`export-dataset-metadata-api`), and values as objects with the following properties: * ``displayName`` * ``mediaType`` @@ -5001,6 +5552,27 @@ The fully expanded example above (without environment variables) looks like this curl "https://demo.dataverse.org/api/datasetfields/facetables" +.. _setDisplayOnCreate: + +Set displayOnCreate for a Dataset Field +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Set displayOnCreate for a dataset field. See also :doc:`/admin/metadatacustomization` in the Admin Guide. + +.. code-block:: bash + + export SERVER_URL=http://localhost:8080 + export FIELD=subtitle + export BOOLEAN=true + + curl -X POST "$SERVER_URL/api/admin/datasetfield/setDisplayOnCreate?datasetFieldType=$FIELD&setDisplayOnCreate=$BOOLEAN" + +The fully expanded example above (without environment variables) looks like this: + +.. code-block:: bash + + curl -X POST "http://localhost:8080/api/admin/datasetfield/setDisplayOnCreate?datasetFieldType=studyAssayCellType&setDisplayOnCreate=true" + .. _Notifications: Notifications @@ -5128,7 +5700,7 @@ Create a Harvesting Set To create a harvesting set you must supply a JSON file that contains the following fields: -- Name: Alpha-numeric may also contain -, _, or %, but no spaces. Must also be unique in the installation. +- Name: Alpha-numeric may also contain -, _, or %, but no spaces. It must also be unique in the installation. - Definition: A search query to select the datasets to be harvested. For example, a query containing authorName:YYY would include all datasets where โ€˜YYYโ€™ is the authorName. - Description: Text that describes the harvesting set. The description appears in the Manage Harvesting Sets dashboard and in API responses. This field is optional. @@ -5224,20 +5796,43 @@ The following API can be used to create and manage "Harvesting Clients". A Harve List All Configured Harvesting Clients ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Shows all the Harvesting Clients configured:: +Shows all the harvesting clients configured. - GET http://$SERVER/api/harvest/clients/ +.. note:: See :ref:`curl-examples-and-environment-variables` if you are unfamiliar with the use of export below. + +.. code-block:: bash + + export SERVER_URL=https://demo.dataverse.org + + curl "$SERVER_URL/api/harvest/clients" + +The fully expanded example above (without the environment variables) looks like this: + +.. code-block:: bash + + curl "https://demo.dataverse.org/api/harvest/clients" Show a Specific Harvesting Client ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Shows a Harvesting Client with a defined nickname:: +Shows a harvesting client by nickname. + +.. code-block:: bash + + export SERVER_URL=https://demo.dataverse.org + export NICKNAME=myclient + + curl "$SERVER_URL/api/harvest/clients/$NICKNAME" - GET http://$SERVER/api/harvest/clients/$nickname +The fully expanded example above (without the environment variables) looks like this: .. code-block:: bash - curl "http://localhost:8080/api/harvest/clients/myclient" + curl "https://demo.dataverse.org/api/harvest/clients/myclient" + +The output will look something like the following. + +.. code-block:: bash { "status":"OK", @@ -5253,8 +5848,10 @@ Shows a Harvesting Client with a defined nickname:: "type": "oai", "dataverseAlias": "fooData", "nickName": "myClient", + "sourceName": "", "set": "fooSet", - "useOaiIdentifiersAsPids": false + "useOaiIdentifiersAsPids": false, + "useListRecords": false, "schedule": "none", "status": "inActive", "lastHarvest": "Thu Oct 13 14:48:57 EDT 2022", @@ -5266,64 +5863,60 @@ Shows a Harvesting Client with a defined nickname:: } +.. _create-a-harvesting-client: + Create a Harvesting Client ~~~~~~~~~~~~~~~~~~~~~~~~~~ - -To create a new harvesting client:: - - POST http://$SERVER/api/harvest/clients/$nickname - -``nickName`` is the name identifying the new client. It should be alpha-numeric and may also contain -, _, or %, but no spaces. Must also be unique in the installation. -You must supply a JSON file that describes the configuration, similarly to the output of the GET API above. The following fields are mandatory: +To create a harvesting client you must supply a JSON file that describes the configuration, similarly to the output of the GET API above. The following fields are mandatory: -- dataverseAlias: The alias of an existing collection where harvested datasets will be deposited -- harvestUrl: The URL of the remote OAI archive -- archiveUrl: The URL of the remote archive that will be used in the redirect links pointing back to the archival locations of the harvested records. It may or may not be on the same server as the harvestUrl above. If this OAI archive is another Dataverse installation, it will be the same URL as harvestUrl minus the "/oai". For example: https://demo.dataverse.org/ vs. https://demo.dataverse.org/oai -- metadataFormat: A supported metadata format. As of writing this the supported formats are "oai_dc", "oai_ddi" and "dataverse_json". +- ``dataverseAlias``: The alias of an existing collection where harvested datasets will be deposited +- ``harvestUrl``: The URL of the remote OAI archive +- ``archiveUrl``: The URL of the remote archive that will be used in the redirect links pointing back to the archival locations of the harvested records. It may or may not be on the same server as the harvestUrl above. If this OAI archive is another Dataverse installation, it will be the same URL as harvestUrl minus the "/oai". For example: https://demo.dataverse.org/ vs. https://demo.dataverse.org/oai +- ``metadataFormat``: A supported metadata format. As of writing this the supported formats are "oai_dc", "oai_ddi" and "dataverse_json". The following optional fields are supported: -- archiveDescription: What the name suggests. If not supplied, will default to "This Dataset is harvested from our partners. Clicking the link will take you directly to the archival source of the data." -- set: The OAI set on the remote server. If not supplied, will default to none, i.e., "harvest everything". -- style: Defaults to "default" - a generic OAI archive. (Make sure to use "dataverse" when configuring harvesting from another Dataverse installation). -- customHeaders: This can be used to configure this client with a specific HTTP header that will be added to every OAI request. This is to accommodate a use case where the remote server requires this header to supply some form of a token in order to offer some content not available to other clients. See the example below. Multiple headers can be supplied separated by `\\n` - actual "backslash" and "n" characters, not a single "new line" character. -- allowHarvestingMissingCVV: Flag to allow datasets to be harvested with Controlled Vocabulary Values that existed in the originating Dataverse Project but are not in the harvesting Dataverse Project. (Default is false). Currently only settable using API. -- useOaiIdentifiersAsPids: Defaults to false; if set to true, the harvester will attempt to use the identifier from the OAI-PMH record header as the **first choice** for the persistent id of the harvested dataset. When set to false, Dataverse will still attempt to use this identifier, but only if none of the `` entries in the OAI_DC record contain a valid persistent id (this is new as of v6.5). - -Generally, the API will accept the output of the GET version of the API for an existing client as valid input, but some fields will be ignored. For example, as of writing this there is no way to configure a harvesting schedule via this API. +- ``sourceName``: When ``index-harvested-metadata-source`` is enabled (see :ref:`feature-flags`), sourceName will override the nickname in the Metadata Source facet. It can be used to group the content from many harvesting clients under the same name. +- ``archiveDescription``: What the name suggests. If not supplied, will default to "This Dataset is harvested from our partners. Clicking the link will take you directly to the archival source of the data." +- ``set``: The OAI set on the remote server. If not supplied, will default to none, i.e., "harvest everything". (Note: see the note below on using sets when harvesting from DataCite; this is new as of v6.6). +- ``style``: Defaults to "default" - a generic OAI archive. (Make sure to use "dataverse" when configuring harvesting from another Dataverse installation). +- ``schedule``: Defaults to "none" (not scheduled). Two formats are supported, for weekly- and daily-scheduled harvests; examples: ``Weekly, Sat 5 AM``; ``Daily, 11 PM``. Note that if a schedule definition is not formatted exactly as described here, it will be ignored silently and the client will be left unscheduled. +- ``customHeaders``: This can be used to configure this client with a specific HTTP header that will be added to every OAI request. This is to accommodate a use case where the remote server requires this header to supply some form of a token in order to offer some content not available to other clients. See the example below. Multiple headers can be supplied separated by `\\n` - actual "backslash" and "n" characters, not a single "new line" character. +- ``allowHarvestingMissingCVV``: Flag to allow datasets to be harvested with Controlled Vocabulary Values that existed in the originating Dataverse Project but are not in the harvesting Dataverse Project. (Default is false). Currently only settable using API. +- ``useOaiIdentifiersAsPids``: Defaults to false; if set to true, the harvester will attempt to use the identifier from the OAI-PMH record header as the **first choice** for the persistent id of the harvested dataset. When set to false, Dataverse will still attempt to use this identifier, but only if none of the ```` entries in the OAI_DC record contain a valid persistent id (this is new as of v6.5). +- ``useListRecords``: Defaults to false; if set to true, the harvester will attempt to retrieve multiple records in a single pass using the OAI-PMH verb ListRecords. By default, our harvester relies on the combination of ListIdentifiers followed by multiple GetRecord calls for each individual record. Note that this option is required when configuring harvesting from DataCite. (this is new as of v6.6). + +Generally, the API will accept the output of the GET version of the API for an existing client as valid input, but some fields will be ignored. -An example JSON file would look like this:: +You can download this :download:`harvesting-client.json <../_static/api/harvesting-client.json>` file to use as a starting point. - { - "nickName": "zenodo", - "dataverseAlias": "zenodoHarvested", - "harvestUrl": "https://zenodo.org/oai2d", - "archiveUrl": "https://zenodo.org", - "archiveDescription": "Moissonnรฉ depuis la collection LMOPS de l'entrepรดt Zenodo. En cliquant sur ce jeu de donnรฉes, vous serez redirigรฉ vers Zenodo.", - "metadataFormat": "oai_dc", - "customHeaders": "x-oai-api-key: xxxyyyzzz", - "set": "user-lmops", - "allowHarvestingMissingCVV":true - } +.. literalinclude:: ../_static/api/harvesting-client.json Something important to keep in mind about this API is that, unlike the harvesting clients GUI, it will create a client with the values supplied without making any attempts to validate them in real time. In other words, for the `harvestUrl` it will accept anything that looks like a well-formed url, without making any OAI calls to verify that the name of the set and/or the metadata format entered are supported by it. This is by design, to give an admin an option to still be able to create a client, in a rare case when it cannot be done via the GUI because of some real time failures in an exchange with an otherwise valid OAI server. This however puts the responsibility on the admin to supply the values already confirmed to be valid. - .. note:: See :ref:`curl-examples-and-environment-variables` if you are unfamiliar with the use of export below. + +``nickName`` in the JSON file and ``$NICKNAME`` in the URL path below is the name identifying the new client. It should be alpha-numeric and may also contain -, _, or %, but no spaces. It must be unique in the installation. + .. code-block:: bash export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx export SERVER_URL=http://localhost:8080 + export NICKNAME=zenodo - curl -H "X-Dataverse-key:$API_TOKEN" -X POST -H "Content-Type: application/json" "$SERVER_URL/api/harvest/clients/zenodo" --upload-file client.json + curl -H "X-Dataverse-key:$API_TOKEN" -X POST -H "Content-Type: application/json" "$SERVER_URL/api/harvest/clients/$NICKNAME" --upload-file harvesting-client.json The fully expanded example above (without the environment variables) looks like this: .. code-block:: bash - curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X POST -H "Content-Type: application/json" "http://localhost:8080/api/harvest/clients/zenodo" --upload-file "client.json" + curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X POST -H "Content-Type: application/json" "http://localhost:8080/api/harvest/clients/zenodo" --upload-file "harvesting-client.json" + +The output will look something like the following. + +.. code-block:: bash { "status": "OK", @@ -5357,14 +5950,85 @@ Similar to the API above, using the same JSON format, but run on an existing cli Delete a Harvesting Client ~~~~~~~~~~~~~~~~~~~~~~~~~~ -Self-explanatory: +.. code-block:: bash + + export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx + export SERVER_URL=http://localhost:8080 + export NICKNAME=zenodo + + curl -H "X-Dataverse-key:$API_TOKEN" -X DELETE "$SERVER_URL/api/harvest/clients/$NICKNAME" + +The fully expanded example above (without the environment variables) looks like this: .. code-block:: bash - curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X DELETE "http://localhost:8080/api/harvest/clients/$nickName" + curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X DELETE "http://localhost:8080/api/harvest/clients/zenodo" Only users with superuser permissions may delete harvesting clients. +Harvesting from DataCite +~~~~~~~~~~~~~~~~~~~~~~~~ + +The following 2 options are **required** when harvesting from DataCite (https://oai.datacite.org/oai): + +.. code-block:: bash + + "useOaiIdentifiersAsPids": true, + "useListRecords": true, + +There are two ways the ``set`` parameter can be used when harvesting from DataCite: + +- DataCite maintains pre-configured OAI sets for every subscribing institution that registers DOIs with them. This can be used to harvest the entire set of metadata registered by this organization or school, etc. (this is identical to how the set parameter is used with any other standard OAI archive); +- As a unique, proprietary DataCite feature, it can be used to harvest virtually any arbitrary subset of records (potentially spanning different institutions and authorities, etc.). Any query that the DataCite search API understands can be used as an OAI set name (!). For example, the following search query finds one specific dataset: + +.. code-block:: bash + + https://api.datacite.org/dois?query=doi:10.7910/DVN/TJCLKP + +you can now create a single-record OAI set by using its base64-encoded form as the set name: + +.. code-block:: bash + + echo "doi:10.7910/DVN/TJCLKP" | base64 + ZG9pOjEwLjc5MTAvRFZOL1RKQ0xLUAo= + +use the encoded string above prefixed by the ``~`` character in your harvesting client configuration: + +.. code-block:: bash + + "set": "~ZG9pOjEwLjc5MTAvRFZOL1RKQ0xLUAo=" + +The following configuration will create a client that will harvest the IQSS dataset specified above on a weekly schedule: + +.. code-block:: bash + + { + "useOaiIdentifiersAsPids": true, + "useListRecords": true, + "set": "~ZG9pOjEwLjc5MTAvRFZOL1RKQ0xLUAo=", + "nickName": "iqssTJCLKP", + "dataverseAlias": "harvestedCollection", + "type": "oai", + "style": "default", + "harvestUrl": "https://oai.datacite.org/oai", + "archiveUrl": "https://oai.datacite.org", + "archiveDescription": "The metadata for this IQSS Dataset was harvested from DataCite. Clicking the dataset link will take you directly to the original archival location, as registered with DataCite.", + "schedule": "Weekly, Tue 4 AM", + "metadataFormat": "oai_dc" + } + +The queries can be as complex and/or long as necessary, with sub-queries combined via logical ANDs and ORs. Please keep in mind that white spaces must be encoded as ``%20``. For example, the following query: + +.. code-block:: bash + + prefix:10.17603 AND (types.resourceType:Report* OR types.resourceType:Mission*) + +must be encoded as follows: + +.. code-block:: bash + + echo "prefix:10.17603%20AND%20(types.resourceType:Report*%20OR%20types.resourceType:Mission*)" | base64 + cHJlZml4OjEwLjE3NjAzJTIwQU5EJTIwKHR5cGVzLnJlc291cmNlVHlwZTpSZXBvcnQqJTIwT1IlMjB0eXBlcy5yZXNvdXJjZVR5cGU6TWlzc2lvbiopCg== .. _pids-api: @@ -5712,22 +6376,43 @@ Creates a global role in the Dataverse installation. The data POSTed are assumed .. code-block:: bash export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx - export SERVER_URL=https://demo.dataverse.org - export ID=root + export SERVER_URL=http://localhost:8080 - curl -H "X-Dataverse-key:$API_TOKEN" -X POST "$SERVER_URL/api/admin/roles" --upload-file roles.json + curl -H "Content-Type: application/json" -H "X-Dataverse-key:$API_TOKEN" -X POST "$SERVER_URL/api/admin/roles" --upload-file roles.json + +``roles.json`` see :ref:`json-representation-of-a-role` + +Update Global Role +~~~~~~~~~~~~~~~~~~ + +Update a global role in the Dataverse installation. The PUTed data is assumed to be a complete JSON role as it will overwrite the existing role. :: + + PUT http://$SERVER/api/admin/roles/$ID + +A curl example using an ``ID`` + +.. code-block:: bash + + export SERVER_URL=http://localhost:8080 + export ID=24 + + curl -H "Content-Type: application/json" -X PUT "$SERVER_URL/api/admin/roles/$ID" --upload-file roles.json ``roles.json`` see :ref:`json-representation-of-a-role` Delete Global Role ~~~~~~~~~~~~~~~~~~ +Deletes an ``DataverseRole`` whose ``id`` is passed. :: + + DELETE http://$SERVER/api/admin/roles/$ID + A curl example using an ``ID`` .. code-block:: bash export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx - export SERVER_URL=https://demo.dataverse.org + export SERVER_URL=http://localhost:8080 export ID=24 curl -H "X-Dataverse-key:$API_TOKEN" -X DELETE "$SERVER_URL/api/admin/roles/$ID" @@ -6131,6 +6816,27 @@ Example: List permissions a user (based on API Token used) has on a dataset whos curl -H "X-Dataverse-key:$API_TOKEN" "$SERVER_URL/api/admin/permissions/:persistentId?persistentId=$PERSISTENT_IDENTIFIER" +List Dataverse collections a user can act on based on their permissions +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +List Dataverse collections a user can act on based on a particular permission :: + + GET http://$SERVER/api/users/$identifier/allowedCollections/$permission + +.. note:: This API can only be called by an Administrator or by a User requesting their own list of accessible collections. + +The ``$identifier`` is the username of the requested user. +The ``$permission`` is the permission (tied to the roles) that gives the user access to the collection. +Passing ``$permission`` as 'any' will return the collection as long as the user has any access/permission on the collection + +.. code-block:: bash + + export SERVER_URL=https://demo.dataverse.org + export $USERNAME=jsmith + export PERMISSION=PublishDataverse + + curl -H "X-Dataverse-key:$API_TOKEN" "$SERVER_URL/api/users/$USERNAME/allowedCollections/$PERMISSION" + Show Role Assignee ~~~~~~~~~~~~~~~~~~ @@ -6466,6 +7172,10 @@ View the details of the standard license with the database ID specified in ``$ID Superusers can add a new license by posting a JSON file adapted from this example :download:`add-license.json <../_static/api/add-license.json>`. The ``name`` and ``uri`` of the new license must be unique. Sort order field is mandatory. If you are interested in adding a Creative Commons license, you are encouarged to use the JSON files under :ref:`adding-creative-commons-licenses`: +Licenses must have a "name" and "uri" and may have the following optional fields: "shortDescription", "iconUri", "rightsIdentifier", "rightsIdentifierScheme", "schemeUri", "languageCode", "active", "sortOrder". +The "name" and "uri" are used to display the license in the user interface, with "shortDescription" and "iconUri" being used to enhance the display if available. +The "rightsIdentifier", "rightsIdentifierScheme", and "schemeUri" should be added if the license is available from https://spdx.org . "languageCode" should be sent if the language is not in English ("en"). "active" is a boolean indicating whether the license should be shown to users as an option. "sortOrder" is a numeric value - licenses are shown in the relative numeric order of this value. + .. code-block:: bash export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx @@ -6557,10 +7267,10 @@ A curl example using allowing access to a dataset's metadata Please see :ref:`dataverse.api.signature-secret` for the configuration option to add a shared secret, enabling extra security. -.. _send-feedback: +.. _send-feedback-admin: -Send Feedback To Contact(s) -~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Send Feedback To Contact(s) Admin API +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ This API call allows sending an email to the contacts for a collection, dataset, or datafile or to the support email address when no object is specified. The call is protected by the normal /admin API protections (limited to localhost or requiring a separate key), but does not otherwise limit the sending of emails. @@ -6583,6 +7293,44 @@ A curl example using an ``ID`` Note that this call could be useful in coordinating with dataset authors (assuming they are also contacts) as an alternative/addition to the functionality provided by :ref:`return-a-dataset`. +.. _send-feedback: + +Send Feedback To Contact(s) +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +This API call allows sending an email to the contacts for a collection, dataset, or datafile or to the support email address when no object is specified. +The call is protected from embedded html in the body as well as the ability to configure body size limits and rate limiting to avoid the potential for spam. + +The call is a POST with a JSON object as input with four keys: +- "targetId" - the id of the collection, dataset, or datafile. Persistent ids and collection aliases are not supported. (Optional) +- "identifier" - the alias of a collection or the persistence id of a dataset or datafile. (Optional) +- "subject" - the email subject line. (Required) +- "body" - the email body to send (Required) +- "fromEmail" - the email to list in the reply-to field. (Dataverse always sends mail from the system email, but does it "on behalf of" and with a reply-to for the specified user. Authenticated users will have the 'fromEmail' filled in from their profile if this field is not specified) + +A curl example using an ``ID`` + +.. code-block:: bash + + export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx + export SERVER_URL=https://demo.dataverse.org + export JSON='{"targetId":24, "subject":"Data Question", "body":"Please help me understand your data. Thank you!"}' + + curl -X POST -H "X-Dataverse-key:$API_KEY" -H 'Content-Type:application/json' -d "$JSON" "$SERVER_URL/api/sendfeedback" + + +A curl example using a ``Dataverse Alias or Dataset/DataFile PersistentId`` + +.. code-block:: bash + + export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx + export SERVER_URL=https://demo.dataverse.org + export JSON='{"identifier":"root", "subject":"Data Question", "body":"Please help me understand your data. Thank you!"}' + + curl -X POST -H "X-Dataverse-key:$API_KEY" -H 'Content-Type:application/json' -d "$JSON" "$SERVER_URL/api/sendfeedback" + +Note that this call could be useful in coordinating with dataset authors (assuming they are also contacts) as an alternative/addition to the functionality provided by :ref:`return-a-dataset`. + .. _thumbnail_reset: Reset Thumbnail Failure Flags @@ -6616,6 +7364,8 @@ MyData The MyData API is used to get a list of just the datasets, dataverses or datafiles an authenticated user can edit. +The API excludes dataverses linked to an harvesting client. This results in `a known issue `_ where regular datasets in harvesting dataverses are missing from the results. + A curl example listing objects .. code-block:: bash diff --git a/doc/sphinx-guides/source/api/search.rst b/doc/sphinx-guides/source/api/search.rst index 7ca9a5abca6..9a211988979 100755 --- a/doc/sphinx-guides/source/api/search.rst +++ b/doc/sphinx-guides/source/api/search.rst @@ -21,9 +21,9 @@ Please note that in Dataverse Software 4.3 and older the "citation" field wrappe Parameters ---------- -=============== ======= =========== +================ ======= =========== Name Type Description -=============== ======= =========== +================ ======= =========== q string The search term or terms. Using "title:data" will search only the "title" field. "*" can be used as a wildcard either alone or adjacent to a term (i.e. "bird*"). For example, https://demo.dataverse.org/api/search?q=title:data . For a list of fields to search, please see https://github.com/IQSS/dataverse/issues/2558 (for now). type string Can be either "dataverse", "dataset", or "file". Multiple "type" parameters can be used to include multiple types (i.e. ``type=dataset&type=file``). If omitted, all types will be returned. For example, https://demo.dataverse.org/api/search?q=*&type=dataset subtree string The identifier of the Dataverse collection to which the search should be narrowed. The subtree of this Dataverse collection and all its children will be searched. Multiple "subtree" parameters can be used to include multiple Dataverse collections. For example, https://demo.dataverse.org/api/search?q=data&subtree=birds&subtree=cats . @@ -38,7 +38,8 @@ show_entity_ids boolean Whether or not to show the database IDs of the search geo_point string Latitude and longitude in the form ``geo_point=42.3,-71.1``. You must supply ``geo_radius`` as well. See also :ref:`geospatial-search`. geo_radius string Radial distance in kilometers from ``geo_point`` (which must be supplied as well) such as ``geo_radius=1.5``. metadata_fields string Includes the requested fields for each dataset in the response. Multiple "metadata_fields" parameters can be used to include several fields. The value must be in the form "{metadata_block_name}:{field_name}" to include a specific field from a metadata block (see :ref:`example `) or "{metadata_field_set_name}:\*" to include all the fields for a metadata block (see :ref:`example `). "{field_name}" cannot be a subfield of a compound field. If "{field_name}" is a compound field, all subfields are included. -=============== ======= =========== +show_type_counts boolean Whether or not to include total_count_per_object_type for types: Dataverse, Dataset, and Files. +================ ======= =========== Basic Search Example -------------------- @@ -701,7 +702,11 @@ The above example ``metadata_fields=citation:dsDescription&metadata_fields=citat "published_at": "2021-03-16T08:11:54Z" } ], - "count_in_response": 4 + "count_in_response": 4, + "total_count_per_object_type": { + "Datasets": 2, + "Dataverses": 2 + } } } diff --git a/doc/sphinx-guides/source/container/configbaker-image.rst b/doc/sphinx-guides/source/container/configbaker-image.rst index d098bd46436..09e431eb547 100644 --- a/doc/sphinx-guides/source/container/configbaker-image.rst +++ b/doc/sphinx-guides/source/container/configbaker-image.rst @@ -54,7 +54,7 @@ Scripts - Default script when running container without parameters. Lists available scripts and details about them. * - ``update-fields.sh`` - Update a Solr ``schema.xml`` with a given list of metadata fields. See ``update-fields.sh -h`` for usage details - and :ref:`update-solr-schema` for an example use case. + and example use cases at :ref:`update-solr-schema` and :ref:`update-solr-schema-dev`. Solr Template ^^^^^^^^^^^^^ diff --git a/doc/sphinx-guides/source/container/index.rst b/doc/sphinx-guides/source/container/index.rst index abf871dd340..38641cce642 100644 --- a/doc/sphinx-guides/source/container/index.rst +++ b/doc/sphinx-guides/source/container/index.rst @@ -4,6 +4,7 @@ Container Guide **Contents:** .. toctree:: + :maxdepth: 2 intro running/index diff --git a/doc/sphinx-guides/source/container/running/demo.rst b/doc/sphinx-guides/source/container/running/demo.rst index b1945070714..3cb0274c936 100644 --- a/doc/sphinx-guides/source/container/running/demo.rst +++ b/doc/sphinx-guides/source/container/running/demo.rst @@ -29,6 +29,8 @@ To stop the containers hit ``Ctrl-c`` (hold down the ``Ctrl`` key and then hit t To start the containers, run ``docker compose up``. +.. _starting-over: + Deleting Data and Starting Over ------------------------------- @@ -46,6 +48,8 @@ Starting Fresh For this exercise, please start fresh by stopping all containers and removing the ``data`` directory. +.. _demo-persona: + Creating and Running a Demo Persona +++++++++++++++++++++++++++++++++++ @@ -137,6 +141,29 @@ In the example below of configuring :ref:`:FooterCopyright` we use the default u One you make this change it should be visible in the copyright in the bottom left of every page. +Root Collection Customization (Alias, Name, etc.) ++++++++++++++++++++++++++++++++++++++++++++++++++ + +Before running ``docker compose up`` for the first time, you can customize the root collection by placing a JSON file in the right place. + +First, in the "demo" directory you created (see :ref:`demo-persona`), create a subdirectory called "config": + +``mkdir demo/config`` + +Next, download :download:`dataverse-complete.json <../../_static/api/dataverse-complete.json>` and put it in the "config" directory you just created. The contents of your "demo" directory should look something like this: + +.. code-block:: bash + + % find demo + demo + demo/config + demo/config/dataverse-complete.json + demo/init.sh + +Edit ``dataverse-complete.json`` to have the values you want. You'll want to refer to :ref:`update-dataverse-api` in the API Guide to understand the format. In that documentation you can find optional parameters as well. + +To test your JSON file, run ``docker compose up``. Again, this only works when you are running ``docker compose up`` for the first time. (You can always start over. See :ref:`starting-over`.) + Multiple Languages ++++++++++++++++++ @@ -160,6 +187,11 @@ Next, set up the UI toggle between English and French, again using the unblock k Stop and start the Dataverse container in order for the language toggle to work. +PID Providers ++++++++++++++ + +Dataverse supports multiple Persistent ID (PID) providers. The ``compose.yml`` file uses the Permalink PID provider. Follow :ref:`pids-configuration` to reconfigure as needed. + Next Steps ---------- diff --git a/doc/sphinx-guides/source/contributor/index.md b/doc/sphinx-guides/source/contributor/index.md index 1017f15f0ed..f7979b1dd0c 100644 --- a/doc/sphinx-guides/source/contributor/index.md +++ b/doc/sphinx-guides/source/contributor/index.md @@ -4,7 +4,7 @@ Thank you for your interest in contributing to Dataverse! We are open to contri ```{contents} Contents: :local: -:depth: 3 +:depth: 2 ``` ## Ideas and Feature Requests diff --git a/doc/sphinx-guides/source/developers/big-data-support.rst b/doc/sphinx-guides/source/developers/big-data-support.rst index f3d98fae0bf..75a50e2513d 100644 --- a/doc/sphinx-guides/source/developers/big-data-support.rst +++ b/doc/sphinx-guides/source/developers/big-data-support.rst @@ -44,7 +44,7 @@ Features that are Disabled if S3 Direct Upload is Enabled The following features are disabled when S3 direct upload is enabled. - Unzipping of zip files. (See :ref:`compressed-files`.) -- Detection of file type based on JHOVE and custom code that reads the first few bytes. (See :ref:`redetect-file-type`.) +- Detection of file type based on JHOVE and custom code that reads the first few bytes except for the refinement of Stata file types to include the version. (See :ref:`redetect-file-type`.) - Extraction of metadata from FITS files. (See :ref:`fits`.) - Creation of NcML auxiliary files (See :ref:`netcdf-and-hdf5`.) - Extraction of a geospatial bounding box from NetCDF and HDF5 files (see :ref:`netcdf-and-hdf5`) unless :ref:`dataverse.netcdf.geo-extract-s3-direct-upload` is set to true. diff --git a/doc/sphinx-guides/source/developers/classic-dev-env.rst b/doc/sphinx-guides/source/developers/classic-dev-env.rst index d305019004e..2e32b8d4bfb 100755 --- a/doc/sphinx-guides/source/developers/classic-dev-env.rst +++ b/doc/sphinx-guides/source/developers/classic-dev-env.rst @@ -93,15 +93,15 @@ On Linux, install ``jq`` from your package manager or download a binary from htt Install Payara ~~~~~~~~~~~~~~ -Payara 6.2024.6 or higher is required. +Payara 6.2025.2 or higher is required. To install Payara, run the following commands: ``cd /usr/local`` -``sudo curl -O -L https://nexus.payara.fish/repository/payara-community/fish/payara/distributions/payara/6.2024.6/payara-6.2024.6.zip`` +``sudo curl -O -L https://nexus.payara.fish/repository/payara-community/fish/payara/distributions/payara/6.2025.2/payara-6.2025.2.zip`` -``sudo unzip payara-6.2024.6.zip`` +``sudo unzip payara-6.2025.2.zip`` ``sudo chown -R $USER /usr/local/payara6`` @@ -113,30 +113,30 @@ Install Service Dependencies Directly on localhost Install PostgreSQL ^^^^^^^^^^^^^^^^^^ -The Dataverse Software has been tested with PostgreSQL versions up to 13. PostgreSQL version 10+ is required. +The Dataverse Software has been tested with PostgreSQL versions up to 17. PostgreSQL version 10+ is required. -On Mac, go to https://www.postgresql.org/download/macosx/ and choose "Interactive installer by EDB" option. Note that version 13.5 is used in the command line examples below, but the process should be similar for other versions. When prompted to set a password for the "database superuser (postgres)" just enter "password". +On Mac, go to https://www.postgresql.org/download/macosx/ and choose "Interactive installer by EDB" option. Note that version 16 is used in the command line examples below, but the process should be similar for other versions. When prompted to set a password for the "database superuser (postgres)" just enter "password". After installation is complete, make a backup of the ``pg_hba.conf`` file like this: -``sudo cp /Library/PostgreSQL/13/data/pg_hba.conf /Library/PostgreSQL/13/data/pg_hba.conf.orig`` +``sudo cp /Library/PostgreSQL/16/data/pg_hba.conf /Library/PostgreSQL/16/data/pg_hba.conf.orig`` Then edit ``pg_hba.conf`` with an editor such as vi: -``sudo vi /Library/PostgreSQL/13/data/pg_hba.conf`` +``sudo vi /Library/PostgreSQL/16/data/pg_hba.conf`` In the "METHOD" column, change all instances of "scram-sha-256" (or whatever is in that column) to "trust". This will make it so PostgreSQL doesn't require a password. -In the Finder, click "Applications" then "PostgreSQL 13" and launch the "Reload Configuration" app. Click "OK" after you see "server signaled". +In the Finder, click "Applications" then "PostgreSQL 16" and launch the "Reload Configuration" app. Click "OK" after you see "server signaled". -Next, to confirm the edit worked, launch the "pgAdmin" application from the same folder. Under "Browser", expand "Servers" and double click "PostgreSQL 13". When you are prompted for a password, leave it blank and click "OK". If you have successfully edited "pg_hba.conf", you can get in without a password. +Next, to confirm the edit worked, launch the "pgAdmin" application from the same folder. Under "Browser", expand "Servers" and double click "PostgreSQL 16". When you are prompted for a password, leave it blank and click "OK". If you have successfully edited "pg_hba.conf", you can get in without a password. On Linux, you should just install PostgreSQL using your favorite package manager, such as ``yum``. (Consult the PostgreSQL section of :doc:`/installation/prerequisites` in the main Installation guide for more info and command line examples). Find ``pg_hba.conf`` and set the authentication method to "trust" and restart PostgreSQL. Install Solr ^^^^^^^^^^^^ -`Solr `_ 9.4.1 is required. +`Solr `_ 9.8.0 is required. Follow the instructions in the "Installing Solr" section of :doc:`/installation/prerequisites` in the main Installation guide. diff --git a/doc/sphinx-guides/source/developers/dataset-migration-api.rst b/doc/sphinx-guides/source/developers/dataset-migration-api.rst index fc86b7ccdcf..941527133ef 100644 --- a/doc/sphinx-guides/source/developers/dataset-migration-api.rst +++ b/doc/sphinx-guides/source/developers/dataset-migration-api.rst @@ -5,10 +5,15 @@ The Dataverse software includes several ways to add Datasets originally created This experimental migration API offers an additional option with some potential advantages: -* metadata can be specified using the json-ld format used in the OAI-ORE metadata export -* existing publication dates and PIDs are maintained (currently limited to the case where the PID can be managed by the Dataverse software, e.g. where the authority and shoulder match those the software is configured for) -* updating the PID at the provider can be done immediately or later (with other existing APIs) -* adding files can be done via the standard APIs, including using direct-upload to S3 +* Metadata can be specified using the json-ld format used in the OAI-ORE metadata export. Please note that the json-ld generated by OAI-ORE metadata export is not directly compatible with the Migration API. OAI-ORE export nests resource metadata under :code:`ore:describes` wrapper and Dataset Migration API requires that metadata is on the root level. Please check example file below for reference. + + * If you need a tool to convert OAI-ORE exported json-ld into a format compatible with the Dataset Migration API, or if you need to generate compatible json-ld from sources other than an existing Dataverse installation, the `BaseX `_ database engine, used together with the XQuery language, provides an efficient solution. Please see example script :download:`transform-oai-ore-jsonld.xq <../_static/api/transform-oai-ore-jsonld.xq>` for a simple conversion from exported OAI-ORE json-ld to a Dataset Migration API -compatible version. + +* Existing publication dates and PIDs are maintained (currently limited to the case where the PID can be managed by the Dataverse software, e.g. where the authority and shoulder match those the software is configured for) + +* Updating the PID at the provider can be done immediately or later (with other existing APIs). + +* Adding files can be done via the standard APIs, including using direct-upload to S3. This API consists of 2 calls: one to create an initial Dataset version, and one to 'republish' the dataset through Dataverse with a specified publication date. Both calls require super-admin privileges. @@ -31,7 +36,13 @@ To import a dataset with an existing persistent identifier (PID), the provided j curl -H X-Dataverse-key:$API_TOKEN -X POST $SERVER_URL/api/dataverses/$DATAVERSE_ID/datasets/:startmigration --upload-file dataset-migrate.jsonld -An example jsonld file is available at :download:`dataset-migrate.jsonld <../_static/api/dataset-migrate.jsonld>` . Note that you would need to replace the PID in the sample file with one supported in your Dataverse instance. +An example jsonld file is available at :download:`dataset-migrate.jsonld <../_static/api/dataset-migrate.jsonld>` . Note that you would need to replace the PID in the sample file with one supported in your Dataverse instance. + +You also need to replace the :code:`dataverse.siteUrl` in the json-ld :code:`@context` with your current Dataverse site URL. This is necessary to define a local URI for metadata terms originating from community metadata blocks (in the case of the example file, from the Social Sciences and Humanities and Geospatial blocks). + +Currently, as of Dataverse 6.5 and earlier, community metadata blocks do not assign a default global URI to the terms used in the block in contrast to citation metadata, which has global URI defined. + + Publish a Migrated Dataset -------------------------- diff --git a/doc/sphinx-guides/source/developers/index.rst b/doc/sphinx-guides/source/developers/index.rst index 6d01e13d78e..90ccc261b1d 100755 --- a/doc/sphinx-guides/source/developers/index.rst +++ b/doc/sphinx-guides/source/developers/index.rst @@ -9,6 +9,7 @@ Developer Guide **Contents:** .. toctree:: + :maxdepth: 2 intro dev-environment diff --git a/doc/sphinx-guides/source/developers/make-data-count.rst b/doc/sphinx-guides/source/developers/make-data-count.rst index f347e7b8ff9..9fb41f67be4 100644 --- a/doc/sphinx-guides/source/developers/make-data-count.rst +++ b/doc/sphinx-guides/source/developers/make-data-count.rst @@ -49,7 +49,7 @@ Once you are done with your configuration, you can run Counter Processor like th ``su - counter`` -``cd /usr/local/counter-processor-1.05`` +``cd /usr/local/counter-processor-1.06`` ``CONFIG_FILE=counter-processor-config.yaml python39 main.py`` @@ -82,7 +82,7 @@ Second, if you are also sending your SUSHI report to Make Data Count, you will n ``curl -H "Authorization: Bearer $JSON_WEB_TOKEN" -X DELETE https://$MDC_SERVER/reports/$REPORT_ID`` -To get the ``REPORT_ID``, look at the logs generated in ``/usr/local/counter-processor-1.05/tmp/datacite_response_body.txt`` +To get the ``REPORT_ID``, look at the logs generated in ``/usr/local/counter-processor-1.06/tmp/datacite_response_body.txt`` To read more about the Make Data Count api, see https://github.com/datacite/sashimi @@ -110,9 +110,11 @@ The script will process the newest set of log files (merging files from multiple APIs to manage the states include GET, POST, and DELETE (for testing), as shown below. Note: ``yearMonth`` must be in the format ``yyyymm`` or ``yyyymmdd``. +Note: If running the new script on multiple servers add the query parameter &server=serverName on the first POST call. The server name can not be changed once set. To clear the name out you must delete the state and post a new one. ``curl -X GET http://localhost:8080/api/admin/makeDataCount/{yearMonth}/processingState`` +``curl -X POST http://localhost:8080/api/admin/makeDataCount/{yearMonth}/processingState?state=processing&server=server1`` ``curl -X POST http://localhost:8080/api/admin/makeDataCount/{yearMonth}/processingState?state=done`` ``curl -X DELETE http://localhost:8080/api/admin/makeDataCount/{yearMonth}/processingState`` diff --git a/doc/sphinx-guides/source/developers/making-releases.rst b/doc/sphinx-guides/source/developers/making-releases.rst index aed174f60d4..8f9b43eabcb 100755 --- a/doc/sphinx-guides/source/developers/making-releases.rst +++ b/doc/sphinx-guides/source/developers/making-releases.rst @@ -30,7 +30,24 @@ Early on, make sure it's clear what type of release this is. The steps below des Ensure Issues Have Been Created ------------------------------- -In advance of a release, GitHub issues should have been created already that capture certain steps. See https://github.com/IQSS/dataverse-pm/issues/335 for examples. +Some of the steps in this document are well-served by having their own dedicated GitHub issue. You'll see a label like this on them: + +|dedicated| + +There are a variety of reasons why a step might deserve its own dedicated issue: + +- The step can be done by a team member other than the person doing the release. +- Stakeholders might be interested in the status of a step (e.g. has the released been deployed to the demo site). + +Steps don't get their own dedicated issue if it would be confusing to have multiple people involved. Too many cooks in the kitchen, as they say. Also, some steps are so small the overhead of an issue isn't worth it. + +Before the release even begins you can coordinate with the project manager about the creation of these issues. + +.. |dedicated| raw:: html + + + Dedicated Issue +   Declare a Code Freeze --------------------- @@ -40,18 +57,25 @@ The following steps are made more difficult if code is changing in the "develop" Conduct Performance Testing --------------------------- +|dedicated| + See :doc:`/qa/performance-tests` for details. -Conduct Smoke Testing ---------------------- +Conduct Regression Testing +--------------------------- + +|dedicated| See :doc:`/qa/testing-approach` for details. +Refer to the provided regression checklist for the list of items to verify during the testing process: `Regression Checklist `_. .. _write-release-notes: Write Release Notes ------------------- +|dedicated| + Developers express the need for an addition to release notes by creating a "release note snippet" in ``/doc/release-notes`` containing the name of the issue they're working on. The name of the branch could be used for the filename with ".md" appended (release notes are written in Markdown) such as ``5053-apis-custom-homepage.md``. See :ref:`writing-release-note-snippets` for how this is described for contributors. The task at or near release time is to collect these snippets into a single file. @@ -62,17 +86,22 @@ The task at or near release time is to collect these snippets into a single file - Include instructions describing the steps required to upgrade the application from the previous version. These must be customized for release numbers and special circumstances such as changes to metadata blocks and infrastructure. - Take the release notes .md through the regular Code Review and QA process. That is, make a pull request. Here's an example: https://github.com/IQSS/dataverse/pull/10866 -Upgrade Instructions for Internal ---------------------------------- +Deploy Release Candidate to Internal +------------------------------------ + +|dedicated| To upgrade internal, go to /doc/release-notes, open the release-notes.md file for the current release and perform all the steps under "Upgrade Instructions". Deploy Release Candidate to Demo -------------------------------- +|dedicated| + First, build the release candidate. ssh into the dataverse-internal server and undeploy the current war file. +Go to /doc/release-notes, open the release-notes.md file for the current release, and perform all the steps under "Upgrade Instructions". Go to https://jenkins.dataverse.org/job/IQSS_Dataverse_Internal/ and make the following adjustments to the config: @@ -91,6 +120,8 @@ ssh into the demo server and follow the upgrade instructions in the release note Prepare Release Branch ---------------------- +|dedicated| + The release branch will have the final changes such as bumping the version number. Usually we branch from the "develop" branch to create the release branch. If we are creating a hotfix for a particular version (5.11, for example), we branch from the tag (e.g. ``v5.11``). @@ -116,18 +147,20 @@ Return to the parent pom and make the following change, which is necessary for p (Before you make this change the value should be ``${parsedVersion.majorVersion}.${parsedVersion.nextMinorVersion}``. Later on, after cutting a release, we'll change it back to that value.) -For a regular release, make the changes above in the release branch you created, make a pull request, and merge it into the "develop" branch. Like usual, you can safely delete the branch after the merge is complete. +For a regular release, make the changes above in the release branch you created, but hold off for a moment on making a pull requests because Jenkins will fail because it will be testing the previous release. -If you are making a hotfix release, make the pull request against the "master" branch. Do not delete the branch after merging because we will later merge it into the "develop" branch to pick up the hotfix. More on this later. +In the dataverse-ansible repo make bump the version in `jenkins.yml `_ and make a pull request such as https://github.com/gdcc/dataverse-ansible/pull/386. Wait for it to be merged. Note that bumping on the Jenkins side like this will mean that all pull requests will show failures in Jenkins until they are updated to the version we are releasing. -Either way, as usual, you should ensure that all tests are passing. Please note that you will need to bump the version in `jenkins.yml `_ in dataverse-ansible to get the tests to pass. Consider doing this before making the pull request. Alternatively, you can bump jenkins.yml after making the pull request and re-run the Jenkins job to make sure tests pass. +Once dataverse-ansible has been merged, return to the branch you created above ("10852-bump-to-6.4" or whatever) and make a pull request. Ensure that all tests are passing and then put the PR through the normal review and QA process. + +If you are making a hotfix release, make the pull request against the "master" branch. Do not delete the branch after merging because we will later merge it into the "develop" branch to pick up the hotfix. More on this later. Merge "develop" into "master" ----------------------------- If this is a regular (non-hotfix) release, create a pull request to merge the "develop" branch into the "master" branch using this "compare" link: https://github.com/IQSS/dataverse/compare/master...develop -Once important tests have passed (compile, unit tests, etc.), merge the pull request. Don't worry about style tests failing such as for shell scripts. +Once important tests have passed (compile, unit tests, etc.), merge the pull request (skipping code review is ok). Don't worry about style tests failing such as for shell scripts. If this is a hotfix release, skip this whole "merge develop to master" step (the "develop" branch is not involved until later). @@ -160,7 +193,7 @@ Go to https://jenkins.dataverse.org/job/guides.dataverse.org/ and make the follo - Repository URL: ``https://github.com/IQSS/dataverse.git`` - Branch Specifier (blank for 'any'): ``*/master`` -- ``VERSION`` (under "Build Steps"): ``5.10.1`` (for example) +- ``VERSION`` (under "Build Steps"): bump to the next release. Don't prepend a "v". Use ``5.10.1`` (for example) Click "Save" then "Build Now". @@ -265,24 +298,37 @@ Close Milestone on GitHub and Create a New One You can find our milestones at https://github.com/IQSS/dataverse/milestones -Now that we've published the release, close the milestone and create a new one. +Now that we've published the release, close the milestone and create a new one for the **next** release, the release **after** the one we're working on, that is. Note that for milestones we use just the number without the "v" (e.g. "5.10.1"). +On the project board at https://github.com/orgs/IQSS/projects/34 edit the tab (view) that shows the milestone to show the next milestone. + Update the Container Base Image Version Property ------------------------------------------------ +|dedicated| + Create a new branch (any name is fine but ``prepare-next-iteration`` is suggested) and update the following files to prepare for the next development cycle: - modules/dataverse-parent/pom.xml -> ```` -> profile "ct" -> ```` -> Set ```` to ``${parsedVersion.majorVersion}.${parsedVersion.nextMinorVersion}`` -Now create a pull request and merge it. +Create a pull request and put it through code review, like usual. Give it a milestone of the next release, the one **after** the one we're working on. Once the pull request has been approved, merge it. It should the the first PR merged of the next release. For more background, see :ref:`base-supported-image-tags`. For an example, see https://github.com/IQSS/dataverse/pull/10896 +Lift the Code Freeze and Encourage Developers to Update Their Branches +---------------------------------------------------------------------- + +It's now safe to lift the code freeze. We can start merging pull requests into the "develop" branch for the next release. + +Let developers know that they should merge the latest from the "develop" branch into any branches they are working on. + Deploy Final Release on Demo ---------------------------- +|dedicated| + Above you already did the hard work of deploying a release candidate to https://demo.dataverse.org. It should be relatively straightforward to undeploy the release candidate and deploy the final release. Update SchemaSpy @@ -316,6 +362,11 @@ Announce the Release on the Mailing List Post a message at https://groups.google.com/g/dataverse-community +Announce the Release on Zulip +----------------------------- + +Post a message under #community at https://dataverse.zulipchat.com + For Hotfixes, Merge Hotfix Branch into "develop" and Rename SQL Scripts ----------------------------------------------------------------------- diff --git a/doc/sphinx-guides/source/developers/tips.rst b/doc/sphinx-guides/source/developers/tips.rst index f5ffbac0c07..d466dff5c76 100755 --- a/doc/sphinx-guides/source/developers/tips.rst +++ b/doc/sphinx-guides/source/developers/tips.rst @@ -185,7 +185,24 @@ Solr Once some Dataverse collections, datasets, and files have been created and indexed, you can experiment with searches directly from Solr at http://localhost:8983/solr/#/collection1/query and look at the JSON output of searches, such as this wildcard search: http://localhost:8983/solr/collection1/select?q=*%3A*&wt=json&indent=true . You can also get JSON output of static fields Solr knows about: http://localhost:8983/solr/collection1/schema/fields -You can simply double-click "start.jar" rather that running ``java -jar start.jar`` from the command line. Figuring out how to stop Solr after double-clicking it is an exercise for the reader. +You can simply double-click "start.jar" rather than running ``java -jar start.jar`` from the command line. Figuring out how to stop Solr after double-clicking it is an exercise for the reader. + +.. _update-solr-schema-dev: + +Updating the Solr Schema (Developers) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Both developers and sysadmins need to update the Solr schema from time to time. One difference is that developers will be committing changes to ``conf/solr/schema.xml`` in git. To prevent cross-platform differences in the git history, when running the ``update-fields.sh`` script, we ask all developers to run the script from within Docker. (See :doc:`/container/configbaker-image` for more on the image we'll use below.) + +.. code-block:: + + curl http://localhost:8080/api/admin/index/solr/schema | docker run -i --rm -v ./docker-dev-volumes/solr/data:/var/solr gdcc/configbaker:unstable update-fields.sh /var/solr/data/collection1/conf/schema.xml + + cp docker-dev-volumes/solr/data/data/collection1/conf/schema.xml conf/solr/schema.xml + +At this point you can do a ``git diff`` and see if your changes make sense before committing. + +Sysadmins are welcome to run ``update-fields.sh`` however they like. See :ref:`update-solr-schema` in the Admin Guide for details. Git --- @@ -279,3 +296,14 @@ with the following code in ``SettingsWrapper.java``: A more serious example would be direct calls to PermissionServiceBean methods used in render logic expressions. This is something that has happened and caused some problems in real life. A simple permission service lookup (for example, whether a user is authorized to create a dataset in the current dataverse) can easily take 15 database queries. Repeated multiple times, this can quickly become a measurable delay in rendering the page. PermissionsWrapper must be used exclusively for any such lookups from JSF pages. See also :doc:`performance`. + +JSF1103 Errors +~~~~~~~~~~~~~~ + +Errors of the form ``JSF1103: The metadata facet must be a direct child of the view in viewId /dataverse.xhtml`` come from use of the f:metadata tag at the wrong depth in the .xhtml. + +Most/all known instances of the problem were corrected in https://github.com/IQSS/dataverse/pull/11128. + +Any page that used was including the f:metadata farther down in the tree rather than as a direct child of the view. +As of Payara 6.2025.2, it is not clear that this error was resulting in changes to UI behavior, but the error messages were in the log. +If you see these errors, this note and the examples in the PR will hopefully provide some insight as to how to fix them. diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index 30a36da9499..6aa5f5c8ff6 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -151,6 +151,18 @@ Password complexity rules for "builtin" accounts can be adjusted with a variety - :ref:`:PVGoodStrength` - :ref:`:PVCustomPasswordResetAlertMessage` +.. _samesite-cookie-attribute: + +SameSite Cookie Attribute +^^^^^^^^^^^^^^^^^^^^^^^^^ + +The SameSite cookie attribute is defined in an upcoming revision to `RFC 6265 `_ (HTTP State Management Mechanism) called `6265bis `_ ("bis" meaning "repeated"). The possible values are "None", "Lax", and "Strict". "Strict" is intended to help prevent Cross-Site Request Forgery (CSRF) attacks, as described in the RFC proposal and an OWASP `cheetsheet `_. We don't recommend "None" for security reasons. + +By default, Payara doesn't send the SameSite cookie attribute, which browsers should interpret as "Lax" according to `MDN `_. +Dataverse installations are explicity set to "Lax" out of the box by the installer (in the case of a "classic" installation) or through the base image (in the case of a Docker installation). For classic, see :ref:`http.cookie-same-site-value` and :ref:`http.cookie-same-site-enabled` for how to change the values. For Docker, you must rebuild the :doc:`base image `. See also Payara's `documentation `_ for the settings above. + +To inspect cookie attributes like SameSite, you can use ``curl -s -I http://localhost:8080 | grep JSESSIONID``, for example, looking for the "Set-Cookie" header. + .. _ongoing-security: Ongoing Security of Your Installation @@ -307,7 +319,7 @@ to be compatible with the MicroProfile specification which means that Global Settings ^^^^^^^^^^^^^^^ -The following three global settings are required to configure PID Providers in the Dataverse software: +The following two global settings are required to configure PID Providers in the Dataverse software: .. _dataverse.pid.providers: @@ -581,6 +593,7 @@ Note: - If you configure ``base-url``, it should include a "/" after the hostname like this: ``https://demo.dataverse.org/``. - When using multiple PermaLink providers, you should avoid ambiguous authority/separator/shoulder combinations that would result in the same overall prefix. +- Configuring PermaLink providers differing only by their separator values is not supported. - In general, PermaLink authority/shoulder values should be alphanumeric. For other cases, admins may need to consider the potential impact of special characters in S3 storage identifiers, resolver URLs, exports, etc. .. _dataverse.pid.*.handlenet: @@ -1093,6 +1106,8 @@ The Dataverse Software S3 driver supports multi-part upload for large files (ove First: Set Up Accounts and Access Credentials ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +**Note:** As of version 5.14, if Dataverse is running in an EC2 instance it will prefer Role-Based Access Control over the S3 default profile, even if administrators configure Dataverse with programmatic access keys. Named profiles can still be used to override RBAC for specific datastores. RBAC is preferential from a security perspective as there are no keys to rotate or have stolen. If you intend to assign a role to your EC2 instance, you will still need the ``~/.aws/config`` file to specify the region but you need not generate credentials for the default profile. For more information please see https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2.html + The Dataverse Software and the AWS SDK make use of the "AWS credentials profile file" and "AWS config profile file" located in ``~/.aws/`` where ``~`` is the home directory of the user you run Payara as. This file can be generated via either of two methods described below: @@ -1116,13 +1131,6 @@ To **create a user** with full S3 access and nothing more for security reasons, for more info on this process. To use programmatic access, **Generate the user keys** needed for a Dataverse installation afterwards by clicking on the created user. -(You can skip this step when running on EC2, see below.) - -.. TIP:: - If you are hosting your Dataverse installation on an AWS EC2 instance alongside storage in S3, it is possible to use IAM Roles instead - of the credentials file (the file at ``~/.aws/credentials`` mentioned below). Please note that you will still need the - ``~/.aws/config`` file to specify the region. For more information on this option, see - https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2.html Preparation When Using Custom S3-Compatible Service ################################################### @@ -1854,6 +1862,128 @@ For Google Analytics, the example script at :download:`analytics-code.html + + + +2. Add to ``analytics-code.html``: + +```` + +3. Go to https://playground.cookieconsent.orestbida.com to configure, download and copy contents of ``cookieconsent-config.js`` to ``analytics-code.html``. It should look something like this: + +.. code-block:: html + + + +After restarting or reloading Dataverse the cookie consent popup should appear, looking something like this: + +|cookieconsent| + +.. |cookieconsent| image:: ./img/cookie-consent-example.png + :class: img-responsive + +If you change the cookie consent config in ``CookieConsent.run()`` and want to test your changes, you should remove the cookie called ``cc_cookie`` in your browser and reload the Dataverse page to have the popup appear again. To remove cookies use Application > Cookies in the Chrome/Edge dev tool, and Storage > Cookies in Firefox and Safari. + .. _license-config: Configuring Licenses @@ -1916,6 +2046,11 @@ JSON files for software licenses are provided below. - :download:`licenseMIT.json <../../../../scripts/api/data/licenses/licenseMIT.json>` - :download:`licenseApache-2.0.json <../../../../scripts/api/data/licenses/licenseApache-2.0.json>` +Adding Country-Specific Licenses +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +- :download:`licenseEtalab-2.0.json <../../../../scripts/api/data/licenses/licenseEtalab-2.0.json>` used in France (Etalab Open License 2.0, CC-BY 2.0 compliant). + Contributing to the Collection of Standard Licenses Above ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -3323,12 +3458,22 @@ dataverse.files.globus-monitoring-server This setting is required in conjunction with the ``globus-use-experimental-async-framework`` feature flag (see :ref:`feature-flags`). Setting it to true designates the Dataverse instance to serve as the dedicated polling server. It is needed so that the new framework can be used in a multi-node installation. +.. _dataverse.csl.common-styles: + +dataverse.csl.common-styles ++++++++++++++++++++++++++++ + +This setting allows admins to highlight a few of the 1000+ CSL citation styles available from the dataset page. The value should be a comma-separated list of styles. +These will be listed above the alphabetical list of all styles in the "View Styled Citations" pop-up. +The default value when not set is "chicago-author-date, ieee". + + .. _feature-flags: Feature Flags ------------- -Certain features might be deactivated because they are experimental and/or opt-in previews. If you want to enable these, +Certain features might be deactivated because they are experimental and/or opt-in capabilities. If you want to enable these, please find all known feature flags below. Any of these flags can be activated using a boolean value (case-insensitive, one of "true", "1", "YES", "Y", "ON") for the setting. @@ -3343,6 +3488,15 @@ please find all known feature flags below. Any of these flags can be activated u * - api-session-auth - Enables API authentication via session cookie (JSESSIONID). **Caution: Enabling this feature flag exposes the installation to CSRF risks!** We expect this feature flag to be temporary (only used by frontend developers, see `#9063 `_) and for the feature to be removed in the future. - ``Off`` + * - api-bearer-auth + - Enables API authentication via Bearer Token. + - ``Off`` + * - api-bearer-auth-provide-missing-claims + - Enables sending missing user claims in the request JSON provided during OIDC user registration, when these claims are not returned by the identity provider and are required for registration. This feature only works when the feature flag ``api-bearer-auth`` is also enabled. **Caution: Enabling this feature flag exposes the installation to potential user impersonation issues.** + - ``Off`` + * - api-bearer-auth-handle-tos-acceptance-in-idp + - Specifies that Terms of Service acceptance is handled by the IdP, eliminating the need to include ToS acceptance boolean parameter (termsAccepted) in the OIDC user registration request body. This feature only works when the feature flag ``api-bearer-auth`` is also enabled. + - ``Off`` * - avoid-expensive-solr-join - Changes the way Solr queries are constructed for public content (published Collections, Datasets and Files). It removes a very expensive Solr join on all such documents, improving overall performance, especially for large instances under heavy load. Before this feature flag is enabled, the corresponding indexing feature (see next feature flag) must be turned on and a full reindex performed (otherwise public objects are not going to be shown in search results). See :doc:`/admin/solr-search-index`. - ``Off`` @@ -3352,9 +3506,6 @@ please find all known feature flags below. Any of these flags can be activated u * - reduce-solr-deletes - Avoids deleting and recreating solr documents for dataset files when reindexing. - ``Off`` - * - reduce-solr-deletes - - Avoids deleting and recreating solr documents for dataset files when reindexing. - - ``Off`` * - disable-return-to-author-reason - Removes the reason field in the `Publish/Return To Author` dialog that was added as a required field in v6.2 and makes the reason an optional parameter in the :ref:`return-a-dataset` API call. - ``Off`` @@ -3362,7 +3513,13 @@ please find all known feature flags below. Any of these flags can be activated u - Turns off automatic selection of a dataset thumbnail from image files in that dataset. When set to ``On``, a user can still manually pick a thumbnail image or upload a dedicated thumbnail image. - ``Off`` * - globus-use-experimental-async-framework - - Activates a new experimental implementation of Globus polling of ongoing remote data transfers that does not rely on the instance staying up continuously for the duration of the transfers and saves the state information about Globus upload requests in the database. Added in v6.4. Affects :ref:`:GlobusPollingInterval`. Note that the JVM option :ref:`dataverse.files.globus-monitoring-server` described above must also be enabled on one (and only one, in a multi-node installation) Dataverse instance. + - Activates a new experimental implementation of Globus polling of ongoing remote data transfers that does not rely on the instance staying up continuously for the duration of the transfers and saves the state information about Globus upload requests in the database. Added in v6.4; extended in v6.6 to cover download transfers, in addition to uploads. Affects :ref:`:GlobusPollingInterval`. Note that the JVM option :ref:`dataverse.files.globus-monitoring-server` described above must also be enabled on one (and only one, in a multi-node installation) Dataverse instance. + - ``Off`` + * - index-harvested-metadata-source + - Index the nickname or the source name (See the optional ``sourceName`` field in :ref:`create-a-harvesting-client`) of the harvesting client as the "metadata source" of harvested datasets and files. If enabled, the Metadata Source facet will show separate groupings of the content harvested from different sources (by harvesting client nickname or source name) instead of the default behavior where there is one "Harvested" grouping for all harvested content. + - ``Off`` + * - enable-version-note + - Turns on the ability to add/view/edit/delete per-dataset-version notes intended to provide :ref:`provenance` information about why the dataset/version was created. - ``Off`` **Note:** Feature flags can be set via any `supported MicroProfile Config API source`_, e.g. the environment variable @@ -3384,6 +3541,32 @@ To facilitate large file upload and download, the Dataverse Software installer b and restart Payara to apply your change. +.. _http.cookie-same-site-value: + +http.cookie-same-site-value +++++++++++++++++++++++++++++ + +See :ref:`samesite-cookie-attribute` for context. + +The Dataverse installer configures the Payara **server-config.network-config.protocols.protocol.http-listener-1.http.cookie-same-site-value** setting to "Lax". From `Payara's documentation `_, the other possible values are "Strict" or "None". To change this to "Strict", for example, you could run the following command... + +``./asadmin set server-config.network-config.protocols.protocol.http-listener-1.http.cookie-same-site-value=Strict`` + +... and restart Payara to apply your change. + +.. _http.cookie-same-site-enabled: + +http.cookie-same-site-enabled ++++++++++++++++++++++++++++++ + +See :ref:`samesite-cookie-attribute` for context. + +The Dataverse installer configures the Payara **server-config.network-config.protocols.protocol.http-listener-1.http.cookie-same-site-enabled** setting to true. To change this to false, you could run the following command... + +``./asadmin set server-config.network-config.protocols.protocol.http-listener-1.http.cookie-same-site-enabled=true`` + +... and restart Payara to apply your change. + mp.config.profile +++++++++++++++++ @@ -4357,9 +4540,10 @@ Limit on how many guestbook entries to display on the guestbook-responses page. :CustomDatasetSummaryFields +++++++++++++++++++++++++++ -You can replace the default dataset metadata fields that are displayed above files table on the dataset page with a custom list separated by commas using the curl command below. +You can replace the default dataset metadata fields that are displayed above files table on the dataset page with a custom list separated by commas (with optional spaces) using the curl command below. +Note that the License is always displayed and that the description, subject, keywords, etc. will NOT be displayed if you do not include them in the :CustomDatasetSummaryFields. -``curl http://localhost:8080/api/admin/settings/:CustomDatasetSummaryFields -X PUT -d 'producer,subtitle,alternativeTitle'`` +``curl http://localhost:8080/api/admin/settings/:CustomDatasetSummaryFields -X PUT -d 'producer,subtitle, alternativeTitle'`` You have to put the datasetFieldType name attribute in the :CustomDatasetSummaryFields setting for this to work. @@ -4418,7 +4602,12 @@ This is enabled via the new setting `:MDCStartDate` that specifies the cut-over ``curl -X PUT -d '2019-10-01' http://localhost:8080/api/admin/settings/:MDCStartDate`` +:ContactFeedbackMessageSizeLimit +++++++++++++++++++++++++++++++++ + +Maximum length of the text body that can be sent to the contacts of a Collection, Dataset, or DataFile. Setting this limit to Zero will denote unlimited length. +``curl -X PUT -d 1080 http://localhost:8080/api/admin/settings/:ContactFeedbackMessageSizeLimit`` .. _:Languages: @@ -4653,6 +4842,9 @@ The commands below should give you an idea of how to load the configuration, but ``curl -X PUT --upload-file cvoc-conf.json http://localhost:8080/api/admin/settings/:CVocConf`` +Since external vocabulary scripts can change how fields are indexed (storing an identifier and name and/or values in different languages), +updating the Solr schema as described in :ref:`update-solr-schema` should be done after adding new scripts to your configuration. + .. _:ControlledVocabularyCustomJavaScript: :ControlledVocabularyCustomJavaScript diff --git a/doc/sphinx-guides/source/installation/img/cookie-consent-example.png b/doc/sphinx-guides/source/installation/img/cookie-consent-example.png new file mode 100644 index 00000000000..0dfe1fb113b Binary files /dev/null and b/doc/sphinx-guides/source/installation/img/cookie-consent-example.png differ diff --git a/doc/sphinx-guides/source/installation/index.rst b/doc/sphinx-guides/source/installation/index.rst index 1965448aedb..7f3a2cc6e1d 100755 --- a/doc/sphinx-guides/source/installation/index.rst +++ b/doc/sphinx-guides/source/installation/index.rst @@ -9,6 +9,7 @@ Installation Guide **Contents:** .. toctree:: + :maxdepth: 2 intro prep @@ -19,5 +20,6 @@ Installation Guide shibboleth oauth2 oidc + orcid external-tools advanced diff --git a/doc/sphinx-guides/source/installation/installation-main.rst b/doc/sphinx-guides/source/installation/installation-main.rst index 3c3376e3c85..837ca6f5a88 100755 --- a/doc/sphinx-guides/source/installation/installation-main.rst +++ b/doc/sphinx-guides/source/installation/installation-main.rst @@ -22,7 +22,7 @@ You should have already downloaded the installer from https://github.com/IQSS/da Unpack the zip file - this will create the directory ``dvinstall``. -**Important:** The installer will need to use the PostgreSQL command line utility ``psql`` in order to configure the database. If the executable is not in your system PATH, the installer will try to locate it on your system. However, we strongly recommend that you check and make sure it is in the PATH. This is especially important if you have multiple versions of PostgreSQL installed on your system. Make sure the psql that came with the version that you want to use with your Dataverse installation is the first on your path. For example, if the PostgreSQL distribution you are running is installed in /Library/PostgreSQL/13, add /Library/PostgreSQL/13/bin to the beginning of your $PATH variable. If you are *running* multiple PostgreSQL servers, make sure you know the port number of the one you want to use, as the installer will need it in order to connect to the database (the first PostgreSQL distribution installed on your system is likely using the default port 5432; but the second will likely be on 5433, etc.) Does every word in this paragraph make sense? If it does, great - because you definitely need to be comfortable with basic system tasks in order to install the Dataverse Software. If not - if you don't know how to check where your PostgreSQL is installed, or what port it is running on, or what a $PATH is... it's not too late to stop. Because it will most likely not work. And if you contact us for help, these will be the questions we'll be asking you - so, again, you need to be able to answer them comfortably for it to work. +**Important:** The installer will need to use the PostgreSQL command line utility ``psql`` in order to configure the database. If the executable is not in your system PATH, the installer will try to locate it on your system. However, we strongly recommend that you check and make sure it is in the PATH. This is especially important if you have multiple versions of PostgreSQL installed on your system. Make sure the psql that came with the version that you want to use with your Dataverse installation is the first on your path. For example, if the PostgreSQL distribution you are running is installed in /Library/PostgreSQL/16, add /Library/PostgreSQL/16/bin to the beginning of your $PATH variable. If you are *running* multiple PostgreSQL servers, make sure you know the port number of the one you want to use, as the installer will need it in order to connect to the database (the first PostgreSQL distribution installed on your system is likely using the default port 5432; but the second will likely be on 5433, etc.) Does every word in this paragraph make sense? If it does, great - because you definitely need to be comfortable with basic system tasks in order to install the Dataverse Software. If not - if you don't know how to check where your PostgreSQL is installed, or what port it is running on, or what a $PATH is... it's not too late to stop. Because it will most likely not work. And if you contact us for help, these will be the questions we'll be asking you - so, again, you need to be able to answer them comfortably for it to work. **It is no longer necessary to run the installer as root!** diff --git a/doc/sphinx-guides/source/installation/oauth2.rst b/doc/sphinx-guides/source/installation/oauth2.rst index 7a0e938b572..82d82c202b1 100644 --- a/doc/sphinx-guides/source/installation/oauth2.rst +++ b/doc/sphinx-guides/source/installation/oauth2.rst @@ -38,8 +38,10 @@ URLs to help you request a Client ID and Client Secret from the providers suppor Each of these providers will require the following information from you: - Basic information about your Dataverse installation such as a name, description, URL, logo, privacy policy, etc. -- OAuth2 Redirect URI (ORCID) or Redirect URI (Microsoft Azure AD) or Authorization Callback URL (GitHub) or Authorized Redirect URIs (Google): This is the URL on the Dataverse installation side to which the user will be sent after successfully authenticating with the identity provider. This should be the advertised URL of your Dataverse installation (the protocol, fully qualified domain name, and optional port configured via the ``dataverse.siteUrl`` JVM option mentioned in the :doc:`config` section) appended with ``/oauth2/callback.xhtml`` such as ``https://dataverse.example.edu/oauth2/callback.xhtml``. +- OAuth2 Redirect URI(s) (ORCID) or Redirect URI (Microsoft Azure AD) or Authorization Callback URL (GitHub) or Authorized Redirect URIs (Google): This is the URL on the Dataverse installation side to which the user will be sent after successfully authenticating with the identity provider. This should be the advertised URL of your Dataverse installation (the protocol, fully qualified domain name, and optional port configured via the ``dataverse.siteUrl`` JVM option mentioned in the :doc:`config` section) appended with ``/oauth2/callback.xhtml`` such as ``https://dataverse.example.edu/oauth2/callback.xhtml``. +For ORCID, if you also want to enable the ability to associate ORCIDs with user accounts (when users did not login via ORCID) as discussed in :doc:`orcid`, you must add a second redirect URL: to ``/oauth2/orcidConfirm.xhtml`` such as ``https://dataverse.example.edu/oauth2/orcidConfirm.xhtml``. + When you are finished you should have a Client ID and Client Secret from the provider. Keep them safe and secret. Dataverse Installation Side diff --git a/doc/sphinx-guides/source/installation/orcid.rst b/doc/sphinx-guides/source/installation/orcid.rst new file mode 100644 index 00000000000..c998b27f828 --- /dev/null +++ b/doc/sphinx-guides/source/installation/orcid.rst @@ -0,0 +1,26 @@ +ORCID Integration +================= + +.. contents:: |toctitle| + :local: + +Introduction +------------ + +Dataverse leverages ORCIDs (and other types of persistent identifiers (PIDs)) to improve the findability of data and to simplify the process of adding metadata. +When ORCIDs are included as metadata about authors, Dataverse includes them in metadata exports, advertises them through :ref:`discovery-sign-posting` and via metadata embedded in dataset pages, and includes them in the metadata associated with dataset DOIs. + +Dataverse can be configured to make it easier to include ORCIDs +- via use of an ORCID "External Vocabulary Script" that allows users to lookup authors, depositors, etc. based on their ORCID profile metadata and then records these ORCIDs automatically and adds links to ORCID profiles in metadata displays. With this configured, there is no need enter ORCIDs directly. See :ref:`using-external-vocabulary-services` in the Admin Guide. +- via association of ORCIDs with Dataverse user accounts, through the use of ORCID logins or, in addition or instead, a separate authenticated ORCID linking mechanism. When an ORCID is associated with a Dataverse account, it will automatically be added to the dataset metadata when a user creates a dataset and is added as an initial author. + +See also :ref:`orcid-integration` in the User Guide. + +Configuration +-------------- + +The steps needed to configure Dataverse to support lookup of ORCIDs for the author metadata field (and ROR identifiers for organizations as author affiliations) is described in the `Dataverse Author Field Example page `_ in the `Dataverse External Vocabulary Suport Github Repository `_. Briefly, this involves changing the :ref:`:CVocConf` setting and potentially creating local web-acessible copies of the relevant scripts. + +To configure Dataverse to support adding ORCIDs to user profiles, one must configure ORCID as an OAuth2 provider as described in :doc:`oauth2`. The ability to link ORCIDs to user accounts is automatically enabled if an ORCID provider is configured. To avoid also enabling ORCID login, the provider can be registered with "enabled":false. + + diff --git a/doc/sphinx-guides/source/installation/prerequisites.rst b/doc/sphinx-guides/source/installation/prerequisites.rst index f61321ef245..0e17bc47166 100644 --- a/doc/sphinx-guides/source/installation/prerequisites.rst +++ b/doc/sphinx-guides/source/installation/prerequisites.rst @@ -44,7 +44,7 @@ On RHEL/derivative you can make Java 17 the default with the ``alternatives`` co Payara ------ -Payara 6.2024.6 is recommended. Newer versions might work fine. Regular updates are recommended. +Payara 6.2025.2 is recommended. Newer versions might work fine. Regular updates are recommended. Installing Payara ================= @@ -55,8 +55,8 @@ Installing Payara - Download and install Payara (installed in ``/usr/local/payara6`` in the example commands below):: - # wget https://nexus.payara.fish/repository/payara-community/fish/payara/distributions/payara/6.2024.6/payara-6.2024.6.zip - # unzip payara-6.2024.6.zip + # wget https://nexus.payara.fish/repository/payara-community/fish/payara/distributions/payara/6.2025.2/payara-6.2025.2.zip + # unzip payara-6.2025.2.zip # mv payara6 /usr/local If nexus.payara.fish is ever down for maintenance, Payara distributions are also available from https://repo1.maven.org/maven2/fish/payara/distributions/payara/ @@ -97,21 +97,21 @@ Also note that Payara may utilize more than the default number of file descripto PostgreSQL ---------- -PostgreSQL 13 is recommended because it's the version we test against. Version 10 or higher is required because that's what's `supported by Flyway `_, which we use for database migrations. +PostgreSQL 16 is recommended because it's the version we test against. Version 10 or higher is required because that's what's `supported by Flyway `_, which we use for database migrations. You are welcome to experiment with newer versions of PostgreSQL, but please note that as of PostgreSQL 15, permissions have been restricted on the ``public`` schema (`release notes `_, `EDB blog post `_, `Crunchy Data blog post `_). The Dataverse installer has been updated to restore the old permissions, but this may not be a long term solution. Installing PostgreSQL ===================== -*For example*, to install PostgreSQL 13 under RHEL7/derivative:: +*For example*, to install PostgreSQL 16 under RHEL9/derivative:: - # yum install -y https://download.postgresql.org/pub/repos/yum/reporpms/EL-7-x86_64/pgdg-redhat-repo-latest.noarch.rpm + # yum install -y https://download.postgresql.org/pub/repos/yum/reporpms/EL-9-x86_64/pgdg-redhat-repo-latest.noarch.rpm # yum makecache fast - # yum install -y postgresql13-server - # /usr/pgsql-13/bin/postgresql-13-setup initdb - # /usr/bin/systemctl start postgresql-13 - # /usr/bin/systemctl enable postgresql-13 + # yum install -y postgresql16-server + # /usr/pgsql-16/bin/postgresql-16-setup initdb + # /usr/bin/systemctl start postgresql-16 + # /usr/bin/systemctl enable postgresql-16 For RHEL8/derivative the process would be identical, except for the first two commands: you would need to install the "EL-8" yum repository configuration and run ``yum makecache`` instead. @@ -149,7 +149,7 @@ Configuring Database Access for the Dataverse Installation (and the Dataverse So - **Important: PostgreSQL must be restarted** for the configuration changes to take effect! On RHEL7/derivative and similar (provided you installed Postgres as instructed above):: - # systemctl restart postgresql-13 + # systemctl restart postgresql-16 On MacOS X a "Reload Configuration" icon is usually supplied in the PostgreSQL application folder. Or you could look up the process id of the PostgreSQL postmaster process, and send it the SIGHUP signal:: @@ -163,7 +163,7 @@ The Dataverse software search index is powered by Solr. Supported Versions ================== -The Dataverse software has been tested with Solr version 9.4.1. Future releases in the 9.x series are likely to be compatible. Please get in touch (:ref:`support`) if you are having trouble with a newer version. +The Dataverse software has been tested with Solr version 9.8.0. Future releases in the 9.x series are likely to be compatible. Please get in touch (:ref:`support`) if you are having trouble with a newer version. Installing Solr =============== @@ -178,19 +178,19 @@ Become the ``solr`` user and then download and configure Solr:: su - solr cd /usr/local/solr - wget https://archive.apache.org/dist/solr/solr/9.4.1/solr-9.4.1.tgz - tar xvzf solr-9.4.1.tgz - cd solr-9.4.1 + wget https://archive.apache.org/dist/solr/solr/9.8.0/solr-9.8.0.tgz + tar xvzf solr-9.8.0.tgz + cd solr-9.8.0 cp -r server/solr/configsets/_default server/solr/collection1 You should already have a "dvinstall.zip" file that you downloaded from https://github.com/IQSS/dataverse/releases . Unzip it into ``/tmp``. Then copy the files into place:: - cp /tmp/dvinstall/schema*.xml /usr/local/solr/solr-9.4.1/server/solr/collection1/conf - cp /tmp/dvinstall/solrconfig.xml /usr/local/solr/solr-9.4.1/server/solr/collection1/conf + cp /tmp/dvinstall/schema*.xml /usr/local/solr/solr-9.8.0/server/solr/collection1/conf + cp /tmp/dvinstall/solrconfig.xml /usr/local/solr/solr-9.8.0/server/solr/collection1/conf Note: The Dataverse Project team has customized Solr to boost results that come from certain indexed elements inside the Dataverse installation, for example prioritizing results from Dataverse collections over Datasets. If you would like to remove this, edit your ``solrconfig.xml`` and remove the ```` element and its contents. If you have ideas about how this boosting could be improved, feel free to contact us through our Google Group https://groups.google.com/forum/#!forum/dataverse-dev . -A Dataverse installation requires a change to the ``jetty.xml`` file that ships with Solr. Edit ``/usr/local/solr/solr-9.4.1/server/etc/jetty.xml`` , increasing ``requestHeaderSize`` from ``8192`` to ``102400`` +A Dataverse installation requires a change to the ``jetty.xml`` file that ships with Solr. Edit ``/usr/local/solr/solr-9.8.0/server/etc/jetty.xml`` , increasing ``requestHeaderSize`` from ``8192`` to ``102400`` Solr will warn about needing to increase the number of file descriptors and max processes in a production environment but will still run with defaults. We have increased these values to the recommended levels by adding ulimit -n 65000 to the init script, and the following to ``/etc/security/limits.conf``:: @@ -209,7 +209,7 @@ Solr launches asynchronously and attempts to use the ``lsof`` binary to watch fo Finally, you need to tell Solr to create the core "collection1" on startup:: - echo "name=collection1" > /usr/local/solr/solr-9.4.1/server/solr/collection1/core.properties + echo "name=collection1" > /usr/local/solr/solr-9.8.0/server/solr/collection1/core.properties Dataverse collection ("dataverse") page uses Solr very heavily. On a busy instance this may cause the search engine to become the performance bottleneck, making these pages take increasingly longer to load, potentially affecting the overall performance of the application and/or causing Solr itself to crash. If this is observed on your instance, we recommend uncommenting the following lines in the ```` section of the ``solrconfig.xml`` file:: @@ -438,9 +438,9 @@ A scripted installation using Ansible is mentioned in the :doc:`/developers/make As root, download and install Counter Processor:: cd /usr/local - wget https://github.com/gdcc/counter-processor/archive/refs/tags/v1.05.tar.gz - tar xvfz v1.05.tar.gz - cd /usr/local/counter-processor-1.05 + wget https://github.com/gdcc/counter-processor/archive/refs/tags/v1.06.tar.gz + tar xvfz v1.06.tar.gz + cd /usr/local/counter-processor-1.06 Installing GeoLite Country Database =================================== @@ -451,7 +451,7 @@ The process required to sign up, download the database, and to configure automat As root, change to the Counter Processor directory you just created, download the GeoLite2-Country tarball from MaxMind, untar it, and copy the geoip database into place:: - + tar xvfz GeoLite2-Country.tar.gz cp GeoLite2-Country_*/GeoLite2-Country.mmdb maxmind_geoip @@ -461,12 +461,12 @@ Creating a counter User As root, create a "counter" user and change ownership of Counter Processor directory to this new user:: useradd counter - chown -R counter:counter /usr/local/counter-processor-1.05 + chown -R counter:counter /usr/local/counter-processor-1.06 Installing Counter Processor Python Requirements ================================================ -Counter Processor version 1.05 requires Python 3.7 or higher. This version of Python is available in many operating systems, and is purportedly available for RHEL7 or CentOS 7 via Red Hat Software Collections. Alternately, one may compile it from source. +Counter Processor version 1.06 requires Python 3.7 or higher. The following commands are intended to be run as root but we are aware that Pythonistas might prefer fancy virtualenv or similar setups. Pull requests are welcome to improve these steps! @@ -477,7 +477,7 @@ Install Python 3.9:: Install Counter Processor Python requirements:: python3.9 -m ensurepip - cd /usr/local/counter-processor-1.05 + cd /usr/local/counter-processor-1.06 pip3 install -r requirements.txt See the :doc:`/admin/make-data-count` section of the Admin Guide for how to configure and run Counter Processor. diff --git a/doc/sphinx-guides/source/qa/index.md b/doc/sphinx-guides/source/qa/index.md index f16cd1d38fc..623b93ef31b 100644 --- a/doc/sphinx-guides/source/qa/index.md +++ b/doc/sphinx-guides/source/qa/index.md @@ -1,6 +1,8 @@ # QA Guide ```{toctree} +:caption: "Contents:" +:maxdepth: 2 overview.md testing-approach.md testing-infrastructure.md diff --git a/doc/sphinx-guides/source/qa/test-automation.md b/doc/sphinx-guides/source/qa/test-automation.md index fe0d51f9174..73e7e570879 100644 --- a/doc/sphinx-guides/source/qa/test-automation.md +++ b/doc/sphinx-guides/source/qa/test-automation.md @@ -52,7 +52,7 @@ Go to the end of the log and then scroll up, looking for the failure. A failed A ``` TASK [dataverse : download payara zip] ***************************************** -fatal: [localhost]: FAILED! => {"changed": false, "dest": "/tmp/payara.zip", "elapsed": 10, "msg": "Request failed: ", "url": "https://nexus.payara.fish/repository/payara-community/fish/payara/distributions/payara/6.2024.6/payara-6.2024.6.zip"} +fatal: [localhost]: FAILED! => {"changed": false, "dest": "/tmp/payara.zip", "elapsed": 10, "msg": "Request failed: ", "url": "https://nexus.payara.fish/repository/payara-community/fish/payara/distributions/payara/6.2025.2/payara-6.2025.2.zip"} ``` In the example above, if Payara can't be downloaded, we're obviously going to have problems deploying Dataverse to it! diff --git a/doc/sphinx-guides/source/style/index.rst b/doc/sphinx-guides/source/style/index.rst index 0e93716e146..65a9f0b2b55 100755 --- a/doc/sphinx-guides/source/style/index.rst +++ b/doc/sphinx-guides/source/style/index.rst @@ -11,6 +11,7 @@ This style guide is meant to help developers implement clear and appropriate UI **Contents:** .. toctree:: + :maxdepth: 2 foundations patterns diff --git a/doc/sphinx-guides/source/user/account.rst b/doc/sphinx-guides/source/user/account.rst index bb73ff20dc7..936dca93c0b 100755 --- a/doc/sphinx-guides/source/user/account.rst +++ b/doc/sphinx-guides/source/user/account.rst @@ -165,6 +165,20 @@ Microsoft Azure AD, GitHub, and Google Log In You can also convert your Dataverse installation account to use authentication provided by GitHub, Microsoft, or Google. These options may be found in the "Other options" section of the log in page, and function similarly to how ORCID is outlined above. If you would like to convert your account away from using one of these services for log in, then you can follow the same steps as listed above for converting away from the ORCID log in. + +.. _orcid-integration: + +Linking ORCID with Your Account Profile +--------------------------------------- + +If you login using ORCID, Dataverse will add the link to your ORCID account in your account profile and, when you create datasets, will automatically add you, with your ORCID, as an author. + +If you login via other methods, you can add a link to your ORCID account as you create an account or later via the "Account Information" page. +As when using ORCID login, you will be redirected to the ORCID website to log in there and allow the connection with Dataverse. +Once you've done that, the link to your ORCID will be shown in the Account Information page and your ORCID will be added as your identifier when you create datasets (exactly the same as if you had logged in via ORCID). + +Note that the ability to login via ORCID (or other providers) and the ability to link to your ORCID profile are separate configuration options :doc:`available ` to Dataverse administrators. + .. _my-data: My Data diff --git a/doc/sphinx-guides/source/user/appendix.rst b/doc/sphinx-guides/source/user/appendix.rst index df9b6704209..96b426a483c 100755 --- a/doc/sphinx-guides/source/user/appendix.rst +++ b/doc/sphinx-guides/source/user/appendix.rst @@ -22,14 +22,15 @@ Supported Metadata Detailed below are what metadata schemas we support for Citation and Domain Specific Metadata in the Dataverse Project: -- Citation Metadata (`see .tsv `__): compliant with `DDI Lite `_, `DDI 2.5 Codebook `__, `DataCite 4.0 `__, and Dublin Core's `DCMI Metadata Terms `__ . Language field uses `ISO 639-1 `__ controlled vocabulary. -- Geospatial Metadata (`see .tsv `__): compliant with `DDI Lite `_, `DDI 2.5 Codebook `__, `DataCite 4.0 `__, and Dublin Core. Country / Nation field uses `ISO 3166-1 `_ controlled vocabulary. +- Citation Metadata (`see .tsv `__): compliant with `DDI Lite `_, `DDI 2.5 Codebook `__, `DataCite 4.5 `__, and Dublin Core's `DCMI Metadata Terms `__ . Language field uses `ISO 639-1 `__ controlled vocabulary. +- Geospatial Metadata (`see .tsv `__): compliant with `DDI Lite `_, `DDI 2.5 Codebook `__, `DataCite 4.5 `__, and Dublin Core. Country / Nation field uses `ISO 3166-1 `_ controlled vocabulary. - Social Science & Humanities Metadata (`see .tsv `__): compliant with `DDI Lite `_, `DDI 2.5 Codebook `__, and Dublin Core. - Astronomy and Astrophysics Metadata (`see .tsv `__): These metadata elements can be mapped/exported to the International Virtual Observatory Allianceโ€™s (IVOA) `VOResource Schema format `__ and is based on `Virtual Observatory (VO) Discovery and Provenance Metadata `__. - Life Sciences Metadata (`see .tsv `__): based on `ISA-Tab Specification `__, along with controlled vocabulary from subsets of the `OBI Ontology `__ and the `NCBI Taxonomy for Organisms `__. - Journal Metadata (`see .tsv `__): based on the `Journal Archiving and Interchange Tag Set, version 1.2 `__. +- 3D Objects Metadata (`see .tsv `__). Experimental Metadata ~~~~~~~~~~~~~~~~~~~~~ @@ -38,6 +39,7 @@ Unlike supported metadata, experimental metadata is not enabled by default in a - `CodeMeta Software Metadata `__: based on the `CodeMeta Software Metadata Schema, version 2.0 `__ (`see .tsv version `__) - Computational Workflow Metadata (`see .tsv `__): adapted from `Bioschemas Computational Workflow Profile, version 1.0 `__ and `Codemeta `__. +- Archival Metadata (`see .tsv `__): Enables repositories to register metadata relating to the potential archiving of the dataset at a depositor archive, whether that be your own institutional archive or an external archive, i.e. a historical archive. Please note: these custom metadata schemas are not included in the Solr schema for indexing by default, you will need to add them as necessary for your custom metadata blocks. See "Update the Solr Schema" in :doc:`../admin/metadatacustomization`. diff --git a/doc/sphinx-guides/source/user/dataset-management.rst b/doc/sphinx-guides/source/user/dataset-management.rst index b3a14554b40..37656d1e243 100755 --- a/doc/sphinx-guides/source/user/dataset-management.rst +++ b/doc/sphinx-guides/source/user/dataset-management.rst @@ -43,6 +43,8 @@ Additional formats can be enabled. See :ref:`inventory-of-external-exporters` in Each of these metadata exports contains the metadata of the most recently published version of the dataset. +For each dataset, links to each enabled metadata format are available programmatically via Signposting. For details, see :ref:`discovery-sign-posting` in the Admin Guide and :ref:`signposting-api` in the API Guide. + .. _adding-new-dataset: Adding a New Dataset @@ -50,8 +52,10 @@ Adding a New Dataset #. Navigate to the Dataverse collection in which you want to add a dataset. #. Click on the "Add Data" button and select "New Dataset" in the dropdown menu. **Note:** If you are on the root Dataverse collection, your My Data page or click the "Add Data" link in the navbar, the dataset you create will be hosted in the root Dataverse collection. You can change this by selecting another Dataverse collection you have proper permissions to create datasets in, from the Host Dataverse collection dropdown in the create dataset form. This option to choose will not be available after you create the dataset. -#. To quickly get started, enter at minimum all the required fields with an asterisk (e.g., the Dataset Title, Author Name, - Description Text, Point of Contact Email, and Subject) to get a Data Citation with a DOI. +#. To quickly get started, enter at minimum all the required fields with an asterisk (e.g., the Dataset Title, Author Name, Description Text, Point of Contact Email, and Subject) to get a Data Citation with a DOI. + + #. When entering author identifiers, select the type from the dropdown (e.g. "ORCID") and under "Identifier" enter the full URL (e.g. "https://orcid.org/0000-0002-1825-0097") for identifiers that have a URL form. The shorter form of the unique identifier (e.g. "0000-0002-1825-0097") can also be entered, but URL form is preferred when available. + #. Scroll down to the "Files" section and click on "Select Files to Add" to add all the relevant files to your Dataset. You can also upload your files directly from your Dropbox. **Tip:** You can drag and drop or select multiple files at a time from your desktop directly into the upload widget. Your files will appear below the "Select Files to Add" button where you can add a @@ -169,7 +173,7 @@ Certain file types in the Dataverse installation are supported by additional fun File Previews ------------- -Dataverse installations can add previewers for common file types uploaded by their research communities. The previews appear on the file page. If a preview tool for a specific file type is available, the preview will be created and will display automatically, after terms have been agreed to or a guestbook entry has been made, if necessary. File previews are not available for restricted files unless they are being accessed using a Preview URL. See also :ref:`previewUrl`. +Dataverse installations can add previewers for common file types uploaded by their research communities. The previews appear on the file page. If a preview tool for a specific file type is available, the preview will be created and will display automatically, after terms have been agreed to or a guestbook entry has been made, if necessary. File previews are not available for restricted files unless they are being accessed using a Preview URL. See also :ref:`previewUrl`. When the dataset license is not the default license, users will be prompted to accept the license/data use agreement before the preview is shown. See also :ref:`license-terms`. Previewers are available for the following file types: @@ -572,7 +576,26 @@ When you access a dataset's file-level permissions page, you will see two sectio Data Provenance =============== -Data Provenance is a record of where your data came from and how it reached its current form. It describes the origin of a data file, any transformations that have been made to that file, and any persons or organizations associated with that file. A data file's provenance can aid in reproducibility and compliance with legal regulations. The Dataverse Software can help you keep track of your data's provenance. Currently, the Dataverse Software only makes provenance information available to those who have edit permissions on your dataset, but in the future we plan to expand this feature to make provenance information available to the public. +Dataset-Level +------------- +When configured, the Dataverse software can allow data depositors, curators, and administrators +to provide information about why a new version of a dataset was created and/or how its contents +differ from a prior version. These users can add an optional "Version Note" to a draft dataset +version in the dataset page/versions tab or during publication. This information is publicly +available via the user interface (dataset page/versions tab), API, and in metadata exports +(including the DataCite, JSON, DDI, and OAI_ORE exports). + +File-Level +---------- + +Data Provenance is a record of where your data came from and how it reached its current form. +It describes the origin of a data file, any transformations that have been made to that file, +and any persons or organizations associated with that file. A data file's provenance can aid in +reproducibility and compliance with legal regulations. When configured to support provenance, +the Dataverse Software can help you keep track of your data's provenance. Currently, the Dataverse +Software only makes provenance information available to those who have edit permissions on your +dataset, but in the future we plan to expand this feature to make provenance information available +to the public. .. COMMENTED OUT UNTIL PROV FILE DOWNLOAD IS ADDED: , and make it available to those who need it. @@ -681,17 +704,26 @@ If you have a Contributor role (can edit metadata, upload files, and edit files, Preview URL to Review Unpublished Dataset ========================================= -Creating a Preview URL for your dataset allows you to share your dataset (for viewing and downloading of files) before it is published to a wide group of individuals who may not have a user account on the Dataverse installation. Anyone you send the Preview URL to will not have to log into the Dataverse installation to view the dataset. +Creating a Preview URL for a draft version of your dataset allows you to share your dataset (for viewing and downloading of files) before it is published to a wide group of individuals who may not have a user account on the Dataverse installation. Anyone you send the Preview URL to will not have to log into the Dataverse installation to view the unpublished dataset. Once a dataset has been published you may create new General Preview URLs for subsequent draft versions, but the Anonymous Preview URL will no longer be available. -**Note:** To create a Preview URL, you must have the *ManageDatasetPermissions* permission for your dataset, usually given by the :ref:`roles ` *Curator* or *Administrator*. +**Note:** To create a Preview URL, you must have the *ManageDatasetPermissions* permission for your draft dataset, usually given by the :ref:`roles ` *Curator* or *Administrator*. #. Go to your unpublished dataset #. Select the โ€œEditโ€ button #. Select โ€œPreview URLโ€ in the dropdown menu -#. In the pop-up select โ€œCreate General Preview URLโ€ or "Create URL for Anonymized Access". The latter supports anonymous review by removing author names and other potentially identifying information from citations, version history tables, and some metadata fields (as configured by the administrator). +#. In the pop-up select โ€œCreate General Preview URLโ€ or "Create Anonymous Preview URL". The latter supports anonymous review by removing author names and other potentially identifying information from citations, version history tables, and some metadata fields (as configured by the administrator). #. Copy the Preview URL which has been created for this dataset and it can now be shared with anyone you wish to have access to view or download files in your unpublished dataset. To disable a Preview URL and to revoke access, follow the same steps as above until step #3 when you return to the popup, click the โ€œDisable Preview URLโ€ button. + +**Note:** Before distributing an anonymized Preview URL it is recommended that you view the dataset as a potential user to verify that the metadata available does not reveal authorship, etc. + +#. Create Anonymous Preview URL for your unpublished dataset via the Preview URL popup from Edit Dataset button +#. Copy the Anonymous Preview URL to your clipboard +#. Log out of Dataverse application +#. Open the dataset using the Anonymous Preview URL you plan to distribute to view it as a reviewer would. +#. It may be necessary for you to further edit your draft dataset's metadata to remove identifying items before you distribute the Anonymous Preview URL + Note that only one Preview URL (normal or with anonymized access) can be configured per dataset at a time. Embargoes @@ -790,13 +822,15 @@ If you deaccession the most recently published version of the dataset but not al Dataset Types ============= +.. note:: Development of the dataset types feature is ongoing. Please see https://github.com/IQSS/dataverse-pm/issues/307 for details. + Out of the box, all datasets have a dataset type of "dataset". Superusers can add additional types such as "software" or "workflow" using the :ref:`api-add-dataset-type` API endpoint. Once more than one type appears in search results, a facet called "Dataset Type" will appear allowing you to filter down to a certain type. If your installation is configured to use DataCite as a persistent ID (PID) provider, the appropriate type ("Dataset", "Software", "Workflow") will be sent to DataCite when the dataset is published for those three types. -Currently, the dataset type can only be specified via API and only when the dataset is created. For details, see the following sections of the API guide: +Currently, specifying a type for a dataset can only be done via API and only when the dataset is created. The type can't currently be changed afterward. For details, see the following sections of the API guide: - :ref:`api-create-dataset-with-type` (Native API) - :ref:`api-semantic-create-dataset-with-type` (Semantic API) @@ -804,7 +838,7 @@ Currently, the dataset type can only be specified via API and only when the data Dataset types can be listed, added, or deleted via API. See :ref:`api-dataset-types` in the API Guide for more. -Development of the dataset types feature is ongoing. Please see https://github.com/IQSS/dataverse/issues/10489 for details. +Dataset types can be linked with metadata blocks to make fields from those blocks available when datasets of that type are created or edited. See :ref:`api-link-dataset-type` and :ref:`list-metadata-blocks-for-a-collection` for details. .. |image1| image:: ./img/DatasetDiagram.png :class: img-responsive diff --git a/doc/sphinx-guides/source/user/find-use-data.rst b/doc/sphinx-guides/source/user/find-use-data.rst index 4bf45774b53..8e65108d680 100755 --- a/doc/sphinx-guides/source/user/find-use-data.rst +++ b/doc/sphinx-guides/source/user/find-use-data.rst @@ -96,7 +96,7 @@ Files can be organized in one or more folders (directories) within a dataset. If Cite Data --------- -You can find the citation for the dataset at the top of the dataset page in a blue box. Additionally, there is a Cite Data button that offers the option to download the citation as EndNote XML, RIS Format, or BibTeX Format. +You can find the citation for the dataset at the top of the dataset page in a blue box. Additionally, there is a Cite Data button that offers the option to download the citation as EndNote XML, RIS Format, or BibTeX Format, or to cut/paste the citation in any of the 1000+ standard journal/society/other formats defined via the `Citation Style Language `_. .. _download_files: diff --git a/doc/sphinx-guides/source/user/index.rst b/doc/sphinx-guides/source/user/index.rst index 857bd27ca22..cd6ccdbd421 100755 --- a/doc/sphinx-guides/source/user/index.rst +++ b/doc/sphinx-guides/source/user/index.rst @@ -9,6 +9,7 @@ User Guide **Contents:** .. toctree:: + :maxdepth: 2 account find-use-data diff --git a/doc/sphinx-guides/source/versions.rst b/doc/sphinx-guides/source/versions.rst index 9d640bd22bd..cd19837dff1 100755 --- a/doc/sphinx-guides/source/versions.rst +++ b/doc/sphinx-guides/source/versions.rst @@ -7,7 +7,8 @@ Dataverse Software Documentation Versions This list provides a way to refer to the documentation for previous and future versions of the Dataverse Software. In order to learn more about the updates delivered from one version to another, visit the `Releases `__ page in our GitHub repo. - pre-release `HTML (not final!) `__ and `PDF (experimental!) `__ built from the :doc:`develop ` branch :doc:`(how to contribute!) ` -- 6.5 +- 6.6 +- `6.5 `__ - `6.4 `__ - `6.3 `__ - `6.2 `__ diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml index c8515f43136..0de90f7ec2a 100644 --- a/docker-compose-dev.yml +++ b/docker-compose-dev.yml @@ -17,6 +17,8 @@ services: SKIP_DEPLOY: "${SKIP_DEPLOY}" DATAVERSE_JSF_REFRESH_PERIOD: "1" DATAVERSE_FEATURE_API_BEARER_AUTH: "1" + DATAVERSE_FEATURE_INDEX_HARVESTED_METADATA_SOURCE: "1" + DATAVERSE_FEATURE_API_BEARER_AUTH_PROVIDE_MISSING_CLAIMS: "1" DATAVERSE_MAIL_SYSTEM_EMAIL: "dataverse@localhost" DATAVERSE_MAIL_MTA_HOST: "smtp" DATAVERSE_AUTH_OIDC_ENABLED: "1" @@ -89,6 +91,8 @@ services: - dev networks: - dataverse + volumes: + - ./docker-dev-volumes/solr/data:/var/solr dev_dv_initializer: container_name: "dev_dv_initializer" diff --git a/docker/compose/demo/compose.yml b/docker/compose/demo/compose.yml index 60ed130612e..3c8bee29e3e 100644 --- a/docker/compose/demo/compose.yml +++ b/docker/compose/demo/compose.yml @@ -20,12 +20,12 @@ services: -Ddataverse.files.file1.type=file -Ddataverse.files.file1.label=Filesystem -Ddataverse.files.file1.directory=${STORAGE_DIR}/store - -Ddataverse.pid.providers=fake - -Ddataverse.pid.default-provider=fake - -Ddataverse.pid.fake.type=FAKE - -Ddataverse.pid.fake.label=FakeDOIProvider - -Ddataverse.pid.fake.authority=10.5072 - -Ddataverse.pid.fake.shoulder=FK2/ + -Ddataverse.pid.providers=perma1 + -Ddataverse.pid.default-provider=perma1 + -Ddataverse.pid.perma1.type=perma + -Ddataverse.pid.perma1.label=Perma1 + -Ddataverse.pid.perma1.authority=DV + -Ddataverse.pid.perma1.permalink.separator=/ #-Ddataverse.lang.directory=/dv/lang ports: - "8080:8080" # HTTP (Dataverse Application) @@ -134,7 +134,7 @@ services: solr: container_name: "solr" hostname: "solr" - image: solr:9.4.1 + image: solr:9.8.0 depends_on: - solr_initializer restart: on-failure diff --git a/local_lib/io/gdcc/xoai-common/5.3.2-local/xoai-common-5.3.2-local.jar b/local_lib/io/gdcc/xoai-common/5.3.2-local/xoai-common-5.3.2-local.jar new file mode 100644 index 00000000000..5047caacc5b Binary files /dev/null and b/local_lib/io/gdcc/xoai-common/5.3.2-local/xoai-common-5.3.2-local.jar differ diff --git a/local_lib/io/gdcc/xoai-common/5.3.2-local/xoai-common-5.3.2-local.pom b/local_lib/io/gdcc/xoai-common/5.3.2-local/xoai-common-5.3.2-local.pom new file mode 100644 index 00000000000..b838f27c671 --- /dev/null +++ b/local_lib/io/gdcc/xoai-common/5.3.2-local/xoai-common-5.3.2-local.pom @@ -0,0 +1,82 @@ + + + + + + xoai + io.gdcc + 5.3.2-local + + 4.0.0 + + XOAI Commons + xoai-common + OAI-PMH base functionality used for both data and service providers. + + + + jakarta.xml.bind + jakarta.xml.bind-api + + + org.hamcrest + hamcrest + + compile + + + io.gdcc + xoai-xmlio + + + org.codehaus.woodstox + stax2-api + + + + com.fasterxml.woodstox + woodstox-core + runtime + true + + + + + org.junit.jupiter + junit-jupiter + test + + + org.xmlunit + xmlunit-core + test + + + org.xmlunit + xmlunit-matchers + test + + + org.openjdk.jmh + jmh-core + 1.37 + test + + + org.openjdk.jmh + jmh-generator-annprocess + 1.37 + test + + + diff --git a/local_lib/io/gdcc/xoai-data-provider/5.3.2-local/xoai-data-provider-5.3.2-local.jar b/local_lib/io/gdcc/xoai-data-provider/5.3.2-local/xoai-data-provider-5.3.2-local.jar new file mode 100644 index 00000000000..cf6fb9c7d0c Binary files /dev/null and b/local_lib/io/gdcc/xoai-data-provider/5.3.2-local/xoai-data-provider-5.3.2-local.jar differ diff --git a/local_lib/io/gdcc/xoai-data-provider/5.3.2-local/xoai-data-provider-5.3.2-local.pom b/local_lib/io/gdcc/xoai-data-provider/5.3.2-local/xoai-data-provider-5.3.2-local.pom new file mode 100644 index 00000000000..e17ae894e98 --- /dev/null +++ b/local_lib/io/gdcc/xoai-data-provider/5.3.2-local/xoai-data-provider-5.3.2-local.pom @@ -0,0 +1,72 @@ + + + + + + xoai + io.gdcc + 5.3.2-local + + + 4.0.0 + + XOAI Data Provider + xoai-data-provider + OAI-PMH data provider implementation. Use it to build an OAI-PMH endpoint, providing your data records as harvestable resources. + + + + + org.apache.maven.plugins + maven-jar-plugin + + + + test-jar + + + + + + + + + + io.gdcc + xoai-common + ${project.version} + + + org.slf4j + slf4j-api + + + + + org.junit.jupiter + junit-jupiter + test + + + org.xmlunit + xmlunit-core + test + + + org.xmlunit + xmlunit-matchers + test + + + org.slf4j + slf4j-simple + test + + + diff --git a/local_lib/io/gdcc/xoai-service-provider/5.3.2-local/xoai-service-provider-5.3.2-local.jar b/local_lib/io/gdcc/xoai-service-provider/5.3.2-local/xoai-service-provider-5.3.2-local.jar new file mode 100644 index 00000000000..201eecc061c Binary files /dev/null and b/local_lib/io/gdcc/xoai-service-provider/5.3.2-local/xoai-service-provider-5.3.2-local.jar differ diff --git a/local_lib/io/gdcc/xoai-service-provider/5.3.2-local/xoai-service-provider-5.3.2-local.pom b/local_lib/io/gdcc/xoai-service-provider/5.3.2-local/xoai-service-provider-5.3.2-local.pom new file mode 100644 index 00000000000..aa5a65824d7 --- /dev/null +++ b/local_lib/io/gdcc/xoai-service-provider/5.3.2-local/xoai-service-provider-5.3.2-local.pom @@ -0,0 +1,65 @@ + + + + + + io.gdcc + xoai + 5.3.2-local + + 4.0.0 + + XOAI Service Provider + xoai-service-provider + OAI-PMH service provider implementation. Use it as a harvesting client to read remote repositories. + + + + io.gdcc + xoai-common + ${project.version} + + + io.gdcc + xoai-xmlio + ${project.version} + + + + org.slf4j + slf4j-api + + + + + io.gdcc + xoai-data-provider + ${project.version} + test + + + io.gdcc + xoai-data-provider + ${project.version} + test-jar + test + + + org.junit.jupiter + junit-jupiter + test + + + org.slf4j + slf4j-simple + test + + + + diff --git a/local_lib/io/gdcc/xoai-xmlio/5.3.2-local/xoai-xmlio-5.3.2-local.jar b/local_lib/io/gdcc/xoai-xmlio/5.3.2-local/xoai-xmlio-5.3.2-local.jar new file mode 100644 index 00000000000..eebdac42f00 Binary files /dev/null and b/local_lib/io/gdcc/xoai-xmlio/5.3.2-local/xoai-xmlio-5.3.2-local.jar differ diff --git a/local_lib/io/gdcc/xoai-xmlio/5.3.2-local/xoai-xmlio-5.3.2-local.pom b/local_lib/io/gdcc/xoai-xmlio/5.3.2-local/xoai-xmlio-5.3.2-local.pom new file mode 100644 index 00000000000..50b13e50925 --- /dev/null +++ b/local_lib/io/gdcc/xoai-xmlio/5.3.2-local/xoai-xmlio-5.3.2-local.pom @@ -0,0 +1,63 @@ + + 4.0.0 + + + io.gdcc + xoai + 5.3.2-local + + + xoai-xmlio + jar + XOAI XML IO Commons + Basic XML IO routines used for XOAI OAI-PMH implementation. Forked from obsolete Lyncode sources. + + + + The Apache Software License, Version 2.0 + https://www.apache.org/licenses/LICENSE-2.0.txt + repo + + + + + + org.codehaus.woodstox + stax2-api + + + + com.fasterxml.woodstox + woodstox-core + runtime + true + + + + + org.hamcrest + hamcrest + + + + + org.xmlunit + xmlunit-core + test + + + org.xmlunit + xmlunit-matchers + test + + + org.junit.jupiter + junit-jupiter + test + + + diff --git a/local_lib/io/gdcc/xoai/5.3.2-local/xoai-5.3.2-local.pom b/local_lib/io/gdcc/xoai/5.3.2-local/xoai-5.3.2-local.pom new file mode 100644 index 00000000000..2ebbd698d98 --- /dev/null +++ b/local_lib/io/gdcc/xoai/5.3.2-local/xoai-5.3.2-local.pom @@ -0,0 +1,235 @@ + + + + 4.0.0 + pom + + + io.gdcc + parent + 0.10.2 + + + + xoai-common + xoai-data-provider + xoai-service-provider + xoai-xmlio + report + xoai-data-provider-tck + + + xoai + 5.3.2-local + + XOAI : OAI-PMH Java Toolkit + + An OAI-PMH data and/or service provider implementation, integration ready for your service. + https://github.com/${project.github.org}/${project.github.repo} + + + 11 + xoai + true + + + 4.0.1 + 4.0.4 + 4.2.2 + 7.0.0 + + + 10.0.4 + + + + + DuraSpace BSD License + https://raw.github.com/DSpace/DSpace/master/LICENSE + repo + + A BSD 3-Clause license for the DSpace codebase. + + + + + + + + + com.diffplug.spotless + spotless-maven-plugin + ${spotless.version} + + + origin/branch-5.0 + + + + + + *.md + .gitignore + + + + + + true + 4 + + + + + + + + + + + + + 1.15.0 + + true + + + + + + + + + + + + + + + + + jakarta.xml.bind + jakarta.xml.bind-api + ${jakarta.jaxb.version} + + + + com.sun.xml.bind + jaxb-impl + ${jakarta.jaxb-impl.version} + runtime + true + + + + + com.fasterxml.woodstox + woodstox-core + ${woodstox.version} + + + org.codehaus.woodstox + stax2-api + ${stax2.api.version} + + + + io.gdcc + xoai-xmlio + ${project.version} + + + + + + + Oliver Bertuch + https://github.com/poikilotherm + xoai-lib@gdcc.io + Forschungszentrum Jรผlich GmbH + https://www.fz-juelich.de/en/zb + + + DSpace @ Lyncode + dspace@lyncode.com + Lyncode + http://www.lyncode.com + + + + + + coverage + + ${maven.multiModuleProjectDirectory}/report/target/site/jacoco-aggregate/jacoco.xml + + + + benchmark + + + + org.apache.maven.plugins + maven-surefire-plugin + ${maven-surefire-plugin.version} + + ${skipUT} + **/*Benchmark + + true + + + + + + + + owasp + + + + + org.owasp + dependency-check-maven + ${dependency-check-maven.version} + + 7 + true + true + + SARIF + owaspSuppression.xml + + + + + check + + + + + + + + + + diff --git a/modules/container-base/src/main/docker/Dockerfile b/modules/container-base/src/main/docker/Dockerfile index 802db62e5e4..ed13b77c449 100644 --- a/modules/container-base/src/main/docker/Dockerfile +++ b/modules/container-base/src/main/docker/Dockerfile @@ -199,6 +199,13 @@ RUN < - 6.5 + 6.6 17 UTF-8 @@ -148,9 +148,9 @@ -Duser.timezone=${project.timezone} -Dfile.encoding=${project.build.sourceEncoding} -Duser.language=${project.language} -Duser.region=${project.region} - 6.2024.6 + 6.2025.2 42.7.4 - 9.4.1 + 9.8.0 1.12.748 26.30.0 @@ -164,7 +164,8 @@ 4.4.14 - 5.2.0 + + 5.3.2-local 1.19.7 @@ -374,6 +375,12 @@ + + + dvn.private + Local repository for hosting jars not available from network repositories. + file://${project.basedir}/local_lib + payara-nexus-artifacts Payara Nexus Artifacts diff --git a/pom.xml b/pom.xml index 5ecbd7059c1..7b15a97ee73 100644 --- a/pom.xml +++ b/pom.xml @@ -29,8 +29,8 @@ 1.2.18.4 10.19.0 1.20.1 - 5.2.1 - 2.9.1 + 5.2.5 + 2.9.2 5.5.3 Dataverse API @@ -51,6 +51,16 @@ org.apache.abdera abdera-core 1.1.3 + + + org.apache.geronimo.specs + geronimo-stax-api_1.0_spec + + + org.apache.james + apache-mime4j-core + + org.apache.abdera @@ -125,18 +135,36 @@ io.gdcc sword2-server 2.0.0 + + + xml-apis + xml-apis + + org.apache.abdera abdera-core + + + org.apache.geronimo.specs + geronimo-stax-api_1.0_spec + + org.apache.abdera abdera-i18n + + + org.apache.geronimo.specs + geronimo-stax-api_1.0_spec + + @@ -147,12 +175,12 @@ com.apicatalog titanium-json-ld - 1.3.2 + 1.4.0 com.google.code.gson gson - 2.8.9 + 2.9.1 compile @@ -168,11 +196,9 @@ provided - - org.everit.json - org.everit.json.schema - 1.5.1 + com.github.erosb + everit-json-schema + 1.14.1 org.mindrot @@ -247,7 +273,7 @@ org.eclipse.parsson jakarta.json - provided + test @@ -335,7 +361,7 @@ org.apache.solr solr-solrj - 9.4.1 + 9.8.0 colt @@ -406,7 +432,7 @@ com.github.jai-imageio jai-imageio-core - 1.3.1 + 1.4.0 org.ocpsoft.rewrite @@ -466,13 +492,23 @@ com.nimbusds oauth2-oidc-sdk - 10.13.2 + 11.22.1 com.github.ben-manes.caffeine caffeine 3.1.8 + + + javax.xml.stream + stax-api + + + stax + stax-api + + @@ -490,7 +526,7 @@ com.google.auto.service auto-service - 1.0-rc2 + 1.1.1 true jar @@ -559,6 +595,12 @@ org.apache.tika tika-parsers-standard-package ${tika.version} + + + xml-apis + xml-apis + + @@ -609,6 +651,21 @@ javax.cache cache-api + + de.undercouch + citeproc-java + 3.1.0 + + + org.citationstyles + styles + 24.3 + + + org.citationstyles + locales + 24.3 + org.junit.jupiter diff --git a/scripts/api/data/dataset-create-new-additional-default-fields.json b/scripts/api/data/dataset-create-new-additional-default-fields.json new file mode 100644 index 00000000000..30d6bde4355 --- /dev/null +++ b/scripts/api/data/dataset-create-new-additional-default-fields.json @@ -0,0 +1,1533 @@ +{ + "datasetVersion": { + "license": { + "name": "CC0 1.0", + "uri": "http://creativecommons.org/publicdomain/zero/1.0" + }, + "metadataBlocks": { + "citation": { + "displayName": "Citation Metadata", + "fields": [ + { + "typeName": "title", + "multiple": false, + "typeClass": "primitive", + "value": "Replication Data for: Title" + }, + { + "typeName": "subtitle", + "multiple": false, + "typeClass": "primitive", + "value": "Subtitle" + }, + { + "typeName": "alternativeTitle", + "multiple": true, + "typeClass": "primitive", + "value": ["Alternative Title"] + }, + { + "typeName": "alternativeURL", + "multiple": false, + "typeClass": "primitive", + "value": "http://AlternativeURL.org" + }, + { + "typeName": "otherId", + "multiple": true, + "typeClass": "compound", + "value": [ + { + "otherIdAgency": { + "typeName": "otherIdAgency", + "multiple": false, + "typeClass": "primitive", + "value": "OtherIDAgency1" + }, + "otherIdValue": { + "typeName": "otherIdValue", + "multiple": false, + "typeClass": "primitive", + "value": "OtherIDIdentifier1" + } + }, + { + "otherIdAgency": { + "typeName": "otherIdAgency", + "multiple": false, + "typeClass": "primitive", + "value": "OtherIDAgency2" + }, + "otherIdValue": { + "typeName": "otherIdValue", + "multiple": false, + "typeClass": "primitive", + "value": "OtherIDIdentifier2" + } + } + ] + }, + { + "typeName": "author", + "multiple": true, + "typeClass": "compound", + "value": [ + { + "authorName": { + "typeName": "authorName", + "multiple": false, + "typeClass": "primitive", + "value": "LastAuthor1, FirstAuthor1" + }, + "authorAffiliation": { + "typeName": "authorAffiliation", + "multiple": false, + "typeClass": "primitive", + "value": "AuthorAffiliation1" + }, + "authorIdentifierScheme": { + "typeName": "authorIdentifierScheme", + "multiple": false, + "typeClass": "controlledVocabulary", + "value": "ORCID" + }, + "authorIdentifier": { + "typeName": "authorIdentifier", + "multiple": false, + "typeClass": "primitive", + "value": "AuthorIdentifier1" + } + }, + { + "authorName": { + "typeName": "authorName", + "multiple": false, + "typeClass": "primitive", + "value": "LastAuthor2, FirstAuthor2" + }, + "authorAffiliation": { + "typeName": "authorAffiliation", + "multiple": false, + "typeClass": "primitive", + "value": "AuthorAffiliation2" + }, + "authorIdentifierScheme": { + "typeName": "authorIdentifierScheme", + "multiple": false, + "typeClass": "controlledVocabulary", + "value": "ISNI" + }, + "authorIdentifier": { + "typeName": "authorIdentifier", + "multiple": false, + "typeClass": "primitive", + "value": "AuthorIdentifier2" + } + } + ] + }, + { + "typeName": "datasetContact", + "multiple": true, + "typeClass": "compound", + "value": [ + { + "datasetContactName": { + "typeName": "datasetContactName", + "multiple": false, + "typeClass": "primitive", + "value": "LastContact1, FirstContact1" + }, + "datasetContactAffiliation": { + "typeName": "datasetContactAffiliation", + "multiple": false, + "typeClass": "primitive", + "value": "ContactAffiliation1" + }, + "datasetContactEmail": { + "typeName": "datasetContactEmail", + "multiple": false, + "typeClass": "primitive", + "value": "ContactEmail1@mailinator.com" + } + }, + { + "datasetContactName": { + "typeName": "datasetContactName", + "multiple": false, + "typeClass": "primitive", + "value": "LastContact2, FirstContact2" + }, + "datasetContactAffiliation": { + "typeName": "datasetContactAffiliation", + "multiple": false, + "typeClass": "primitive", + "value": "ContactAffiliation2" + }, + "datasetContactEmail": { + "typeName": "datasetContactEmail", + "multiple": false, + "typeClass": "primitive", + "value": "ContactEmail2@mailinator.com" + } + } + ] + }, + { + "typeName": "dsDescription", + "multiple": true, + "typeClass": "compound", + "value": [ + { + "dsDescriptionValue": { + "typeName": "dsDescriptionValue", + "multiple": false, + "typeClass": "primitive", + "value": "DescriptionText1" + }, + "dsDescriptionDate": { + "typeName": "dsDescriptionDate", + "multiple": false, + "typeClass": "primitive", + "value": "1000-01-01" + } + }, + { + "dsDescriptionValue": { + "typeName": "dsDescriptionValue", + "multiple": false, + "typeClass": "primitive", + "value": "DescriptionText2" + }, + "dsDescriptionDate": { + "typeName": "dsDescriptionDate", + "multiple": false, + "typeClass": "primitive", + "value": "1000-02-02" + } + } + ] + }, + { + "typeName": "subject", + "multiple": true, + "typeClass": "controlledVocabulary", + "value": [ + "Agricultural Sciences", + "Business and Management", + "Engineering", + "Law" + ] + }, + { + "typeName": "keyword", + "multiple": true, + "typeClass": "compound", + "value": [ + { + "keywordValue": { + "typeName": "keywordValue", + "multiple": false, + "typeClass": "primitive", + "value": "KeywordTerm1" + }, + "keywordTermURI": { + "typeName": "keywordTermURI", + "multiple": false, + "typeClass": "primitive", + "value": "http://keywordTermURI1.org" + }, + "keywordVocabulary": { + "typeName": "keywordVocabulary", + "multiple": false, + "typeClass": "primitive", + "value": "KeywordVocabulary1" + }, + "keywordVocabularyURI": { + "typeName": "keywordVocabularyURI", + "multiple": false, + "typeClass": "primitive", + "value": "http://KeywordVocabularyURL1.org" + } + }, + { + "keywordValue": { + "typeName": "keywordValue", + "multiple": false, + "typeClass": "primitive", + "value": "KeywordTerm2" + }, + "keywordTermURI": { + "typeName": "keywordTermURI", + "multiple": false, + "typeClass": "primitive", + "value": "http://keywordTermURI2.org" + }, + "keywordVocabulary": { + "typeName": "keywordVocabulary", + "multiple": false, + "typeClass": "primitive", + "value": "KeywordVocabulary2" + }, + "keywordVocabularyURI": { + "typeName": "keywordVocabularyURI", + "multiple": false, + "typeClass": "primitive", + "value": "http://KeywordVocabularyURL2.org" + } + } + ] + }, + { + "typeName": "topicClassification", + "multiple": true, + "typeClass": "compound", + "value": [ + { + "topicClassValue": { + "typeName": "topicClassValue", + "multiple": false, + "typeClass": "primitive", + "value": "Topic Classification Term1" + }, + "topicClassVocab": { + "typeName": "topicClassVocab", + "multiple": false, + "typeClass": "primitive", + "value": "Topic Classification Vocab1" + }, + "topicClassVocabURI": { + "typeName": "topicClassVocabURI", + "multiple": false, + "typeClass": "primitive", + "value": "https://TopicClassificationURL1.com" + } + }, + { + "topicClassValue": { + "typeName": "topicClassValue", + "multiple": false, + "typeClass": "primitive", + "value": "Topic Classification Term2" + }, + "topicClassVocab": { + "typeName": "topicClassVocab", + "multiple": false, + "typeClass": "primitive", + "value": "Topic Classification Vocab2" + }, + "topicClassVocabURI": { + "typeName": "topicClassVocabURI", + "multiple": false, + "typeClass": "primitive", + "value": "https://TopicClassificationURL2.com" + } + } + ] + }, + { + "typeName": "publication", + "multiple": true, + "typeClass": "compound", + "value": [ + { + "publicationRelationType" : { + "typeName" : "publicationRelationType", + "multiple" : false, + "typeClass" : "controlledVocabulary", + "value" : "IsSupplementTo" + }, + "publicationCitation": { + "typeName": "publicationCitation", + "multiple": false, + "typeClass": "primitive", + "value": "RelatedPublicationCitation1" + }, + "publicationIDType": { + "typeName": "publicationIDType", + "multiple": false, + "typeClass": "controlledVocabulary", + "value": "ark" + }, + "publicationIDNumber": { + "typeName": "publicationIDNumber", + "multiple": false, + "typeClass": "primitive", + "value": "RelatedPublicationIDNumber1" + }, + "publicationURL": { + "typeName": "publicationURL", + "multiple": false, + "typeClass": "primitive", + "value": "http://RelatedPublicationURL1.org" + } + }, + { + "publicationCitation": { + "typeName": "publicationCitation", + "multiple": false, + "typeClass": "primitive", + "value": "RelatedPublicationCitation2" + }, + "publicationIDType": { + "typeName": "publicationIDType", + "multiple": false, + "typeClass": "controlledVocabulary", + "value": "arXiv" + }, + "publicationIDNumber": { + "typeName": "publicationIDNumber", + "multiple": false, + "typeClass": "primitive", + "value": "RelatedPublicationIDNumber2" + }, + "publicationURL": { + "typeName": "publicationURL", + "multiple": false, + "typeClass": "primitive", + "value": "http://RelatedPublicationURL2.org" + } + } + ] + }, + { + "typeName": "notesText", + "multiple": false, + "typeClass": "primitive", + "value": "Notes1" + }, + { + "typeName": "language", + "multiple": true, + "typeClass": "controlledVocabulary", + "value": [ + "Abkhaz", + "Afar" + ] + }, + { + "typeName": "producer", + "multiple": true, + "typeClass": "compound", + "value": [ + { + "producerName": { + "typeName": "producerName", + "multiple": false, + "typeClass": "primitive", + "value": "LastProducer1, FirstProducer1" + }, + "producerAffiliation": { + "typeName": "producerAffiliation", + "multiple": false, + "typeClass": "primitive", + "value": "ProducerAffiliation1" + }, + "producerAbbreviation": { + "typeName": "producerAbbreviation", + "multiple": false, + "typeClass": "primitive", + "value": "ProducerAbbreviation1" + }, + "producerURL": { + "typeName": "producerURL", + "multiple": false, + "typeClass": "primitive", + "value": "http://ProducerURL1.org" + }, + "producerLogoURL": { + "typeName": "producerLogoURL", + "multiple": false, + "typeClass": "primitive", + "value": "http://ProducerLogoURL1.org" + } + }, + { + "producerName": { + "typeName": "producerName", + "multiple": false, + "typeClass": "primitive", + "value": "LastProducer2, FirstProducer2" + }, + "producerAffiliation": { + "typeName": "producerAffiliation", + "multiple": false, + "typeClass": "primitive", + "value": "ProducerAffiliation2" + }, + "producerAbbreviation": { + "typeName": "producerAbbreviation", + "multiple": false, + "typeClass": "primitive", + "value": "ProducerAbbreviation2" + }, + "producerURL": { + "typeName": "producerURL", + "multiple": false, + "typeClass": "primitive", + "value": "http://ProducerURL2.org" + }, + "producerLogoURL": { + "typeName": "producerLogoURL", + "multiple": false, + "typeClass": "primitive", + "value": "http://ProducerLogoURL2.org" + } + } + ] + }, + { + "typeName": "productionDate", + "multiple": false, + "typeClass": "primitive", + "value": "1003-01-01" + }, + { + "typeName": "productionPlace", + "multiple": true, + "typeClass": "primitive", + "value": ["ProductionPlace","Second ProductionPlace"] + }, + { + "typeName": "contributor", + "multiple": true, + "typeClass": "compound", + "value": [ + { + "contributorType": { + "typeName": "contributorType", + "multiple": false, + "typeClass": "controlledVocabulary", + "value": "Data Collector" + }, + "contributorName": { + "typeName": "contributorName", + "multiple": false, + "typeClass": "primitive", + "value": "LastContributor1, FirstContributor1" + } + }, + { + "contributorType": { + "typeName": "contributorType", + "multiple": false, + "typeClass": "controlledVocabulary", + "value": "Data Curator" + }, + "contributorName": { + "typeName": "contributorName", + "multiple": false, + "typeClass": "primitive", + "value": "LastContributor2, FirstContributor2" + } + } + ] + }, + { + "typeName": "grantNumber", + "multiple": true, + "typeClass": "compound", + "value": [ + { + "grantNumberAgency": { + "typeName": "grantNumberAgency", + "multiple": false, + "typeClass": "primitive", + "value": "GrantInformationGrantAgency1" + }, + "grantNumberValue": { + "typeName": "grantNumberValue", + "multiple": false, + "typeClass": "primitive", + "value": "GrantInformationGrantNumber1" + } + }, + { + "grantNumberAgency": { + "typeName": "grantNumberAgency", + "multiple": false, + "typeClass": "primitive", + "value": "GrantInformationGrantAgency2" + }, + "grantNumberValue": { + "typeName": "grantNumberValue", + "multiple": false, + "typeClass": "primitive", + "value": "GrantInformationGrantNumber2" + } + } + ] + }, + { + "typeName": "distributor", + "multiple": true, + "typeClass": "compound", + "value": [ + { + "distributorName": { + "typeName": "distributorName", + "multiple": false, + "typeClass": "primitive", + "value": "LastDistributor1, FirstDistributor1" + }, + "distributorAffiliation": { + "typeName": "distributorAffiliation", + "multiple": false, + "typeClass": "primitive", + "value": "DistributorAffiliation1" + }, + "distributorAbbreviation": { + "typeName": "distributorAbbreviation", + "multiple": false, + "typeClass": "primitive", + "value": "DistributorAbbreviation1" + }, + "distributorURL": { + "typeName": "distributorURL", + "multiple": false, + "typeClass": "primitive", + "value": "http://DistributorURL1.org" + }, + "distributorLogoURL": { + "typeName": "distributorLogoURL", + "multiple": false, + "typeClass": "primitive", + "value": "http://DistributorLogoURL1.org" + } + }, + { + "distributorName": { + "typeName": "distributorName", + "multiple": false, + "typeClass": "primitive", + "value": "LastDistributor2, FirstDistributor2" + }, + "distributorAffiliation": { + "typeName": "distributorAffiliation", + "multiple": false, + "typeClass": "primitive", + "value": "DistributorAffiliation2" + }, + "distributorAbbreviation": { + "typeName": "distributorAbbreviation", + "multiple": false, + "typeClass": "primitive", + "value": "DistributorAbbreviation2" + }, + "distributorURL": { + "typeName": "distributorURL", + "multiple": false, + "typeClass": "primitive", + "value": "http://DistributorURL2.org" + }, + "distributorLogoURL": { + "typeName": "distributorLogoURL", + "multiple": false, + "typeClass": "primitive", + "value": "http://DistributorLogoURL2.org" + } + } + ] + }, + { + "typeName": "distributionDate", + "multiple": false, + "typeClass": "primitive", + "value": "1004-01-01" + }, + { + "typeName": "depositor", + "multiple": false, + "typeClass": "primitive", + "value": "LastDepositor, FirstDepositor" + }, + { + "typeName": "dateOfDeposit", + "multiple": false, + "typeClass": "primitive", + "value": "1002-01-01" + }, + { + "typeName": "timePeriodCovered", + "multiple": true, + "typeClass": "compound", + "value": [ + { + "timePeriodCoveredStart": { + "typeName": "timePeriodCoveredStart", + "multiple": false, + "typeClass": "primitive", + "value": "1005-01-01" + }, + "timePeriodCoveredEnd": { + "typeName": "timePeriodCoveredEnd", + "multiple": false, + "typeClass": "primitive", + "value": "1005-01-02" + } + }, + { + "timePeriodCoveredStart": { + "typeName": "timePeriodCoveredStart", + "multiple": false, + "typeClass": "primitive", + "value": "1005-02-01" + }, + "timePeriodCoveredEnd": { + "typeName": "timePeriodCoveredEnd", + "multiple": false, + "typeClass": "primitive", + "value": "1005-02-02" + } + } + ] + }, + { + "typeName": "dateOfCollection", + "multiple": true, + "typeClass": "compound", + "value": [ + { + "dateOfCollectionStart": { + "typeName": "dateOfCollectionStart", + "multiple": false, + "typeClass": "primitive", + "value": "1006-01-01" + }, + "dateOfCollectionEnd": { + "typeName": "dateOfCollectionEnd", + "multiple": false, + "typeClass": "primitive", + "value": "1006-01-01" + } + }, + { + "dateOfCollectionStart": { + "typeName": "dateOfCollectionStart", + "multiple": false, + "typeClass": "primitive", + "value": "1006-02-01" + }, + "dateOfCollectionEnd": { + "typeName": "dateOfCollectionEnd", + "multiple": false, + "typeClass": "primitive", + "value": "1006-02-02" + } + } + ] + }, + { + "typeName": "kindOfData", + "multiple": true, + "typeClass": "primitive", + "value": [ + "KindOfData1", + "KindOfData2" + ] + }, + { + "typeName": "series", + "multiple": true, + "typeClass": "compound", + "value": [{ + "seriesName": { + "typeName": "seriesName", + "multiple": false, + "typeClass": "primitive", + "value": "SeriesName" + }, + "seriesInformation": { + "typeName": "seriesInformation", + "multiple": false, + "typeClass": "primitive", + "value": "SeriesInformation" + } + }] + }, + { + "typeName": "software", + "multiple": true, + "typeClass": "compound", + "value": [ + { + "softwareName": { + "typeName": "softwareName", + "multiple": false, + "typeClass": "primitive", + "value": "SoftwareName1" + }, + "softwareVersion": { + "typeName": "softwareVersion", + "multiple": false, + "typeClass": "primitive", + "value": "SoftwareVersion1" + } + }, + { + "softwareName": { + "typeName": "softwareName", + "multiple": false, + "typeClass": "primitive", + "value": "SoftwareName2" + }, + "softwareVersion": { + "typeName": "softwareVersion", + "multiple": false, + "typeClass": "primitive", + "value": "SoftwareVersion2" + } + } + ] + }, + { + "typeName": "relatedMaterial", + "multiple": true, + "typeClass": "primitive", + "value": [ + "RelatedMaterial1", + "RelatedMaterial2" + ] + }, + { + "typeName": "relatedDatasets", + "multiple": true, + "typeClass": "primitive", + "value": [ + "RelatedDatasets1", + "RelatedDatasets2" + ] + }, + { + "typeName": "otherReferences", + "multiple": true, + "typeClass": "primitive", + "value": [ + "OtherReferences1", + "OtherReferences2" + ] + }, + { + "typeName": "dataSources", + "multiple": true, + "typeClass": "primitive", + "value": [ + "DataSources1", + "DataSources2" + ] + }, + { + "typeName": "originOfSources", + "multiple": false, + "typeClass": "primitive", + "value": "OriginOfSources" + }, + { + "typeName": "characteristicOfSources", + "multiple": false, + "typeClass": "primitive", + "value": "CharacteristicOfSourcesNoted" + }, + { + "typeName": "accessToSources", + "multiple": false, + "typeClass": "primitive", + "value": "DocumentationAndAccessToSources" + } + ] + }, + "geospatial": { + "displayName": "Geospatial Metadata", + "fields": [ + { + "typeName": "geographicCoverage", + "multiple": true, + "typeClass": "compound", + "value": [ + { + "country": { + "typeName": "country", + "multiple": false, + "typeClass": "controlledVocabulary", + "value": "Afghanistan" + }, + "state": { + "typeName": "state", + "multiple": false, + "typeClass": "primitive", + "value": "GeographicCoverageStateProvince1" + }, + "city": { + "typeName": "city", + "multiple": false, + "typeClass": "primitive", + "value": "GeographicCoverageCity1" + }, + "otherGeographicCoverage": { + "typeName": "otherGeographicCoverage", + "multiple": false, + "typeClass": "primitive", + "value": "GeographicCoverageOther1" + } + }, + { + "country": { + "typeName": "country", + "multiple": false, + "typeClass": "controlledVocabulary", + "value": "Albania" + }, + "state": { + "typeName": "state", + "multiple": false, + "typeClass": "primitive", + "value": "GeographicCoverageStateProvince2" + }, + "city": { + "typeName": "city", + "multiple": false, + "typeClass": "primitive", + "value": "GeographicCoverageCity2" + }, + "otherGeographicCoverage": { + "typeName": "otherGeographicCoverage", + "multiple": false, + "typeClass": "primitive", + "value": "GeographicCoverageOther2" + } + } + ] + }, + { + "typeName": "geographicUnit", + "multiple": true, + "typeClass": "primitive", + "value": [ + "GeographicUnit1", + "GeographicUnit2" + ] + }, + { + "typeName": "geographicBoundingBox", + "multiple": true, + "typeClass": "compound", + "value": [ + { + "westLongitude": { + "typeName": "westLongitude", + "multiple": false, + "typeClass": "primitive", + "value": "-72" + }, + "eastLongitude": { + "typeName": "eastLongitude", + "multiple": false, + "typeClass": "primitive", + "value": "-70" + }, + "northLatitude": { + "typeName": "northLatitude", + "multiple": false, + "typeClass": "primitive", + "value": "43" + }, + "southLatitude": { + "typeName": "southLatitude", + "multiple": false, + "typeClass": "primitive", + "value": "42" + } + }, + { + "westLongitude": { + "typeName": "westLongitude", + "multiple": false, + "typeClass": "primitive", + "value": "-18" + }, + "eastLongitude": { + "typeName": "eastLongitude", + "multiple": false, + "typeClass": "primitive", + "value": "-13" + }, + "northLatitude": { + "typeName": "northLatitude", + "multiple": false, + "typeClass": "primitive", + "value": "29" + }, + "southLatitude": { + "typeName": "southLatitude", + "multiple": false, + "typeClass": "primitive", + "value": "28" + } + } + ] + } + ] + }, + "socialscience": { + "displayName": "Social Science and Humanities Metadata", + "fields": [ + { + "typeName": "unitOfAnalysis", + "multiple": true, + "typeClass": "primitive", + "value": [ + "UnitOfAnalysis1", + "UnitOfAnalysis2" + ] + }, + { + "typeName": "universe", + "multiple": true, + "typeClass": "primitive", + "value": [ + "Universe1", + "Universe2" + ] + }, + { + "typeName": "timeMethod", + "multiple": false, + "typeClass": "primitive", + "value": "TimeMethod" + }, + { + "typeName": "dataCollector", + "multiple": false, + "typeClass": "primitive", + "value": "LastDataCollector1, FirstDataCollector1" + }, + { + "typeName": "collectorTraining", + "multiple": false, + "typeClass": "primitive", + "value": "CollectorTraining" + }, + { + "typeName": "frequencyOfDataCollection", + "multiple": false, + "typeClass": "primitive", + "value": "Frequency" + }, + { + "typeName": "samplingProcedure", + "multiple": false, + "typeClass": "primitive", + "value": "SamplingProcedure" + }, + { + "typeName": "targetSampleSize", + "multiple": false, + "typeClass": "compound", + "value": { + "targetSampleActualSize": { + "typeName": "targetSampleActualSize", + "multiple": false, + "typeClass": "primitive", + "value": "100" + }, + "targetSampleSizeFormula": { + "typeName": "targetSampleSizeFormula", + "multiple": false, + "typeClass": "primitive", + "value": "TargetSampleSizeFormula" + } + } + }, + { + "typeName": "deviationsFromSampleDesign", + "multiple": false, + "typeClass": "primitive", + "value": "MajorDeviationsForSampleDesign" + }, + { + "typeName": "collectionMode", + "multiple": true, + "typeClass": "primitive", + "value": ["CollectionMode"] + }, + { + "typeName": "researchInstrument", + "multiple": false, + "typeClass": "primitive", + "value": "TypeOfResearchInstrument" + }, + { + "typeName": "dataCollectionSituation", + "multiple": false, + "typeClass": "primitive", + "value": "CharacteristicsOfDataCollectionSituation" + }, + { + "typeName": "actionsToMinimizeLoss", + "multiple": false, + "typeClass": "primitive", + "value": "ActionsToMinimizeLosses" + }, + { + "typeName": "controlOperations", + "multiple": false, + "typeClass": "primitive", + "value": "ControlOperations" + }, + { + "typeName": "weighting", + "multiple": false, + "typeClass": "primitive", + "value": "Weighting" + }, + { + "typeName": "cleaningOperations", + "multiple": false, + "typeClass": "primitive", + "value": "CleaningOperations" + }, + { + "typeName": "datasetLevelErrorNotes", + "multiple": false, + "typeClass": "primitive", + "value": "StudyLevelErrorNotes" + }, + { + "typeName": "responseRate", + "multiple": false, + "typeClass": "primitive", + "value": "ResponseRate" + }, + { + "typeName": "samplingErrorEstimates", + "multiple": false, + "typeClass": "primitive", + "value": "EstimatesOfSamplingError" + }, + { + "typeName": "otherDataAppraisal", + "multiple": false, + "typeClass": "primitive", + "value": "OtherFormsOfDataAppraisal" + }, + { + "typeName": "socialScienceNotes", + "multiple": false, + "typeClass": "compound", + "value": { + "socialScienceNotesType": { + "typeName": "socialScienceNotesType", + "multiple": false, + "typeClass": "primitive", + "value": "NotesType" + }, + "socialScienceNotesSubject": { + "typeName": "socialScienceNotesSubject", + "multiple": false, + "typeClass": "primitive", + "value": "NotesSubject" + }, + "socialScienceNotesText": { + "typeName": "socialScienceNotesText", + "multiple": false, + "typeClass": "primitive", + "value": "NotesText" + } + } + } + ] + }, + "astrophysics": { + "displayName": "Astronomy and Astrophysics Metadata", + "fields": [ + { + "typeName": "astroType", + "multiple": true, + "typeClass": "controlledVocabulary", + "value": [ + "Image", + "Mosaic", + "EventList", + "Cube" + ] + }, + { + "typeName": "astroFacility", + "multiple": true, + "typeClass": "primitive", + "value": [ + "Facility1", + "Facility2" + ] + }, + { + "typeName": "astroInstrument", + "multiple": true, + "typeClass": "primitive", + "value": [ + "Instrument1", + "Instrument2" + ] + }, + { + "typeName": "astroObject", + "multiple": true, + "typeClass": "primitive", + "value": [ + "Object1", + "Object2" + ] + }, + { + "typeName": "resolution.Spatial", + "multiple": false, + "typeClass": "primitive", + "value": "SpatialResolution" + }, + { + "typeName": "resolution.Spectral", + "multiple": false, + "typeClass": "primitive", + "value": "SpectralResolution" + }, + { + "typeName": "resolution.Temporal", + "multiple": false, + "typeClass": "primitive", + "value": "TimeResolution" + }, + { + "typeName": "coverage.Spectral.Bandpass", + "multiple": true, + "typeClass": "primitive", + "value": [ + "Bandpass1", + "Bandpass2" + ] + }, + { + "typeName": "coverage.Spectral.CentralWavelength", + "multiple": true, + "typeClass": "primitive", + "value": [ + "3001", + "3002" + ] + }, + { + "typeName": "coverage.Spectral.Wavelength", + "multiple": true, + "typeClass": "compound", + "value": [ + { + "coverage.Spectral.MinimumWavelength": { + "typeName": "coverage.Spectral.MinimumWavelength", + "multiple": false, + "typeClass": "primitive", + "value": "4001" + }, + "coverage.Spectral.MaximumWavelength": { + "typeName": "coverage.Spectral.MaximumWavelength", + "multiple": false, + "typeClass": "primitive", + "value": "4002" + } + }, + { + "coverage.Spectral.MinimumWavelength": { + "typeName": "coverage.Spectral.MinimumWavelength", + "multiple": false, + "typeClass": "primitive", + "value": "4003" + }, + "coverage.Spectral.MaximumWavelength": { + "typeName": "coverage.Spectral.MaximumWavelength", + "multiple": false, + "typeClass": "primitive", + "value": "4004" + } + } + ] + }, + { + "typeName": "coverage.Temporal", + "multiple": true, + "typeClass": "compound", + "value": [ + { + "coverage.Temporal.StartTime": { + "typeName": "coverage.Temporal.StartTime", + "multiple": false, + "typeClass": "primitive", + "value": "1007-01-01" + }, + "coverage.Temporal.StopTime": { + "typeName": "coverage.Temporal.StopTime", + "multiple": false, + "typeClass": "primitive", + "value": "1007-01-02" + } + }, + { + "coverage.Temporal.StartTime": { + "typeName": "coverage.Temporal.StartTime", + "multiple": false, + "typeClass": "primitive", + "value": "1007-02-01" + }, + "coverage.Temporal.StopTime": { + "typeName": "coverage.Temporal.StopTime", + "multiple": false, + "typeClass": "primitive", + "value": "1007-02-02" + } + } + ] + }, + { + "typeName": "coverage.Spatial", + "multiple": true, + "typeClass": "primitive", + "value": [ + "SkyCoverage1", + "SkyCoverage2" + ] + }, + { + "typeName": "coverage.Depth", + "multiple": false, + "typeClass": "primitive", + "value": "200" + }, + { + "typeName": "coverage.ObjectDensity", + "multiple": false, + "typeClass": "primitive", + "value": "300" + }, + { + "typeName": "coverage.ObjectCount", + "multiple": false, + "typeClass": "primitive", + "value": "400" + }, + { + "typeName": "coverage.SkyFraction", + "multiple": false, + "typeClass": "primitive", + "value": "500" + }, + { + "typeName": "coverage.Polarization", + "multiple": false, + "typeClass": "primitive", + "value": "Polarization" + }, + { + "typeName": "redshiftType", + "multiple": false, + "typeClass": "primitive", + "value": "RedshiftType" + }, + { + "typeName": "resolution.Redshift", + "multiple": false, + "typeClass": "primitive", + "value": "600" + }, + { + "typeName": "coverage.RedshiftValue", + "multiple": true, + "typeClass": "compound", + "value": [ + { + "coverage.Redshift.MinimumValue": { + "typeName": "coverage.Redshift.MinimumValue", + "multiple": false, + "typeClass": "primitive", + "value": "701" + }, + "coverage.Redshift.MaximumValue": { + "typeName": "coverage.Redshift.MaximumValue", + "multiple": false, + "typeClass": "primitive", + "value": "702" + } + }, + { + "coverage.Redshift.MinimumValue": { + "typeName": "coverage.Redshift.MinimumValue", + "multiple": false, + "typeClass": "primitive", + "value": "703" + }, + "coverage.Redshift.MaximumValue": { + "typeName": "coverage.Redshift.MaximumValue", + "multiple": false, + "typeClass": "primitive", + "value": "704" + } + } + ] + } + ] + }, + "biomedical": { + "displayName": "Life Sciences Metadata", + "fields": [ + { + "typeName": "studyDesignType", + "multiple": true, + "typeClass": "controlledVocabulary", + "value": [ + "Case Control", + "Cross Sectional", + "Cohort Study", + "Not Specified" + ] + }, + { + "typeName": "studyFactorType", + "multiple": true, + "typeClass": "controlledVocabulary", + "value": [ + "Age", + "Biomarkers", + "Cell Surface Markers", + "Developmental Stage" + ] + }, + { + "typeName": "studyAssayOrganism", + "multiple": true, + "typeClass": "controlledVocabulary", + "value": [ + "Arabidopsis thaliana", + "Bos taurus", + "Caenorhabditis elegans", + "Danio rerio (zebrafish)" + ] + }, + { + "typeName": "studyAssayOtherOrganism", + "multiple": true, + "typeClass": "primitive", + "value": [ + "OtherOrganism1", + "OtherOrganism2" + ] + }, + { + "typeName": "studyAssayMeasurementType", + "multiple": true, + "typeClass": "controlledVocabulary", + "value": [ + "genome sequencing", + "cell sorting", + "clinical chemistry analysis", + "DNA methylation profiling" + ] + }, + { + "typeName": "studyAssayOtherMeasurmentType", + "multiple": true, + "typeClass": "primitive", + "value": [ + "OtherMeasurementType1", + "OtherMeasurementType2" + ] + }, + { + "typeName": "studyAssayTechnologyType", + "multiple": true, + "typeClass": "controlledVocabulary", + "value": [ + "culture based drug susceptibility testing, single concentration", + "culture based drug susceptibility testing, two concentrations", + "culture based drug susceptibility testing, three or more concentrations (minimium inhibitory concentration measurement)", + "flow cytometry" + ] + }, + { + "typeName": "studyAssayPlatform", + "multiple": true, + "typeClass": "controlledVocabulary", + "value": [ + "210-MS GC Ion Trap (Varian)", + "220-MS GC Ion Trap (Varian)", + "225-MS GC Ion Trap (Varian)", + "300-MS quadrupole GC/MS (Varian)" + ] + }, + { + "typeName": "studyAssayCellType", + "multiple": true, + "typeClass": "primitive", + "value": [ + "CellType1", + "CellType2" + ] + } + ] + }, + "journal": { + "displayName": "Journal Metadata", + "fields": [ + { + "typeName": "journalVolumeIssue", + "multiple": true, + "typeClass": "compound", + "value": [ + { + "journalVolume": { + "typeName": "journalVolume", + "multiple": false, + "typeClass": "primitive", + "value": "JournalVolume1" + }, + "journalIssue": { + "typeName": "journalIssue", + "multiple": false, + "typeClass": "primitive", + "value": "JournalIssue1" + }, + "journalPubDate": { + "typeName": "journalPubDate", + "multiple": false, + "typeClass": "primitive", + "value": "1008-01-01" + } + }, + { + "journalVolume": { + "typeName": "journalVolume", + "multiple": false, + "typeClass": "primitive", + "value": "JournalVolume2" + }, + "journalIssue": { + "typeName": "journalIssue", + "multiple": false, + "typeClass": "primitive", + "value": "JournalIssue2" + }, + "journalPubDate": { + "typeName": "journalPubDate", + "multiple": false, + "typeClass": "primitive", + "value": "1008-02-01" + } + } + ] + }, + { + "typeName": "journalArticleType", + "multiple": false, + "typeClass": "controlledVocabulary", + "value": "abstract" + } + ] + } + } + } +} diff --git a/scripts/api/data/licenses/licenseApache-2.0.json b/scripts/api/data/licenses/licenseApache-2.0.json index 5b7c3cf5c95..9b9bb0cc025 100644 --- a/scripts/api/data/licenses/licenseApache-2.0.json +++ b/scripts/api/data/licenses/licenseApache-2.0.json @@ -1,8 +1,11 @@ { - "name": "Apache-2.0", - "uri": "http://www.apache.org/licenses/LICENSE-2.0", - "shortDescription": "Apache License 2.0", - "active": true, - "sortOrder": 9 - } - \ No newline at end of file + "name": "Apache-2.0", + "uri": "http://www.apache.org/licenses/LICENSE-2.0", + "shortDescription": "Apache License 2.0", + "active": true, + "sortOrder": 9, + "rightsIdentifier": "Apache-2.0", + "rightsIdentifierScheme": "SPDX", + "schemeUri": "https://spdx.org/licenses/", + "languageCode": "en" +} \ No newline at end of file diff --git a/scripts/api/data/licenses/licenseCC-BY-4.0.json b/scripts/api/data/licenses/licenseCC-BY-4.0.json index 59201b8d08e..3c723e80123 100644 --- a/scripts/api/data/licenses/licenseCC-BY-4.0.json +++ b/scripts/api/data/licenses/licenseCC-BY-4.0.json @@ -4,5 +4,9 @@ "shortDescription": "Creative Commons Attribution 4.0 International License.", "iconUrl": "https://licensebuttons.net/l/by/4.0/88x31.png", "active": true, - "sortOrder": 2 -} + "sortOrder": 2, + "rightsIdentifier": "CC-BY-4.0", + "rightsIdentifierScheme": "SPDX", + "schemeUri": "https://spdx.org/licenses/", + "languageCode": "en" +} \ No newline at end of file diff --git a/scripts/api/data/licenses/licenseCC-BY-NC-4.0.json b/scripts/api/data/licenses/licenseCC-BY-NC-4.0.json index c19087664db..8c0d5f18fe3 100644 --- a/scripts/api/data/licenses/licenseCC-BY-NC-4.0.json +++ b/scripts/api/data/licenses/licenseCC-BY-NC-4.0.json @@ -4,5 +4,9 @@ "shortDescription": "Creative Commons Attribution-NonCommercial 4.0 International License.", "iconUrl": "https://licensebuttons.net/l/by-nc/4.0/88x31.png", "active": true, - "sortOrder": 4 + "sortOrder": 4, + "rightsIdentifier": "CC-BY-NC-4.0", + "rightsIdentifierScheme": "SPDX", + "schemeUri": "https://spdx.org/licenses/", + "languageCode": "en" } diff --git a/scripts/api/data/licenses/licenseCC-BY-NC-ND-4.0.json b/scripts/api/data/licenses/licenseCC-BY-NC-ND-4.0.json index 2e374917d28..a9963919eae 100644 --- a/scripts/api/data/licenses/licenseCC-BY-NC-ND-4.0.json +++ b/scripts/api/data/licenses/licenseCC-BY-NC-ND-4.0.json @@ -4,5 +4,9 @@ "shortDescription": "Creative Commons Attribution-NonCommercial-NoDerivatives 4.0 International License.", "iconUrl": "https://licensebuttons.net/l/by-nc-nd/4.0/88x31.png", "active": true, - "sortOrder": 7 + "sortOrder": 7, + "rightsIdentifier": "CC-BY-NC-ND-4.0", + "rightsIdentifierScheme": "SPDX", + "schemeUri": "https://spdx.org/licenses/", + "languageCode": "en" } diff --git a/scripts/api/data/licenses/licenseCC-BY-NC-SA-4.0.json b/scripts/api/data/licenses/licenseCC-BY-NC-SA-4.0.json index 5018884f65e..02cf9812a67 100644 --- a/scripts/api/data/licenses/licenseCC-BY-NC-SA-4.0.json +++ b/scripts/api/data/licenses/licenseCC-BY-NC-SA-4.0.json @@ -4,5 +4,9 @@ "shortDescription": "Creative Commons Attribution-NonCommercial-ShareAlike 4.0 International License.", "iconUrl": "https://licensebuttons.net/l/by-nc-sa/4.0/88x31.png", "active": true, - "sortOrder": 3 + "sortOrder": 3, + "rightsIdentifier": "CC-BY-NC-SA-4.0", + "rightsIdentifierScheme": "SPDX", + "schemeUri": "https://spdx.org/licenses/", + "languageCode": "en" } diff --git a/scripts/api/data/licenses/licenseCC-BY-ND-4.0.json b/scripts/api/data/licenses/licenseCC-BY-ND-4.0.json index 317d459a7ae..260efbe19a5 100644 --- a/scripts/api/data/licenses/licenseCC-BY-ND-4.0.json +++ b/scripts/api/data/licenses/licenseCC-BY-ND-4.0.json @@ -4,5 +4,9 @@ "shortDescription": "Creative Commons Attribution-NoDerivatives 4.0 International License.", "iconUrl": "https://licensebuttons.net/l/by-nd/4.0/88x31.png", "active": true, - "sortOrder": 6 + "sortOrder": 6, + "rightsIdentifier": "CC-BY-ND-4.0", + "rightsIdentifierScheme": "SPDX", + "schemeUri": "https://spdx.org/licenses/", + "languageCode": "en" } diff --git a/scripts/api/data/licenses/licenseCC-BY-SA-4.0.json b/scripts/api/data/licenses/licenseCC-BY-SA-4.0.json index 0d28c9423aa..ed7511ded17 100644 --- a/scripts/api/data/licenses/licenseCC-BY-SA-4.0.json +++ b/scripts/api/data/licenses/licenseCC-BY-SA-4.0.json @@ -4,5 +4,9 @@ "shortDescription": "Creative Commons Attribution-ShareAlike 4.0 International License.", "iconUrl": "https://licensebuttons.net/l/by-sa/4.0/88x31.png", "active": true, - "sortOrder": 5 + "sortOrder": 5, + "rightsIdentifier": "CC-BY-SA-4.0", + "rightsIdentifierScheme": "SPDX", + "schemeUri": "https://spdx.org/licenses/", + "languageCode": "en" } diff --git a/scripts/api/data/licenses/licenseCC0-1.0.json b/scripts/api/data/licenses/licenseCC0-1.0.json index 216260a5de8..7f21b81eea5 100644 --- a/scripts/api/data/licenses/licenseCC0-1.0.json +++ b/scripts/api/data/licenses/licenseCC0-1.0.json @@ -4,5 +4,9 @@ "shortDescription": "Creative Commons CC0 1.0 Universal Public Domain Dedication.", "iconUrl": "https://licensebuttons.net/p/zero/1.0/88x31.png", "active": true, - "sortOrder": 1 + "sortOrder": 1, + "rightsIdentifier": "CC0-1.0", + "rightsIdentifierScheme": "SPDX", + "schemeUri": "https://spdx.org/licenses/", + "languageCode": "en" } diff --git a/scripts/api/data/licenses/licenseEtalab-2.0.json b/scripts/api/data/licenses/licenseEtalab-2.0.json new file mode 100644 index 00000000000..42ec90a7540 --- /dev/null +++ b/scripts/api/data/licenses/licenseEtalab-2.0.json @@ -0,0 +1,12 @@ +{ + "name": "etalab 2.0", + "uri": "https://spdx.org/licenses/etalab-2.0", + "shortDescription": "Etalab Open License 2.0, compatible CC-BY 2.0", + "iconUrl": "https://upload.wikimedia.org/wikipedia/commons/thumb/1/18/Logo-licence-ouverte2.svg/25px-Logo-licence-ouverte2.svg.png", + "active": true, + "sortOrder": 10, + "rightsIdentifier": "etalab-2.0", + "rightsIdentifierScheme": "SPDX", + "schemeUri": "https://spdx.org/licenses/", + "languageCode": "fr" +} diff --git a/scripts/api/data/licenses/licenseMIT.json b/scripts/api/data/licenses/licenseMIT.json index a879e8a5595..32d4351ed91 100644 --- a/scripts/api/data/licenses/licenseMIT.json +++ b/scripts/api/data/licenses/licenseMIT.json @@ -3,5 +3,9 @@ "uri": "https://opensource.org/licenses/MIT", "shortDescription": "MIT License", "active": true, - "sortOrder": 8 + "sortOrder": 8, + "rightsIdentifier": "MIT", + "rightsIdentifierScheme": "SPDX", + "schemeUri": "https://spdx.org/licenses/", + "languageCode": "en" } diff --git a/scripts/api/data/metadatablocks/3d_objects.tsv b/scripts/api/data/metadatablocks/3d_objects.tsv new file mode 100644 index 00000000000..e753e4dfbed --- /dev/null +++ b/scripts/api/data/metadatablocks/3d_objects.tsv @@ -0,0 +1,45 @@ +#metadataBlock name dataverseAlias displayName + 3dobjects 3D Objects Metadata +#datasetField name title description watermark fieldType displayOrder displayFormat advancedSearchField allowControlledVocabulary allowmultiples facetable displayoncreate required parent metadatablock_id termURI + 3d3DTechnique 3D Technique The technique used for capturing the 3D data text 0 #VALUE TRUE TRUE FALSE TRUE FALSE FALSE 3dobjects + 3dEquipment Equipment The equipment used for capturing the 3D data text 1 #VALUE TRUE FALSE FALSE FALSE FALSE FALSE 3dobjects + 3dLightingSetup Lighting Setup The lighting used while capturing the 3D data text 2 #VALUE TRUE TRUE FALSE TRUE FALSE FALSE 3dobjects + 3dMasterFilePolygonCount Master File Polygon Count The high-resolution polygon count text 3 #VALUE TRUE FALSE FALSE FALSE FALSE FALSE 3dobjects + 3dExportedFilePolygonCount Exported File Polygon Count The exported mesh polygon count text 4 #VALUE TRUE FALSE TRUE FALSE FALSE FALSE 3dobjects + 3dExportedFileFormat Exported File Format The format of the exported mesh text 5 #VALUE TRUE TRUE FALSE TRUE FALSE FALSE 3dobjects + 3dAltText Alt-Text A physical description of the object modeled textbox 6 #VALUE FALSE FALSE FALSE FALSE FALSE FALSE 3dobjects + 3dMaterialComposition Material Composition The material used to create the object, e.g. stone text 7 #VALUE TRUE FALSE TRUE TRUE FALSE FALSE 3dobjects + 3dObjectDimensions Object Dimensions The general measurements of the physical object none 8 ; FALSE FALSE FALSE FALSE FALSE FALSE 3dobjects + 3dLength Length The rough length of the object text 9 Length: #VALUE FALSE FALSE FALSE FALSE FALSE FALSE 3dObjectDimensions 3dobjects + 3dWidth Width The rough width of the object text 10 Width: #VALUE FALSE FALSE FALSE FALSE FALSE FALSE 3dObjectDimensions 3dobjects + 3dHeight Height The rough height of the object text 11 Height: #VALUE FALSE FALSE FALSE FALSE FALSE FALSE 3dObjectDimensions 3dobjects + 3dWeight Weight The rough weight of the object text 12 Weight:#VALUE FALSE FALSE FALSE FALSE FALSE FALSE 3dObjectDimensions 3dobjects + 3dUnit Unit The unit of measurement used for the object dimensions text 13 Unit: #VALUE FALSE TRUE FALSE TRUE FALSE FALSE 3dObjectDimensions 3dobjects + 3dHandling Instructions Safety and special handling instructions for the object textbox 14 #VALUE FALSE FALSE FALSE FALSE FALSE FALSE 3dobjects +#controlledVocabulary DatasetField Value identifier displayOrder + 3d3DTechnique IR Scanner 0 + 3d3DTechnique Laser 1 + 3d3DTechnique Modelled 2 + 3d3DTechnique Photogrammetry 3 + 3d3DTechnique RTI 4 + 3d3DTechnique Structured Light 5 + 3d3DTechnique Tomographic 6 + 3d3DTechnique Other 7 + 3dLightingSetup Natural Light 8 + 3dLightingSetup Lightbox 9 + 3dLightingSetup LED 10 + 3dLightingSetup Fluorescent 11 + 3dLightingSetup Other 12 + 3dUnit cm 13 + 3dUnit m 14 + 3dUnit in 15 + 3dUnit ft 16 + 3dUnit lbs 17 + 3dExportedFileFormat .fbx 18 + 3dExportedFileFormat .glb 19 + 3dExportedFileFormat .gltf 20 + 3dExportedFileFormat .obj 21 + 3dExportedFileFormat .stl 22 + 3dExportedFileFormat .usdz 23 + 3dExportedFileFormat .x3d 24 + 3dExportedFileFormat other 25 diff --git a/scripts/api/data/metadatablocks/archival.tsv b/scripts/api/data/metadatablocks/archival.tsv new file mode 100644 index 00000000000..89ef5466a44 --- /dev/null +++ b/scripts/api/data/metadatablocks/archival.tsv @@ -0,0 +1,12 @@ +#metadataBlock name dataverseAlias displayName blockURI + archival Archival Metadata +#datasetField name title description watermark fieldType displayOrder displayFormat advancedSearchField allowControlledVocabulary allowmultiples facetable displayoncreate required parent metadatablock_id termURI + submitToArchivalAppraisal Submit to Archival Appraisal Your assessment whether the dataset should be submitted for archival appraisal text 0 #VALUE TRUE TRUE FALSE FALSE TRUE FALSE archival + archivedFrom Archived from A date (YYYY-MM-DD) from which the dataset is archived YYYY-MM-DD date 1 #VALUE TRUE FALSE FALSE FALSE FALSE FALSE archival + holdingArchive Holding Archive Information on the holding archive where the dataset is archived none 3 FALSE FALSE TRUE FALSE FALSE FALSE archival + holdingArchiveName Archived at Holding Archive The name of the holding archive text 4 #VALUE FALSE FALSE FALSE FALSE FALSE FALSE holdingArchive archival https://schema.org/holdingArchive + archivedAt Archived at URL URL to holding archive URL url 5 "#VALUE" FALSE FALSE FALSE FALSE FALSE FALSE holdingArchive archival https://schema.org/archivedAt +#controlledVocabulary DatasetField Value identifier displayOrder + submitToArchivalAppraisal True 0 + submitToArchivalAppraisal False 1 + submitToArchivalAppraisal Unknown 2 diff --git a/scripts/api/data/metadatablocks/citation.tsv b/scripts/api/data/metadatablocks/citation.tsv index abc09465603..dea23aa9a73 100644 --- a/scripts/api/data/metadatablocks/citation.tsv +++ b/scripts/api/data/metadatablocks/citation.tsv @@ -133,13 +133,14 @@ contributorType Work Package Leader 15 contributorType Other 16 authorIdentifierScheme ORCID 0 - authorIdentifierScheme ISNI 1 - authorIdentifierScheme LCNA 2 - authorIdentifierScheme VIAF 3 - authorIdentifierScheme GND 4 - authorIdentifierScheme DAI 5 - authorIdentifierScheme ResearcherID 6 - authorIdentifierScheme ScopusID 7 + authorIdentifierScheme ROR 1 + authorIdentifierScheme ISNI 2 + authorIdentifierScheme LCNA 3 + authorIdentifierScheme VIAF 4 + authorIdentifierScheme GND 5 + authorIdentifierScheme DAI 6 + authorIdentifierScheme ResearcherID 7 + authorIdentifierScheme ScopusID 8 language 'Are'are alu 0 alu language 'Auhelawa kud 1 kud language A'ou aou 2 aou @@ -8061,4 +8062,4 @@ publicationRelationType IsSupplementTo RT3 3 publicationRelationType IsSupplementedBy RT4 4 publicationRelationType IsReferencedBy RT5 5 - publicationRelationType References RT6 6 \ No newline at end of file + publicationRelationType References RT6 6 diff --git a/scripts/api/setup-datasetfields.sh b/scripts/api/setup-datasetfields.sh index 51da677ceb8..908988f8acb 100755 --- a/scripts/api/setup-datasetfields.sh +++ b/scripts/api/setup-datasetfields.sh @@ -11,3 +11,4 @@ curl "${DATAVERSE_URL}/api/admin/datasetfield/load" -X POST --data-binary @"$SCR curl "${DATAVERSE_URL}/api/admin/datasetfield/load" -X POST --data-binary @"$SCRIPT_PATH"/data/metadatablocks/astrophysics.tsv -H "Content-type: text/tab-separated-values" curl "${DATAVERSE_URL}/api/admin/datasetfield/load" -X POST --data-binary @"$SCRIPT_PATH"/data/metadatablocks/biomedical.tsv -H "Content-type: text/tab-separated-values" curl "${DATAVERSE_URL}/api/admin/datasetfield/load" -X POST --data-binary @"$SCRIPT_PATH"/data/metadatablocks/journals.tsv -H "Content-type: text/tab-separated-values" +curl "${DATAVERSE_URL}/api/admin/datasetfield/load" -X POST --data-binary @"$SCRIPT_PATH"/data/metadatablocks/3d_objects.tsv -H "Content-type: text/tab-separated-values" diff --git a/scripts/api/update-datasetfields.sh b/scripts/api/update-datasetfields.sh deleted file mode 100644 index ae099f8dcfd..00000000000 --- a/scripts/api/update-datasetfields.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/sh -curl http://localhost:8080/api/admin/datasetfield/load -X POST --data-binary @data/metadatablocks/citation.tsv -H "Content-type: text/tab-separated-values" -curl http://localhost:8080/api/admin/datasetfield/load -X POST --data-binary @data/metadatablocks/geospatial.tsv -H "Content-type: text/tab-separated-values" -curl http://localhost:8080/api/admin/datasetfield/load -X POST --data-binary @data/metadatablocks/social_science.tsv -H "Content-type: text/tab-separated-values" -curl http://localhost:8080/api/admin/datasetfield/load -X POST --data-binary @data/metadatablocks/astrophysics.tsv -H "Content-type: text/tab-separated-values" -curl http://localhost:8080/api/admin/datasetfield/load -X POST --data-binary @data/metadatablocks/biomedical.tsv -H "Content-type: text/tab-separated-values" -curl http://localhost:8080/api/admin/datasetfield/load -X POST --data-binary @data/metadatablocks/journals.tsv -H "Content-type: text/tab-separated-values" \ No newline at end of file diff --git a/scripts/installer/README_python.txt b/scripts/installer/README_python.txt index ba3dd041e09..13abfeb1941 100644 --- a/scripts/installer/README_python.txt +++ b/scripts/installer/README_python.txt @@ -39,7 +39,7 @@ in your PATH. If you have multiple versions of PostgresQL installed, make sure the version that you will be using with Dataverse is the first on your PATH. For example, - PATH=/usr/pgsql-13/bin:$PATH; export PATH + PATH=/usr/pgsql-16/bin:$PATH; export PATH Certain libraries and source include files, both for PostgresQL and Python, are also needed to compile the module. On @@ -47,7 +47,7 @@ RedHat/CentOS/etc. you may need to install the -devel packages, *for the specific versions* of PostgreSQL and Python you will be using. For example: - yum install postgresql13-devel + yum install postgresql16-devel yum install python37-devel etc. diff --git a/scripts/installer/as-setup.sh b/scripts/installer/as-setup.sh index e87122ba77c..68a53270114 100755 --- a/scripts/installer/as-setup.sh +++ b/scripts/installer/as-setup.sh @@ -124,6 +124,10 @@ function preliminary_setup() # bump the http-listener timeout from 900 to 3600 ./asadmin $ASADMIN_OPTS set server-config.network-config.protocols.protocol.http-listener-1.http.request-timeout-seconds="${GLASSFISH_REQUEST_TIMEOUT}" + # Set SameSite cookie value: https://docs.payara.fish/community/docs/6.2024.6/Technical%20Documentation/Payara%20Server%20Documentation/General%20Administration/Administering%20HTTP%20Connectivity.html + ./asadmin $ASADMIN_OPTS set server-config.network-config.protocols.protocol.http-listener-1.http.cookie-same-site-value="Lax" + ./asadmin $ASADMIN_OPTS set server-config.network-config.protocols.protocol.http-listener-1.http.cookie-same-site-enabled="true" + # so we can front with apache httpd ( ProxyPass / ajp://localhost:8009/ ) ./asadmin $ASADMIN_OPTS create-network-listener --protocol http-listener-1 --listenerport 8009 --jkenabled true jk-connector } diff --git a/src/main/java/edu/harvard/iq/dataverse/DataCitation.java b/src/main/java/edu/harvard/iq/dataverse/DataCitation.java index 02fb59751fb..589fb5fea9c 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DataCitation.java +++ b/src/main/java/edu/harvard/iq/dataverse/DataCitation.java @@ -6,6 +6,7 @@ package edu.harvard.iq.dataverse; import edu.harvard.iq.dataverse.branding.BrandingUtil; +import edu.harvard.iq.dataverse.dataset.DatasetType; import edu.harvard.iq.dataverse.harvest.client.HarvestingClient; import edu.harvard.iq.dataverse.pidproviders.AbstractPidProvider; @@ -29,15 +30,34 @@ import java.util.stream.Collectors; import jakarta.ejb.EJBException; +import jakarta.json.JsonObject; +import jakarta.ws.rs.core.MediaType; + import javax.xml.stream.XMLOutputFactory; import javax.xml.stream.XMLStreamException; import javax.xml.stream.XMLStreamWriter; import edu.harvard.iq.dataverse.util.BundleUtil; import edu.harvard.iq.dataverse.util.DateUtil; +import edu.harvard.iq.dataverse.util.PersonOrOrgUtil; +import edu.harvard.iq.dataverse.util.SystemConfig; +import edu.harvard.iq.dataverse.util.json.JsonUtil; + import org.apache.commons.text.StringEscapeUtils; + +import de.undercouch.citeproc.csl.CSLItemDataBuilder; +import de.undercouch.citeproc.csl.CSLName; +import de.undercouch.citeproc.csl.CSLNameBuilder; +import de.undercouch.citeproc.csl.CSLType; +import de.undercouch.citeproc.helper.json.JsonBuilder; +import de.undercouch.citeproc.helper.json.StringJsonBuilderFactory; + import org.apache.commons.lang3.StringUtils; +import static edu.harvard.iq.dataverse.pidproviders.doi.AbstractDOIProvider.DOI_PROTOCOL; +import static edu.harvard.iq.dataverse.pidproviders.handle.HandlePidProvider.HDL_PROTOCOL; +import static edu.harvard.iq.dataverse.pidproviders.perma.PermaLinkPidProvider.PERMA_PROTOCOL; + /** * * @author gdurand, qqmyers @@ -47,6 +67,7 @@ public class DataCitation { private static final Logger logger = Logger.getLogger(DataCitation.class.getCanonicalName()); private List authors = new ArrayList(); + private List cslAuthors = new ArrayList(); private List producers = new ArrayList(); private String title; private String fileTitle = null; @@ -67,8 +88,18 @@ public class DataCitation { private List spatialCoverages; private List optionalValues = new ArrayList<>(); - private int optionalURLcount = 0; + private int optionalURLcount = 0; + + private DatasetType type; + public enum Format { + Internal, + EndNote, + RIS, + BibTeX, + CSL + } + public DataCitation(DatasetVersion dsv) { this(dsv, false); } @@ -142,8 +173,14 @@ private void getCommonValuesFrom(DatasetVersion dsv) { spatialCoverages = dsv.getSpatialCoverages(); publisher = getPublisherFrom(dsv); version = getVersionFrom(dsv); + type = getTypeFrom(dsv); } + private DatasetType getTypeFrom(DatasetVersion dsv) { + return dsv.getDataset().getDatasetType(); + } + + public String getAuthorsString() { return String.join("; ", authors); } @@ -189,7 +226,45 @@ public String toString() { public String toString(boolean html) { return toString(html, false); } + public String toString(boolean html, boolean anonymized) { + return toString(Format.Internal, html, anonymized); + } + + public String toString(Format format, boolean html, boolean anonymized) { + if(anonymized && (format != Format.Internal)) { + //Only Internal format supports anonymization + return null; + } + switch (format) { + case BibTeX: + return toBibtexString(); + case CSL: + return JsonUtil.prettyPrint(getCSLJsonFormat()); + case EndNote: + return toEndNoteString(); + case Internal: + return formatInternalCitation(html, anonymized); + case RIS: + return toRISString(); + } + return null; + } + + public static String getCitationFormatMediaType(Format format, boolean isHtml) { + switch (format) { + + case CSL: + return MediaType.APPLICATION_JSON; + case EndNote: + return MediaType.TEXT_XML; + case Internal: + return isHtml ? MediaType.TEXT_HTML : MediaType.TEXT_PLAIN; + } + return MediaType.TEXT_PLAIN; + } + + private String formatInternalCitation(boolean html, boolean anonymized) { // first add comma separated parts String separator = ", "; List citationList = new ArrayList<>(); @@ -293,11 +368,13 @@ public void writeAsBibtexCitation(OutputStream os) throws IOException { out.write("version = {"); out.write(version); out.write("},\r\n"); - out.write("doi = {"); - out.write(persistentId.getAuthority()); - out.write("/"); - out.write(persistentId.getIdentifier()); - out.write("},\r\n"); + if("doi".equals(persistentId.getProtocol())) { + out.write("doi = {"); + out.write(persistentId.getAuthority()); + out.write("/"); + out.write(persistentId.getIdentifier()); + out.write("},\r\n"); + } out.write("url = {"); out.write(persistentId.asURL()); out.write("}\r\n"); @@ -595,11 +672,21 @@ private void createEndNoteXML(XMLStreamWriter xmlw) throws XMLStreamException { } xmlw.writeStartElement("urls"); - xmlw.writeStartElement("related-urls"); - xmlw.writeStartElement("url"); - xmlw.writeCharacters(getPersistentId().asURL()); - xmlw.writeEndElement(); // url - xmlw.writeEndElement(); // related-urls + if (persistentId != null) { + if (PERMA_PROTOCOL.equals(persistentId.getProtocol()) || HDL_PROTOCOL.equals(persistentId.getProtocol())) { + xmlw.writeStartElement("web-urls"); + xmlw.writeStartElement("url"); + xmlw.writeCharacters(getPersistentId().asURL()); + xmlw.writeEndElement(); // url + xmlw.writeEndElement(); // web-urls + } else if (DOI_PROTOCOL.equals(persistentId.getProtocol())) { + xmlw.writeStartElement("related-urls"); + xmlw.writeStartElement("url"); + xmlw.writeCharacters(getPersistentId().asURL()); + xmlw.writeEndElement(); // url + xmlw.writeEndElement(); // related-urls + } + } xmlw.writeEndElement(); // urls // a DataFile citation also includes the filename and (for Tabular @@ -617,10 +704,9 @@ private void createEndNoteXML(XMLStreamWriter xmlw) throws XMLStreamException { xmlw.writeEndElement(); // custom2 } } - if (persistentId != null) { + if (persistentId != null && "doi".equals(persistentId.getProtocol())) { xmlw.writeStartElement("electronic-resource-num"); - String electResourceNum = persistentId.getProtocol() + "/" + persistentId.getAuthority() + "/" - + persistentId.getIdentifier(); + String electResourceNum = persistentId.asRawIdentifier(); xmlw.writeCharacters(electResourceNum); xmlw.writeEndElement(); } @@ -650,9 +736,30 @@ public Map getDataCiteMetadata() { metadata.put("datacite.publisher", producerString); metadata.put("datacite.publicationyear", getYear()); return metadata; - } + } + + public JsonObject getCSLJsonFormat() { + CSLItemDataBuilder itemBuilder = new CSLItemDataBuilder(); + if (type.equals(DatasetType.DATASET_TYPE_SOFTWARE)) { + itemBuilder.type(CSLType.SOFTWARE); + } else { + itemBuilder.type(CSLType.DATASET); + } + itemBuilder.title(formatString(title,true)).author((CSLName[]) cslAuthors.toArray(new CSLName[0])).issued(Integer.parseInt(year)); + if (seriesTitles != null) { + itemBuilder.containerTitle(formatString(seriesTitles.get(0), true)); + } + itemBuilder.version(version).DOI(persistentId.asString()); + if (keywords != null) { + itemBuilder + .categories(keywords.stream().map(keyword -> formatString(keyword, true)).toArray(String[]::new)); + } + itemBuilder.abstrct(formatString(description, true)).publisher(formatString(publisher, true)) + .URL(SystemConfig.getDataverseSiteUrlStatic() + "/citation?persistentId=" + persistentId.asString()); + JsonBuilder b = (new StringJsonBuilderFactory()).createJsonBuilder(); + return JsonUtil.getJsonObject((String) itemBuilder.build().toJson(b)); + } - // helper methods private String formatString(String value, boolean escapeHtml) { return formatString(value, escapeHtml, ""); @@ -759,6 +866,20 @@ private void getAuthorsAndProducersFrom(DatasetVersion dsv) { if (!author.isEmpty()) { String an = author.getName().getDisplayValue().trim(); authors.add(an); + boolean isOrg = "ROR".equals(author.getIdType()); + JsonObject authorJson = PersonOrOrgUtil.getPersonOrOrganization(an, false, !isOrg); + if (!authorJson.getBoolean("isPerson")) { + cslAuthors.add(new CSLNameBuilder().literal(formatString(authorJson.getString("fullName"), true)).isInstitution(true).build()); + } else { + if (authorJson.containsKey("givenName") && authorJson.containsKey("familyName")) { + String givenName = formatString(authorJson.getString("givenName"),true); + String familyName = formatString(authorJson.getString("familyName"), true); + cslAuthors.add(new CSLNameBuilder().given(givenName).family(familyName).isInstitution(false).build()); + } else { + cslAuthors.add( + new CSLNameBuilder().literal(formatString(authorJson.getString("fullName"), true)).isInstitution(false).build()); + } + } } }); producers = dsv.getDatasetProducerNames(); diff --git a/src/main/java/edu/harvard/iq/dataverse/DataFile.java b/src/main/java/edu/harvard/iq/dataverse/DataFile.java index 1a610d9ea6e..01c1a48e117 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DataFile.java +++ b/src/main/java/edu/harvard/iq/dataverse/DataFile.java @@ -1142,4 +1142,12 @@ public boolean isDeaccessioned() { } return inDeaccessionedVersions; // since any published version would have already returned } + public boolean isInDatasetVersion(DatasetVersion version) { + for (FileMetadata fmd : getFileMetadatas()) { + if (fmd.getDatasetVersion().equals(version)) { + return true; + } + } + return false; + } } // end of class diff --git a/src/main/java/edu/harvard/iq/dataverse/Dataset.java b/src/main/java/edu/harvard/iq/dataverse/Dataset.java index 78579b1de21..79c64d03d60 100644 --- a/src/main/java/edu/harvard/iq/dataverse/Dataset.java +++ b/src/main/java/edu/harvard/iq/dataverse/Dataset.java @@ -68,6 +68,8 @@ query = "SELECT o FROM Dataset o WHERE o.creator.id=:creatorId"), @NamedQuery(name = "Dataset.findByReleaseUserId", query = "SELECT o FROM Dataset o WHERE o.releaseUser.id=:releaseUserId"), + @NamedQuery(name = "Dataset.countAll", + query = "SELECT COUNT(ds) FROM Dataset ds") }) /* diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetAuthor.java b/src/main/java/edu/harvard/iq/dataverse/DatasetAuthor.java index d33d709107f..bc85ab22e77 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetAuthor.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetAuthor.java @@ -8,6 +8,8 @@ import java.util.Comparator; +import org.apache.commons.lang3.StringUtils; + /** * * @author skraffmiller @@ -81,8 +83,8 @@ public void setIdValue(String idValue) { } public boolean isEmpty() { - return ( (affiliation==null || affiliation.getValue().trim().equals("")) - && (name==null || name.getValue().trim().equals("")) + return ( (affiliation==null || StringUtils.isBlank(affiliation.getValue())) + && (name==null || StringUtils.isBlank(name.getValue())) ); } @@ -97,8 +99,13 @@ public static String getIdentifierAsUrl(String idType, String idValue) { if (idType != null && !idType.isEmpty() && idValue != null && !idValue.isEmpty()) { try { ExternalIdentifier externalIdentifier = ExternalIdentifier.valueOf(idType); - if (externalIdentifier.isValidIdentifier(idValue)) - return externalIdentifier.format(idValue); + if (externalIdentifier.isValidIdentifier(idValue)) { + String uri = externalIdentifier.format(idValue); + //The DAI identifier is a URI starting with "info" - we don't want to return it as a URL (we assume non-null URLs should be links in the display) + if(uri.startsWith("http")) { + return uri; + } + } } catch (Exception e) { // non registered identifier } diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionNoteValidator.java b/src/main/java/edu/harvard/iq/dataverse/DatasetDeaccessionNoteValidator.java similarity index 76% rename from src/main/java/edu/harvard/iq/dataverse/DatasetVersionNoteValidator.java rename to src/main/java/edu/harvard/iq/dataverse/DatasetDeaccessionNoteValidator.java index a5ea487a68f..7c6263fe9b9 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionNoteValidator.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetDeaccessionNoteValidator.java @@ -13,28 +13,28 @@ * * @author skraffmi */ -public class DatasetVersionNoteValidator implements ConstraintValidator { +public class DatasetDeaccessionNoteValidator implements ConstraintValidator { private String versionState; - private String versionNote; + private String deaccessionNote; @Override - public void initialize(ValidateVersionNote constraintAnnotation) { + public void initialize(ValidateDeaccessionNote constraintAnnotation) { versionState = constraintAnnotation.versionState(); - versionNote = constraintAnnotation.versionNote(); + deaccessionNote = constraintAnnotation.deaccessionNote(); } @Override public boolean isValid(DatasetVersion value, ConstraintValidatorContext context) { - if (versionState.equals(DatasetVersion.VersionState.DEACCESSIONED) && versionNote.isEmpty()){ + if (versionState.equals(DatasetVersion.VersionState.DEACCESSIONED) && deaccessionNote.isEmpty()){ if (context != null) { context.buildConstraintViolationWithTemplate(value + " " + BundleUtil.getStringFromBundle("file.deaccessionDialog.dialog.textForReason.error")).addConstraintViolation(); } return false; } - if (versionState.equals(DatasetVersion.VersionState.DEACCESSIONED) && versionNote.length() > DatasetVersion.VERSION_NOTE_MAX_LENGTH){ + if (versionState.equals(DatasetVersion.VersionState.DEACCESSIONED) && deaccessionNote.length() > DatasetVersion.VERSION_NOTE_MAX_LENGTH){ if (context != null) { context.buildConstraintViolationWithTemplate(value + " " + BundleUtil.getStringFromBundle("file.deaccessionDialog.dialog.limitChar.error")).addConstraintViolation(); } diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetFieldServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DatasetFieldServiceBean.java index ded7c83de62..85639de9a59 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetFieldServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetFieldServiceBean.java @@ -1,5 +1,6 @@ package edu.harvard.iq.dataverse; +import edu.harvard.iq.dataverse.dataset.DatasetType; import java.io.IOException; import java.io.StringReader; import java.net.URI; @@ -92,7 +93,7 @@ public class DatasetFieldServiceBean implements java.io.Serializable { String oldHash = null; public List findAllAdvancedSearchFieldTypes() { - return em.createQuery("select object(o) from DatasetFieldType as o where o.advancedSearchFieldType = true and o.title != '' order by o.id", DatasetFieldType.class).getResultList(); + return em.createQuery("select object(o) from DatasetFieldType as o where o.advancedSearchFieldType = true and o.title != '' order by o.displayOrder,o.id", DatasetFieldType.class).getResultList(); } public List findAllFacetableFieldTypes() { @@ -871,7 +872,7 @@ public List findAllDisplayedOnCreateInMetadataBlock(MetadataBl Root metadataBlockRoot = criteriaQuery.from(MetadataBlock.class); Root datasetFieldTypeRoot = criteriaQuery.from(DatasetFieldType.class); - Predicate requiredInDataversePredicate = buildRequiredInDataversePredicate(criteriaBuilder, datasetFieldTypeRoot); + Predicate fieldRequiredInTheInstallation = buildFieldRequiredInTheInstallationPredicate(criteriaBuilder, datasetFieldTypeRoot); criteriaQuery.where( criteriaBuilder.and( @@ -879,7 +880,7 @@ public List findAllDisplayedOnCreateInMetadataBlock(MetadataBl datasetFieldTypeRoot.in(metadataBlockRoot.get("datasetFieldTypes")), criteriaBuilder.or( criteriaBuilder.isTrue(datasetFieldTypeRoot.get("displayOnCreate")), - requiredInDataversePredicate + fieldRequiredInTheInstallation ) ) ); @@ -890,9 +891,9 @@ public List findAllDisplayedOnCreateInMetadataBlock(MetadataBl return typedQuery.getResultList(); } - public List findAllInMetadataBlockAndDataverse(MetadataBlock metadataBlock, Dataverse dataverse, boolean onlyDisplayedOnCreate) { + public List findAllInMetadataBlockAndDataverse(MetadataBlock metadataBlock, Dataverse dataverse, boolean onlyDisplayedOnCreate, DatasetType datasetType) { if (!dataverse.isMetadataBlockRoot() && dataverse.getOwner() != null) { - return findAllInMetadataBlockAndDataverse(metadataBlock, dataverse.getOwner(), onlyDisplayedOnCreate); + return findAllInMetadataBlockAndDataverse(metadataBlock, dataverse.getOwner(), onlyDisplayedOnCreate, datasetType); } CriteriaBuilder criteriaBuilder = em.getCriteriaBuilder(); @@ -900,6 +901,29 @@ public List findAllInMetadataBlockAndDataverse(MetadataBlock m Root metadataBlockRoot = criteriaQuery.from(MetadataBlock.class); Root datasetFieldTypeRoot = criteriaQuery.from(DatasetFieldType.class); + + // Build the main predicate to include fields that belong to the specified dataverse and metadataBlock and match the onlyDisplayedOnCreate value. + Predicate fieldPresentInDataverse = buildFieldPresentInDataversePredicate(dataverse, onlyDisplayedOnCreate, criteriaQuery, criteriaBuilder, datasetFieldTypeRoot, metadataBlockRoot); + + // Build an additional predicate to include fields from the datasetType, if the datasetType is specified and contains the given metadataBlock. + Predicate fieldPresentInDatasetType = buildFieldPresentInDatasetTypePredicate(datasetType, criteriaQuery, criteriaBuilder, datasetFieldTypeRoot, metadataBlockRoot, onlyDisplayedOnCreate); + + // Build the final WHERE clause by combining all the predicates. + criteriaQuery.where( + criteriaBuilder.equal(metadataBlockRoot.get("id"), metadataBlock.getId()), // Match the MetadataBlock ID. + datasetFieldTypeRoot.in(metadataBlockRoot.get("datasetFieldTypes")), // Ensure the DatasetFieldType is part of the MetadataBlock. + criteriaBuilder.or( + fieldPresentInDataverse, + fieldPresentInDatasetType + ) + ); + + criteriaQuery.select(datasetFieldTypeRoot); + + return em.createQuery(criteriaQuery).getResultList(); + } + + private Predicate buildFieldPresentInDataversePredicate(Dataverse dataverse, boolean onlyDisplayedOnCreate, CriteriaQuery criteriaQuery, CriteriaBuilder criteriaBuilder, Root datasetFieldTypeRoot, Root metadataBlockRoot) { Root dataverseRoot = criteriaQuery.from(Dataverse.class); // Join Dataverse with DataverseFieldTypeInputLevel on the "dataverseFieldTypeInputLevels" attribute, using a LEFT JOIN. @@ -917,20 +941,27 @@ public List findAllInMetadataBlockAndDataverse(MetadataBlock m criteriaBuilder.isTrue(datasetFieldTypeInputLevelJoin.get("required")) ); + // Predicate for displayOnCreate in input level + Predicate displayOnCreateInputLevelPredicate = criteriaBuilder.and( + criteriaBuilder.equal(datasetFieldTypeRoot, datasetFieldTypeInputLevelJoin.get("datasetFieldType")), + criteriaBuilder.equal(datasetFieldTypeInputLevelJoin.get("displayOnCreate"), Boolean.TRUE) + ); + // Create a subquery to check for the absence of a specific DataverseFieldTypeInputLevel. Subquery subquery = criteriaQuery.subquery(Long.class); Root subqueryRoot = subquery.from(DataverseFieldTypeInputLevel.class); subquery.select(criteriaBuilder.literal(1L)) .where( criteriaBuilder.equal(subqueryRoot.get("dataverse"), dataverseRoot), - criteriaBuilder.equal(subqueryRoot.get("datasetFieldType"), datasetFieldTypeRoot) + criteriaBuilder.equal(subqueryRoot.get("datasetFieldType"), datasetFieldTypeRoot), + criteriaBuilder.isNotNull(subqueryRoot.get("displayOnCreate")) ); // Define a predicate to exclude DatasetFieldTypes that have no associated input level (i.e., the subquery does not return a result). Predicate hasNoInputLevelPredicate = criteriaBuilder.not(criteriaBuilder.exists(subquery)); // Define a predicate to include the required fields in Dataverse. - Predicate requiredInDataversePredicate = buildRequiredInDataversePredicate(criteriaBuilder, datasetFieldTypeRoot); + Predicate fieldRequiredInTheInstallation = buildFieldRequiredInTheInstallationPredicate(criteriaBuilder, datasetFieldTypeRoot); // Define a predicate for displaying DatasetFieldTypes on create. // If onlyDisplayedOnCreate is true, include fields that: @@ -939,30 +970,68 @@ public List findAllInMetadataBlockAndDataverse(MetadataBlock m // Otherwise, use an always-true predicate (conjunction). Predicate displayedOnCreatePredicate = onlyDisplayedOnCreate ? criteriaBuilder.or( - criteriaBuilder.or( + // 1. Field marked as displayOnCreate in input level + displayOnCreateInputLevelPredicate, + + // 2. Field without input level that is marked as displayOnCreate or required + criteriaBuilder.and( + hasNoInputLevelPredicate, + criteriaBuilder.or( criteriaBuilder.isTrue(datasetFieldTypeRoot.get("displayOnCreate")), - requiredInDataversePredicate + fieldRequiredInTheInstallation + ) ), + + // 3. Field required by input level requiredAsInputLevelPredicate ) : criteriaBuilder.conjunction(); - // Build the final WHERE clause by combining all the predicates. - criteriaQuery.where( + // Combine all the predicates. + return criteriaBuilder.and( criteriaBuilder.equal(dataverseRoot.get("id"), dataverse.getId()), // Match the Dataverse ID. - criteriaBuilder.equal(metadataBlockRoot.get("id"), metadataBlock.getId()), // Match the MetadataBlock ID. metadataBlockRoot.in(dataverseRoot.get("metadataBlocks")), // Ensure the MetadataBlock is part of the Dataverse. - datasetFieldTypeRoot.in(metadataBlockRoot.get("datasetFieldTypes")), // Ensure the DatasetFieldType is part of the MetadataBlock. criteriaBuilder.or(includedAsInputLevelPredicate, hasNoInputLevelPredicate), // Include DatasetFieldTypes based on the input level predicates. displayedOnCreatePredicate // Apply the display-on-create filter if necessary. ); + } - criteriaQuery.select(datasetFieldTypeRoot).distinct(true); - - return em.createQuery(criteriaQuery).getResultList(); + private Predicate buildFieldPresentInDatasetTypePredicate(DatasetType datasetType, + CriteriaQuery criteriaQuery, + CriteriaBuilder criteriaBuilder, + Root datasetFieldTypeRoot, + Root metadataBlockRoot, + boolean onlyDisplayedOnCreate) { + Predicate datasetTypePredicate = criteriaBuilder.isFalse(criteriaBuilder.literal(true)); // Initialize datasetTypePredicate to always false by default + if (datasetType != null) { + // Create a subquery to check for the presence of the specified metadataBlock within the datasetType + Subquery datasetTypeSubquery = criteriaQuery.subquery(Long.class); + Root datasetTypeRoot = criteriaQuery.from(DatasetType.class); + + // Define a predicate for displaying DatasetFieldTypes on create. + // If onlyDisplayedOnCreate is true, include fields that are either marked as displayed on create OR marked as required. + // Otherwise, use an always-true predicate (conjunction). + Predicate displayedOnCreatePredicate = onlyDisplayedOnCreate ? + criteriaBuilder.or( + criteriaBuilder.isTrue(datasetFieldTypeRoot.get("displayOnCreate")), + buildFieldRequiredInTheInstallationPredicate(criteriaBuilder, datasetFieldTypeRoot) + ) + : criteriaBuilder.conjunction(); + + datasetTypeSubquery.select(criteriaBuilder.literal(1L)) + .where( + criteriaBuilder.equal(datasetTypeRoot.get("id"), datasetType.getId()), // Match the DatasetType ID. + metadataBlockRoot.in(datasetTypeRoot.get("metadataBlocks")), // Ensure the metadataBlock is included in the datasetType's list of metadata blocks. + displayedOnCreatePredicate + ); + + // Now set the datasetTypePredicate to true if the subquery finds a matching metadataBlock + datasetTypePredicate = criteriaBuilder.exists(datasetTypeSubquery); + } + return datasetTypePredicate; } - private Predicate buildRequiredInDataversePredicate(CriteriaBuilder criteriaBuilder, Root datasetFieldTypeRoot) { + private Predicate buildFieldRequiredInTheInstallationPredicate(CriteriaBuilder criteriaBuilder, Root datasetFieldTypeRoot) { // Predicate to check if the current DatasetFieldType is required. Predicate isRequired = criteriaBuilder.isTrue(datasetFieldTypeRoot.get("required")); diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetFieldType.java b/src/main/java/edu/harvard/iq/dataverse/DatasetFieldType.java index 01785359e0e..32a23e06761 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetFieldType.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetFieldType.java @@ -282,7 +282,26 @@ public boolean isDisplayOnCreate() { public void setDisplayOnCreate(boolean displayOnCreate) { this.displayOnCreate = displayOnCreate; } + + /** + * Determines whether this field type is displayed in the form when creating + * the Dataset (or only later when editing after the initial creation). + */ + @Transient + private Boolean localDisplayOnCreate; + + public Boolean getLocalDisplayOnCreate() { + return localDisplayOnCreate; + } + + public void setLocalDisplayOnCreate(Boolean localDisplayOnCreate) { + this.localDisplayOnCreate = localDisplayOnCreate; + } + public boolean shouldDisplayOnCreate() { + return (localDisplayOnCreate == null) ? displayOnCreate : localDisplayOnCreate; + } + public boolean isControlledVocabulary() { return allowControlledVocabulary; } @@ -531,28 +550,36 @@ public String getDisplayName() { public SolrField getSolrField() { SolrField.SolrType solrType = SolrField.SolrType.TEXT_EN; if (fieldType != null) { - - /** - * @todo made more decisions based on fieldType: index as dates, - * integers, and floats so we can do range queries etc. - */ if (fieldType.equals(FieldType.DATE)) { solrType = SolrField.SolrType.DATE; } else if (fieldType.equals(FieldType.EMAIL)) { solrType = SolrField.SolrType.EMAIL; + } else if (fieldType.equals(FieldType.INT)) { + solrType = SolrField.SolrType.INTEGER; + } else if (fieldType.equals(FieldType.FLOAT)) { + solrType = SolrField.SolrType.FLOAT; } - Boolean parentAllowsMultiplesBoolean = false; - if (isHasParent()) { - if (getParentDatasetFieldType() != null) { - DatasetFieldType parent = getParentDatasetFieldType(); - parentAllowsMultiplesBoolean = parent.isAllowMultiples(); + Boolean anyParentAllowsMultiplesBoolean = false; + DatasetFieldType currentDatasetFieldType = this; + // Traverse up through all parents of dataset field type + // If any one of them allows multiples, this child's Solr field must be multi-valued + while (currentDatasetFieldType.isHasParent()) { + if (currentDatasetFieldType.getParentDatasetFieldType() != null) { + DatasetFieldType parent = currentDatasetFieldType.getParentDatasetFieldType(); + if (parent.isAllowMultiples()) { + anyParentAllowsMultiplesBoolean = true; + break; // no need to keep traversing + } + currentDatasetFieldType = parent; + } else { + break; } } boolean makeSolrFieldMultivalued; // http://stackoverflow.com/questions/5800762/what-is-the-use-of-multivalued-field-type-in-solr - if (allowMultiples || parentAllowsMultiplesBoolean || isControlledVocabulary()) { + if (allowMultiples || anyParentAllowsMultiplesBoolean || isControlledVocabulary()) { makeSolrFieldMultivalued = true; } else { makeSolrFieldMultivalued = false; diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetFieldValueValidator.java b/src/main/java/edu/harvard/iq/dataverse/DatasetFieldValueValidator.java index 610bb70ff49..74d3cbf73f0 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetFieldValueValidator.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetFieldValueValidator.java @@ -241,10 +241,6 @@ private boolean isValidDate(String dateString, String pattern) { return valid; } - public boolean isValidAuthorIdentifier(String userInput, Pattern pattern) { - return pattern.matcher(userInput).matches(); - } - // Validate child fields against each other and return failure message or Optional.empty() if success public Optional validateChildConstraints(DatasetField dsf) { final String fieldName = dsf.getDatasetFieldType().getName() != null ? dsf.getDatasetFieldType().getName() : ""; diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java index 33a093c8044..af8cdc21968 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java @@ -57,6 +57,7 @@ import edu.harvard.iq.dataverse.settings.SettingsServiceBean; import edu.harvard.iq.dataverse.util.ArchiverUtil; import edu.harvard.iq.dataverse.util.BundleUtil; +import edu.harvard.iq.dataverse.util.CSLUtil; import edu.harvard.iq.dataverse.util.DataFileComparator; import edu.harvard.iq.dataverse.util.FileSortFieldAndOrder; import edu.harvard.iq.dataverse.util.FileUtil; @@ -163,7 +164,7 @@ import edu.harvard.iq.dataverse.util.FileMetadataUtil; import java.util.Comparator; import org.apache.solr.client.solrj.SolrQuery; -import org.apache.solr.client.solrj.impl.HttpSolrClient; +import org.apache.solr.client.solrj.impl.BaseHttpSolrClient.RemoteSolrException; import org.apache.solr.client.solrj.response.FacetField; import org.apache.solr.client.solrj.response.QueryResponse; import org.apache.solr.common.SolrDocument; @@ -184,7 +185,7 @@ public class DatasetPage implements java.io.Serializable { public enum EditMode { - CREATE, INFO, FILE, METADATA, LICENSE + CREATE, INFO, FILE, METADATA, LICENSE, VERSIONNOTE }; public enum DisplayMode { @@ -1041,7 +1042,7 @@ public Set getFileIdsInVersionFromSolr(Long datasetVersionId, String patte try { queryResponse = solrClientService.getSolrClient().query(solrQuery); - } catch (HttpSolrClient.RemoteSolrException ex) { + } catch (RemoteSolrException ex) { logger.fine("Remote Solr Exception: " + ex.getLocalizedMessage()); String msg = ex.getLocalizedMessage(); if (msg.contains(SearchFields.FILE_DELETED)) { @@ -1855,6 +1856,10 @@ private void updateDatasetFieldInputLevels() { if (dsf != null){ // Yes, call "setInclude" dsf.setInclude(oneDSFieldTypeInputLevel.isInclude()); + Boolean displayOnCreate = oneDSFieldTypeInputLevel.getDisplayOnCreate(); + if (displayOnCreate!= null) { + dsf.getDatasetFieldType().setLocalDisplayOnCreate(displayOnCreate); + } // remove from hash mapDatasetFields.remove(oneDSFieldTypeInputLevel.getDatasetFieldType().getId()); } @@ -1985,6 +1990,7 @@ private String init(boolean initFull) { setDataverseSiteUrl(systemConfig.getDataverseSiteUrl()); guestbookResponse = new GuestbookResponse(); + anonymizedAccess = null; String sortOrder = getSortOrder(); if(sortOrder != null) { @@ -2118,6 +2124,7 @@ private String init(boolean initFull) { if (workingVersion.isDraft() && canUpdateDataset()) { readOnly = false; } + publishDialogVersionNote = workingVersion.getVersionNote(); // This will default to all the files in the version, if the search term // parameter hasn't been specified yet: fileMetadatasSearch = selectFileMetadatasForDisplay(); @@ -2609,7 +2616,7 @@ private void resetVersionUI() { } } - String creatorOrcidId = au.getOrcidId(); + String creatorOrcidId = au.getAuthenticatedOrcid(); if (dsf.getDatasetFieldType().getName().equals(DatasetFieldConstant.author) && dsf.isEmpty()) { for (DatasetFieldCompoundValue authorValue : dsf.getDatasetFieldCompoundValues()) { for (DatasetField subField : authorValue.getChildDatasetFields()) { @@ -2774,6 +2781,7 @@ public String releaseDataset() { if(!dataset.getOwner().isReleased()){ releaseParentDV(); } + workingVersion.setVersionNote(publishDialogVersionNote); if(publishDatasetPopup()|| publishBothPopup() || !dataset.getLatestVersion().isMinorUpdate()){ return releaseDataset(false); } @@ -2840,35 +2848,35 @@ private DatasetVersion setDatasetVersionDeaccessionReasonAndURL(DatasetVersion d String deacessionReasonDetail = getDeaccessionReasonText() != null ? ( getDeaccessionReasonText()).trim() : ""; switch (deaccessionReasonCode) { case 1: - dvIn.setVersionNote(BundleUtil.getStringFromBundle("file.deaccessionDialog.reason.selectItem.identifiable") ); + dvIn.setDeaccessionNote(BundleUtil.getStringFromBundle("file.deaccessionDialog.reason.selectItem.identifiable") ); break; case 2: - dvIn.setVersionNote(BundleUtil.getStringFromBundle("file.deaccessionDialog.reason.selectItem.beRetracted") ); + dvIn.setDeaccessionNote(BundleUtil.getStringFromBundle("file.deaccessionDialog.reason.selectItem.beRetracted") ); break; case 3: - dvIn.setVersionNote(BundleUtil.getStringFromBundle("file.deaccessionDialog.reason.selectItem.beTransferred") ); + dvIn.setDeaccessionNote(BundleUtil.getStringFromBundle("file.deaccessionDialog.reason.selectItem.beTransferred") ); break; case 4: - dvIn.setVersionNote(BundleUtil.getStringFromBundle("file.deaccessionDialog.reason.selectItem.IRB")); + dvIn.setDeaccessionNote(BundleUtil.getStringFromBundle("file.deaccessionDialog.reason.selectItem.IRB")); break; case 5: - dvIn.setVersionNote(BundleUtil.getStringFromBundle("file.deaccessionDialog.reason.selectItem.legalIssue")); + dvIn.setDeaccessionNote(BundleUtil.getStringFromBundle("file.deaccessionDialog.reason.selectItem.legalIssue")); break; case 6: - dvIn.setVersionNote(BundleUtil.getStringFromBundle("file.deaccessionDialog.reason.selectItem.notValid")); + dvIn.setDeaccessionNote(BundleUtil.getStringFromBundle("file.deaccessionDialog.reason.selectItem.notValid")); break; case 7: break; } if (!deacessionReasonDetail.isEmpty()){ - if (!StringUtil.isEmpty(dvIn.getVersionNote())){ - dvIn.setVersionNote(dvIn.getVersionNote() + " " + deacessionReasonDetail); + if (!StringUtil.isEmpty(dvIn.getDeaccessionNote())){ + dvIn.setDeaccessionNote(dvIn.getDeaccessionNote() + " " + deacessionReasonDetail); } else { - dvIn.setVersionNote(deacessionReasonDetail); + dvIn.setDeaccessionNote(deacessionReasonDetail); } } - dvIn.setArchiveNote(getDeaccessionForwardURLFor()); + dvIn.setDeaccessionLink(getDeaccessionForwardURLFor()); return dvIn; } @@ -3934,7 +3942,7 @@ public void validateForwardURL(FacesContext context, UIComponent toValidate, Obj return; } - if (value.toString().length() <= DatasetVersion.ARCHIVE_NOTE_MAX_LENGTH) { + if (value.toString().length() <= DatasetVersion.DEACCESSION_NOTE_MAX_LENGTH) { ((UIInput) toValidate).setValid(true); } else { ((UIInput) toValidate).setValid(false); @@ -4105,8 +4113,9 @@ public String save() { } if (editMode.equals(EditMode.FILE)) { JsfHelper.addSuccessMessage(BundleUtil.getStringFromBundle("dataset.message.filesSuccess")); + } if (editMode.equals(EditMode.VERSIONNOTE)) { + JsfHelper.addSuccessMessage(BundleUtil.getStringFromBundle("dataset.message.versionNoteSuccess")); } - } else { // must have been a bulk file update or delete: if (bulkFileDeleteInProgress) { @@ -5695,7 +5704,7 @@ public String getPrivateUrlLink(PrivateUrl privateUrl) { public boolean isAnonymizedAccess() { if (anonymizedAccess == null) { - if (session.getUser() instanceof PrivateUrlUser) { + if (session.getUser() instanceof PrivateUrlUser && workingVersion.isDraft()) { anonymizedAccess = ((PrivateUrlUser) session.getUser()).hasAnonymizedAccess(); } else { anonymizedAccess = false; @@ -5719,6 +5728,22 @@ public boolean isAnonymizedAccessEnabled() { return false; } } + + String anonymizedFieldTypeNames = null; + + public String getAnonymizedFieldTypeNames() { + if (anonymizedFieldTypeNames != null) { + return anonymizedFieldTypeNames; + } + if (settingsWrapper.getValueForKey(SettingsServiceBean.Key.AnonymizedFieldTypeNames) != null) { + anonymizedFieldTypeNames = settingsWrapper.getValueForKey(SettingsServiceBean.Key.AnonymizedFieldTypeNames); + + } else { + anonymizedFieldTypeNames = ""; + + } + return anonymizedFieldTypeNames; + } // todo: we should be able to remove - this is passed in the html pages to other fragments, but they could just access this service bean directly. public FileDownloadServiceBean getFileDownloadService() { @@ -5846,13 +5871,12 @@ public List getDatasetSummaryFields() { return DatasetUtil.getDatasetSummaryFields(workingVersion, customFields); } - public boolean isShowPreviewButton(Long fileId) { - List previewTools = getPreviewToolsForDataFile(fileId); + public boolean isShowPreviewButton(DataFile dataFile) { + List previewTools = getPreviewToolsForDataFile(dataFile); return previewTools.size() > 0; } - public boolean isShowQueryButton(Long fileId) { - DataFile dataFile = datafileService.find(fileId); + public boolean isShowQueryButton(DataFile dataFile) { if(dataFile.isRestricted() || !dataFile.isReleased() @@ -5861,26 +5885,28 @@ public boolean isShowQueryButton(Long fileId) { return false; } - List fileQueryTools = getQueryToolsForDataFile(fileId); + List fileQueryTools = getQueryToolsForDataFile(dataFile); return fileQueryTools.size() > 0; } - public List getPreviewToolsForDataFile(Long fileId) { - return getCachedToolsForDataFile(fileId, ExternalTool.Type.PREVIEW); + public List getPreviewToolsForDataFile(DataFile dataFile) { + return getCachedToolsForDataFile(dataFile, ExternalTool.Type.PREVIEW); } - public List getQueryToolsForDataFile(Long fileId) { - return getCachedToolsForDataFile(fileId, ExternalTool.Type.QUERY); + public List getQueryToolsForDataFile(DataFile dataFile) { + return getCachedToolsForDataFile(dataFile, ExternalTool.Type.QUERY); } - public List getConfigureToolsForDataFile(Long fileId) { - return getCachedToolsForDataFile(fileId, ExternalTool.Type.CONFIGURE); + + public List getConfigureToolsForDataFile(DataFile dataFile) { + return getCachedToolsForDataFile(dataFile, ExternalTool.Type.CONFIGURE); } - public List getExploreToolsForDataFile(Long fileId) { - return getCachedToolsForDataFile(fileId, ExternalTool.Type.EXPLORE); + public List getExploreToolsForDataFile(DataFile dataFile) { + return getCachedToolsForDataFile(dataFile, ExternalTool.Type.EXPLORE); } - public List getCachedToolsForDataFile(Long fileId, ExternalTool.Type type) { + public List getCachedToolsForDataFile(DataFile dataFile, ExternalTool.Type type) { + Long fileId = dataFile.getId(); Map> cachedToolsByFileId = new HashMap<>(); List externalTools = new ArrayList<>(); switch (type) { @@ -5907,7 +5933,6 @@ public List getCachedToolsForDataFile(Long fileId, ExternalTool.Ty if (cachedTools != null) { //if already queried before and added to list return cachedTools; } - DataFile dataFile = datafileService.find(fileId); cachedTools = externalToolService.findExternalToolsByFile(externalTools, dataFile); cachedToolsByFileId.put(fileId, cachedTools); //add to map so we don't have to do the lifting again return cachedTools; @@ -6214,7 +6239,7 @@ public String getEffectiveMetadataLanguage(boolean ofParent) { public String getLocaleDisplayName(String code) { String displayName = settingsWrapper.getBaseMetadataLanguageMap(false).get(code); - if(displayName==null && !code.equals(DvObjectContainer.UNDEFINED_CODE)) { + if(displayName==null && code!=null && !code.equals(DvObjectContainer.UNDEFINED_CODE)) { //Default (for cases such as :when a Dataset has a metadatalanguage code but :MetadataLanguages is no longer defined). displayName = new Locale(code).getDisplayName(); } @@ -6230,6 +6255,11 @@ public List getVocabScripts() { } public String getFieldLanguage(String languages) { + //Prevent NPE in Payara 6-2024-12 with CVoc + logger.fine("Languages: " + languages); + if(languages==null) { + languages=""; + } return fieldService.getFieldLanguage(languages,session.getLocaleCode()); } @@ -6702,6 +6732,7 @@ public boolean isGlobusTransferRequested() { * valid files to transfer. */ public void startGlobusTransfer(boolean transferAll, boolean popupShown) { + logger.fine("inside startGlobusTransfer; "+(transferAll ? "transferAll" : "NOTtransferAll") + " " + (popupShown ? "popupShown" : "NOTpopupShown")); if (transferAll) { this.setSelectedFiles(workingVersion.getFileMetadatas()); } @@ -6763,5 +6794,31 @@ public String getSignpostingLinkHeader() { public boolean isDOI() { return AbstractDOIProvider.DOI_PROTOCOL.equals(dataset.getGlobalId().getProtocol()); } + + public void saveVersionNote() { + this.editMode=EditMode.VERSIONNOTE; + publishDialogVersionNote = workingVersion.getVersionNote(); + save(); + } + String publishDialogVersionNote = null; + + // Make separate property for versionNote - can't have two p:dialogs changing the same property + public String getPublishDialogVersionNote() { + return publishDialogVersionNote; + } + + public void setPublishDialogVersionNote(String note) { + publishDialogVersionNote =note; + } + + String requestedCSL = CSLUtil.getDefaultStyle(); + + public String getRequestedCSL() { + return requestedCSL; + } + + public void setRequestedCSL(String requestedCSL) { + this.requestedCSL = requestedCSL; + } } diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DatasetServiceBean.java index e519614ba55..9a8c43668cb 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetServiceBean.java @@ -1092,4 +1092,12 @@ public List getVersionStates(long id) { } } + /** + * Returns the total number of Datasets. + * @return the number of datasets in the database + */ + public long getDatasetCount() { + return em.createNamedQuery("Dataset.countAll", Long.class).getSingleResult(); + } + } diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetVersion.java b/src/main/java/edu/harvard/iq/dataverse/DatasetVersion.java index a7bbc7c3ad4..6c1000f2170 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetVersion.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetVersion.java @@ -8,7 +8,6 @@ import edu.harvard.iq.dataverse.branding.BrandingUtil; import edu.harvard.iq.dataverse.dataset.DatasetUtil; import edu.harvard.iq.dataverse.license.License; -import edu.harvard.iq.dataverse.pidproviders.PidUtil; import edu.harvard.iq.dataverse.util.FileUtil; import edu.harvard.iq.dataverse.util.StringUtil; import edu.harvard.iq.dataverse.util.SystemConfig; @@ -17,6 +16,9 @@ import edu.harvard.iq.dataverse.util.json.NullSafeJsonBuilder; import edu.harvard.iq.dataverse.workflows.WorkflowComment; import java.io.Serializable; +import java.net.MalformedURLException; +import java.net.URI; +import java.net.URISyntaxException; import java.sql.Timestamp; import java.text.ParseException; import java.text.SimpleDateFormat; @@ -80,7 +82,7 @@ @Entity @Table(indexes = {@Index(columnList="dataset_id")}, uniqueConstraints = @UniqueConstraint(columnNames = {"dataset_id,versionnumber,minorversionnumber"})) -@ValidateVersionNote(versionNote = "versionNote", versionState = "versionState") +@ValidateDeaccessionNote(deaccessionNote = "deaccessionNote", versionState = "versionState") public class DatasetVersion implements Serializable { private static final Logger logger = Logger.getLogger(DatasetVersion.class.getCanonicalName()); @@ -114,7 +116,8 @@ public enum VersionState { DRAFT, RELEASED, ARCHIVED, DEACCESSIONED } - public static final int ARCHIVE_NOTE_MAX_LENGTH = 1000; + public static final int DEACCESSION_NOTE_MAX_LENGTH = 1000; + public static final int DEACCESSION_LINK_MAX_LENGTH = 1260; //Long enough to cover the case where a legacy deaccessionLink(256 char) and archiveNote (1000) are combined (with a space) public static final int VERSION_NOTE_MAX_LENGTH = 1000; //Archival copies: Status message required components @@ -137,10 +140,16 @@ public enum VersionState { private Long versionNumber; private Long minorVersionNumber; + //This is used for the deaccession reason + @Size(min=0, max=DEACCESSION_NOTE_MAX_LENGTH) + @Column(length = DEACCESSION_NOTE_MAX_LENGTH) + private String deaccessionNote; + + //This is a plain text, optional reason for the version's creation @Size(min=0, max=VERSION_NOTE_MAX_LENGTH) @Column(length = VERSION_NOTE_MAX_LENGTH) private String versionNote; - + /* * @todo versionState should never be null so when we are ready, uncomment * the `nullable = false` below. @@ -177,12 +186,6 @@ public enum VersionState { @Temporal(value = TemporalType.TIMESTAMP) private Date archiveTime; - @Size(min=0, max=ARCHIVE_NOTE_MAX_LENGTH) - @Column(length = ARCHIVE_NOTE_MAX_LENGTH) - //@ValidateURL() - this validation rule was making a bunch of older legacy datasets invalid; - // removed pending further investigation (v4.13) - private String archiveNote; - // Originally a simple string indicating the location of the archival copy. As // of v5.12, repurposed to provide a more general json archival status (failure, // pending, success) and message (serialized as a string). The archival copy @@ -191,7 +194,9 @@ public enum VersionState { @Column(nullable=true, columnDefinition = "TEXT") private String archivalCopyLocation; - + //This is used for the deaccession reason + @Size(min=0, max=DEACCESSION_LINK_MAX_LENGTH) + @Column(length = DEACCESSION_LINK_MAX_LENGTH) private String deaccessionLink; @Transient @@ -361,19 +366,6 @@ public void setArchiveTime(Date archiveTime) { this.archiveTime = archiveTime; } - public String getArchiveNote() { - return archiveNote; - } - - public void setArchiveNote(String note) { - // @todo should this be using bean validation for trsting note length? - if (note != null && note.length() > ARCHIVE_NOTE_MAX_LENGTH) { - throw new IllegalArgumentException("Error setting archiveNote: String length is greater than maximum (" + ARCHIVE_NOTE_MAX_LENGTH + ")." - + " StudyVersion id=" + id + ", archiveNote=" + note); - } - this.archiveNote = note; - } - public String getArchivalCopyLocation() { return archivalCopyLocation; } @@ -417,11 +409,21 @@ public String getDeaccessionLink() { } public void setDeaccessionLink(String deaccessionLink) { + if (deaccessionLink != null && deaccessionLink.length() > DEACCESSION_LINK_MAX_LENGTH) { + throw new IllegalArgumentException("Error setting deaccessionLink: String length is greater than maximum (" + DEACCESSION_LINK_MAX_LENGTH + ")." + + " StudyVersion id=" + id + ", deaccessionLink=" + deaccessionLink); + } this.deaccessionLink = deaccessionLink; } - public GlobalId getDeaccessionLinkAsGlobalId() { - return PidUtil.parseAsGlobalID(deaccessionLink); + public String getDeaccessionLinkAsURLString() { + String dLink = null; + try { + dLink = new URI(deaccessionLink).toURL().toExternalForm(); + } catch (URISyntaxException | MalformedURLException e) { + logger.fine("Invalid deaccessionLink - not a URL: " + deaccessionLink); + } + return dLink; } public Date getCreateTime() { @@ -490,8 +492,8 @@ public void setContributorNames(String contributorNames) { } - public String getVersionNote() { - return versionNote; + public String getDeaccessionNote() { + return deaccessionNote; } public DatasetVersionDifference getDefaultVersionDifference() { @@ -541,12 +543,12 @@ public VersionState getPriorVersionState() { return null; } - public void setVersionNote(String note) { - if (note != null && note.length() > VERSION_NOTE_MAX_LENGTH) { - throw new IllegalArgumentException("Error setting versionNote: String length is greater than maximum (" + VERSION_NOTE_MAX_LENGTH + ")." - + " StudyVersion id=" + id + ", versionNote=" + note); + public void setDeaccessionNote(String note) { + if (note != null && note.length() > DEACCESSION_NOTE_MAX_LENGTH) { + throw new IllegalArgumentException("Error setting deaccessionNote: String length is greater than maximum (" + DEACCESSION_NOTE_MAX_LENGTH + ")." + + " StudyVersion id=" + id + ", deaccessionNote=" + note); } - this.versionNote = note; + this.deaccessionNote = note; } public Long getVersionNumber() { @@ -1483,11 +1485,14 @@ public String getCitation() { } public String getCitation(boolean html) { - return getCitation(html, false); + return getCitation(DataCitation.Format.Internal, html, false); } - public String getCitation(boolean html, boolean anonymized) { - return new DataCitation(this).toString(html, anonymized); + return getCitation(DataCitation.Format.Internal, html, anonymized); + } + + public String getCitation(DataCitation.Format format, boolean html, boolean anonymized) { + return new DataCitation(this).toString(format, html, anonymized); } public Date getCitationDate() { @@ -2158,4 +2163,17 @@ public void setExternalStatusLabel(String externalStatusLabel) { this.externalStatusLabel = externalStatusLabel; } + public String getVersionNote() { + return versionNote; + } + + public void setVersionNote(String note) { + if (note != null && note.length() > VERSION_NOTE_MAX_LENGTH) { + throw new IllegalArgumentException("Error setting versionNote: String length is greater than maximum (" + VERSION_NOTE_MAX_LENGTH + ")." + + " StudyVersion id=" + id + ", versionNote=" + note); + } + + this.versionNote = note; + } } + diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionDifference.java b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionDifference.java index c5d6c31386c..3ea6e662751 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionDifference.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionDifference.java @@ -46,8 +46,8 @@ public final class DatasetVersionDifference { private List changedVariableMetadata = new ArrayList<>(); private List replacedFiles = new ArrayList<>(); private List changedTermsAccess = new ArrayList<>(); - private List summaryDataForNote = new ArrayList<>(); - private List blockDataForNote = new ArrayList<>(); + private List summaryDataForNote = new ArrayList<>(); + private List blockDataForNote = new ArrayList<>(); private List differenceSummaryGroups = new ArrayList<>(); @@ -300,32 +300,33 @@ private void addToSummary(DatasetField dsfo, DatasetField dsfn) { private void updateBlockSummary(DatasetField dsf, int added, int deleted, int changed) { boolean addedToAll = false; - for (Object[] blockList : blockDataForNote) { - DatasetField dsft = (DatasetField) blockList[0]; + for (SummaryNote blockList : blockDataForNote) { + + DatasetField dsft = blockList.dsfo; if (dsft.getDatasetFieldType().getMetadataBlock().equals(dsf.getDatasetFieldType().getMetadataBlock())) { - blockList[1] = (Integer) blockList[1] + added; - blockList[2] = (Integer) blockList[2] + deleted; - blockList[3] = (Integer) blockList[3] + changed; + blockList.added = blockList.added + added; + blockList.deleted = blockList.deleted + deleted; + blockList.changed = blockList.changed + changed; addedToAll = true; } } if (!addedToAll) { - Object[] newArray = new Object[4]; - newArray[0] = dsf; - newArray[1] = added; - newArray[2] = deleted; - newArray[3] = changed; - blockDataForNote.add(newArray); + SummaryNote newNote = new SummaryNote(); + newNote.dsfo = dsf; + newNote.added = added; + newNote.deleted = deleted; + newNote.changed = changed; + blockDataForNote.add(newNote); } } - private void addToNoteSummary(DatasetField dsfo, int added, int deleted, int changed) { - Object[] noteArray = new Object[4]; - noteArray[0] = dsfo; - noteArray[1] = added; - noteArray[2] = deleted; - noteArray[3] = changed; - summaryDataForNote.add(noteArray); + private void addToNoteSummary(DatasetField dsfo, Integer added, Integer deleted, Integer changed) { + SummaryNote summaryNote = new SummaryNote(); + summaryNote.dsfo = dsfo; + summaryNote.added = added; + summaryNote.deleted = deleted; + summaryNote.changed = changed; + summaryDataForNote.add(summaryNote); } static boolean compareVarGroup(FileMetadata fmdo, FileMetadata fmdn) { @@ -363,19 +364,24 @@ public static Map> compareFileMetadatas(FileMetadata fmdo, F List.of(StringUtil.nullToEmpty(fmdo.getDescription()), StringUtil.nullToEmpty(fmdn.getDescription()))); } - if (!StringUtils.equals(fmdo.getCategoriesByName().toString(), fmdn.getCategoriesByName().toString())) { + if (!StringUtils.equals(StringUtil.nullToEmpty(fmdo.getCategoriesByName().toString()), StringUtil.nullToEmpty(fmdn.getCategoriesByName().toString()))) { fileMetadataChanged.put("Categories", List.of(fmdo.getCategoriesByName().toString(), fmdn.getCategoriesByName().toString())); } - if (!StringUtils.equals(fmdo.getLabel(), fmdn.getLabel())) { + if (!StringUtils.equals(StringUtil.nullToEmpty(fmdo.getLabel()), StringUtil.nullToEmpty(fmdn.getLabel()))) { fileMetadataChanged.put("Label", - List.of(fmdo.getLabel(), fmdn.getLabel())); + List.of(StringUtil.nullToEmpty(fmdo.getLabel()), StringUtil.nullToEmpty(fmdn.getLabel()))); + } + + if (!StringUtils.equals(StringUtil.nullToEmpty(fmdo.getDirectoryLabel()), StringUtil.nullToEmpty(fmdn.getDirectoryLabel()))) { + fileMetadataChanged.put("File Path", + List.of(StringUtil.nullToEmpty(fmdo.getDirectoryLabel()), StringUtil.nullToEmpty(fmdn.getDirectoryLabel()))); } - if (!StringUtils.equals(fmdo.getProvFreeForm(), fmdn.getProvFreeForm())) { + if (!StringUtils.equals(StringUtil.nullToEmpty(fmdo.getProvFreeForm()), StringUtil.nullToEmpty(fmdn.getProvFreeForm()))) { fileMetadataChanged.put("ProvFreeForm", - List.of(fmdo.getProvFreeForm(), fmdn.getProvFreeForm())); + List.of(StringUtil.nullToEmpty(fmdo.getProvFreeForm()), StringUtil.nullToEmpty(fmdn.getProvFreeForm()))); } if (fmdo.isRestricted() != fmdn.isRestricted()) { @@ -464,6 +470,8 @@ private void compareValues(DatasetField originalField, DatasetField newField, bo } } } + + public String getFileNote() { String retString = ""; @@ -568,19 +576,19 @@ public void setChangedFileMetadata(List changedFileMetadata) { this.changedFileMetadata = changedFileMetadata; } - public List getSummaryDataForNote() { + public List getSummaryDataForNote() { return summaryDataForNote; } - public List getBlockDataForNote() { + public List getBlockDataForNote() { return blockDataForNote; } - public void setSummaryDataForNote(List summaryDataForNote) { + public void setSummaryDataForNote(List summaryDataForNote) { this.summaryDataForNote = summaryDataForNote; } - public void setBlockDataForNote(List blockDataForNote) { + public void setBlockDataForNote(List blockDataForNote) { this.blockDataForNote = blockDataForNote; } @@ -1205,6 +1213,47 @@ public List getDifferenceSummaryItems() { public void setDifferenceSummaryItems(List differenceSummaryItems) { this.differenceSummaryItems = differenceSummaryItems; } + } + + public class SummaryNote { + DatasetField dsfo; + Integer added; + Integer deleted; + Integer changed; + + public void setDatasetField(DatasetField dsfIn){ + dsfo = dsfIn; + } + + public DatasetField getDatasetField (){ + return dsfo; + } + + public void setAdded(Integer addin){ + added = addin; + } + + public Integer getAdded(){ + return added; + } + + public void setDeleted(Integer delin){ + deleted = delin; + } + + public Integer getDeleted(){ + return deleted; + } + + public void setChanged(Integer changedin){ + changed = changedin; + } + + public Integer getChanged(){ + return changed; + } + + } public class DifferenceSummaryItem { @@ -1641,6 +1690,93 @@ List getChangedVariableMetadata() { List getReplacedFiles() { return replacedFiles; } + + public JsonObjectBuilder getSummaryDifferenceAsJson(){ + + JsonObjectBuilder jobVersion = new NullSafeJsonBuilder(); + + JsonObjectBuilder jobDsfOnCreate = new NullSafeJsonBuilder(); + + + for (SummaryNote sn : this.summaryDataForNote) { + jobDsfOnCreate.add( sn.getDatasetField().getDatasetFieldType().getDisplayName(), getSummaryNoteAsJson(sn)); + } + + if (!this.summaryDataForNote.isEmpty()){ + + jobVersion.add("Citation Metadata", jobDsfOnCreate); + + } + + for (SummaryNote sn : this.getBlockDataForNote()){ + String mdbDisplayName = sn.getDatasetField().getDatasetFieldType().getMetadataBlock().getDisplayName(); + if (mdbDisplayName.equals("Citation Metadata")){ + mdbDisplayName = "Additional Citation Metadata"; + } + jobVersion.add( mdbDisplayName, getSummaryNoteAsJson(sn)); + } + + jobVersion.add("files", getFileSummaryAsJson()); + + if (!this.changedTermsAccess.isEmpty()) { + jobVersion.add("termsAccessChanged", true); + } else{ + jobVersion.add("termsAccessChanged", false); + } + + return jobVersion; + } + + private JsonObjectBuilder getSummaryNoteAsJson(SummaryNote sn){ + JsonObjectBuilder job = new NullSafeJsonBuilder(); + //job.add("datasetFieldType", sn.getDatasetField().getDatasetFieldType().getDisplayName()); + job.add("added", sn.added); + job.add("deleted", sn.deleted); + job.add("changed", sn.changed); + return job; + } + + private JsonObjectBuilder getFileSummaryAsJson(){ + JsonObjectBuilder job = new NullSafeJsonBuilder(); + + if (!addedFiles.isEmpty()) { + job.add("added", addedFiles.size()); + + } else{ + job.add("added", 0); + } + + if (!removedFiles.isEmpty()) { + job.add("removed", removedFiles.size()); + + } else{ + job.add("removed", 0); + } + + if (!replacedFiles.isEmpty()) { + job.add("replaced", replacedFiles.size()); + + } else{ + job.add("replaced", 0); + } + + if (!changedFileMetadata.isEmpty()) { + job.add("changedFileMetaData", changedFileMetadata.size()); + + } else{ + job.add("changedFileMetaData", 0); + } + + if (!changedVariableMetadata.isEmpty()) { + job.add("changedVariableMetadata", changedVariableMetadata.size()); + + } else{ + job.add("changedVariableMetadata", 0); + } + + return job; + } + public JsonObjectBuilder compareVersionsAsJson() { JsonObjectBuilder job = new NullSafeJsonBuilder(); JsonObjectBuilder jobVersion = new NullSafeJsonBuilder(); diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionServiceBean.java index 762319884b9..7e9b778c6f3 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionServiceBean.java @@ -1060,42 +1060,6 @@ public boolean doesChecksumExistInDatasetVersion(DatasetVersion datasetVersion, } - public List> getBasicDatasetVersionInfo(Dataset dataset){ - - if (dataset == null){ - throw new NullPointerException("dataset cannot be null"); - } - - String query = "SELECT id, dataset_id, releasetime, versionnumber," - + " minorversionnumber, versionstate, versionnote" - + " FROM datasetversion" - + " WHERE dataset_id = " + dataset.getId() - + " ORDER BY versionnumber DESC," - + " minorversionnumber DESC," - + " versionstate;"; - msg("query: " + query); - Query nativeQuery = em.createNativeQuery(query); - List datasetVersionInfoList = nativeQuery.getResultList(); - - List> hashList = new ArrayList<>(); - - HashMap mMap = new HashMap<>(); - for (Object[] dvInfo : datasetVersionInfoList) { - mMap = new HashMap<>(); - mMap.put("datasetVersionId", dvInfo[0]); - mMap.put("datasetId", dvInfo[1]); - mMap.put("releaseTime", dvInfo[2]); - mMap.put("versionnumber", dvInfo[3]); - mMap.put("minorversionnumber", dvInfo[4]); - mMap.put("versionstate", dvInfo[5]); - mMap.put("versionnote", dvInfo[6]); - hashList.add(mMap); - } - return hashList; - } // end getBasicDatasetVersionInfo - - - public HashMap getFileMetadataHistory(DataFile df){ if (df == null){ diff --git a/src/main/java/edu/harvard/iq/dataverse/Dataverse.java b/src/main/java/edu/harvard/iq/dataverse/Dataverse.java index 1f11725e581..8ab6f537aca 100644 --- a/src/main/java/edu/harvard/iq/dataverse/Dataverse.java +++ b/src/main/java/edu/harvard/iq/dataverse/Dataverse.java @@ -1,5 +1,6 @@ package edu.harvard.iq.dataverse; +import edu.harvard.iq.dataverse.dataverse.featured.DataverseFeaturedItem; import edu.harvard.iq.dataverse.harvest.client.HarvestingClient; import edu.harvard.iq.dataverse.authorization.DataverseRole; import edu.harvard.iq.dataverse.search.savedsearch.SavedSearch; @@ -54,7 +55,8 @@ @NamedQuery(name = "Dataverse.findByReleaseUserId", query="select object(o) from Dataverse as o where o.releaseUser.id =:releaseUserId order by o.name"), @NamedQuery(name = "Dataverse.filterByAlias", query="SELECT dv FROM Dataverse dv WHERE LOWER(dv.alias) LIKE :alias order by dv.alias"), @NamedQuery(name = "Dataverse.filterByAliasNameAffiliation", query="SELECT dv FROM Dataverse dv WHERE (LOWER(dv.alias) LIKE :alias) OR (LOWER(dv.name) LIKE :name) OR (LOWER(dv.affiliation) LIKE :affiliation) order by dv.alias"), - @NamedQuery(name = "Dataverse.filterByName", query="SELECT dv FROM Dataverse dv WHERE LOWER(dv.name) LIKE :name order by dv.alias") + @NamedQuery(name = "Dataverse.filterByName", query="SELECT dv FROM Dataverse dv WHERE LOWER(dv.name) LIKE :name order by dv.alias"), + @NamedQuery(name = "Dataverse.countAll", query = "SELECT COUNT(dv) FROM Dataverse dv") }) @Entity @Table(indexes = {@Index(columnList="defaultcontributorrole_id") @@ -351,6 +353,17 @@ public void setMetadataBlockFacets(List metadataBlo this.metadataBlockFacets = metadataBlockFacets; } + @OneToMany(mappedBy = "dataverse") + private List dataverseFeaturedItems = new ArrayList<>(); + + public List getDataverseFeaturedItems() { + return this.dataverseFeaturedItems; + } + + public void setDataverseFeaturedItems(List dataverseFeaturedItems) { + this.dataverseFeaturedItems = dataverseFeaturedItems; + } + public List getParentGuestbooks() { List retList = new ArrayList<>(); Dataverse testDV = this; @@ -425,6 +438,19 @@ public boolean isDatasetFieldTypeInInputLevels(Long datasetFieldTypeId) { .anyMatch(inputLevel -> inputLevel.getDatasetFieldType().getId().equals(datasetFieldTypeId)); } + public DataverseFieldTypeInputLevel getDatasetFieldTypeInInputLevels(Long datasetFieldTypeId) { + return dataverseFieldTypeInputLevels.stream() + .filter(inputLevel -> inputLevel.getDatasetFieldType().getId().equals(datasetFieldTypeId)) + .findFirst() + .orElse(null); + } + + public boolean isDatasetFieldTypeDisplayOnCreateAsInputLevel(Long datasetFieldTypeId) { + return dataverseFieldTypeInputLevels.stream() + .anyMatch(inputLevel -> inputLevel.getDatasetFieldType().getId().equals(datasetFieldTypeId) + && Boolean.TRUE.equals(inputLevel.getDisplayOnCreate())); + } + public Template getDefaultTemplate() { return defaultTemplate; } @@ -606,7 +632,25 @@ public List getCitationDatasetFieldTypes() { public void setCitationDatasetFieldTypes(List citationDatasetFieldTypes) { this.citationDatasetFieldTypes = citationDatasetFieldTypes; } - + + @Column(nullable = true) + private Boolean requireFilesToPublishDataset; + /** + * Specifies whether the existance of files in a dataset is required when publishing + * @return {@code Boolean.TRUE} if explicitly enabled, {@code Boolean.FALSE} if explicitly disabled. + * {@code null} indicates that the behavior is not explicitly defined, in which + * case the behavior should follow the explicit configuration of the first + * direct ancestor collection. + * @Note: If present, this configuration therefore by default applies to all + * the sub-collections, unless explicitly overwritten there. + */ + public Boolean getRequireFilesToPublishDataset() { + return requireFilesToPublishDataset; + } + public void setRequireFilesToPublishDataset(boolean requireFilesToPublishDataset) { + this.requireFilesToPublishDataset = requireFilesToPublishDataset; + } + /** * @Note: this setting is Nullable, with {@code null} indicating that the * desired behavior is not explicitly configured for this specific collection. @@ -777,6 +821,17 @@ public List getOwners() { return owners; } + public boolean getEffectiveRequiresFilesToPublishDataset() { + Dataverse dv = this; + while (dv != null) { + if (dv.getRequireFilesToPublishDataset() != null) { + return dv.getRequireFilesToPublishDataset(); + } + dv = dv.getOwner(); + } + return false; + } + @Override public boolean equals(Object object) { // TODO: Warning - this method won't work in the case the id fields are not set diff --git a/src/main/java/edu/harvard/iq/dataverse/DataverseFieldTypeInputLevel.java b/src/main/java/edu/harvard/iq/dataverse/DataverseFieldTypeInputLevel.java index a3425987bf8..973a98d246f 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DataverseFieldTypeInputLevel.java +++ b/src/main/java/edu/harvard/iq/dataverse/DataverseFieldTypeInputLevel.java @@ -6,6 +6,8 @@ package edu.harvard.iq.dataverse; import java.io.Serializable; + +import jakarta.persistence.Column; import jakarta.persistence.Entity; import jakarta.persistence.GeneratedValue; import jakarta.persistence.GenerationType; @@ -38,8 +40,9 @@ , uniqueConstraints={ @UniqueConstraint(columnNames={"dataverse_id", "datasetfieldtype_id"})} , indexes = {@Index(columnList="dataverse_id") - , @Index(columnList="datasetfieldtype_id") - , @Index(columnList="required")} + , @Index(columnList="datasetfieldtype_id") + , @Index(columnList="required") + , @Index(columnList="displayOnCreate")} ) @Entity public class DataverseFieldTypeInputLevel implements Serializable { @@ -59,13 +62,18 @@ public class DataverseFieldTypeInputLevel implements Serializable { private boolean include; private boolean required; + + @Column(nullable = true) + private Boolean displayOnCreate; + public DataverseFieldTypeInputLevel () {} - public DataverseFieldTypeInputLevel (DatasetFieldType fieldType, Dataverse dataverse, boolean required, boolean include) { + public DataverseFieldTypeInputLevel (DatasetFieldType fieldType, Dataverse dataverse, boolean required, boolean include, Boolean displayOnCreate) { this.datasetFieldType = fieldType; this.dataverse = dataverse; this.required = required; this.include = include; + this.displayOnCreate = displayOnCreate; } public Long getId() { @@ -115,6 +123,14 @@ public void setRequired(boolean required) { this.required = required; } + public Boolean getDisplayOnCreate() { + return displayOnCreate; + } + + public void setDisplayOnCreate(Boolean displayOnCreate) { + this.displayOnCreate = displayOnCreate; + } + @Override public boolean equals(Object object) { // TODO: Warning - this method won't work in the case the id fields are not set diff --git a/src/main/java/edu/harvard/iq/dataverse/DataverseFieldTypeInputLevelServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DataverseFieldTypeInputLevelServiceBean.java index 1bd290ecc4d..3b4601d173b 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DataverseFieldTypeInputLevelServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/DataverseFieldTypeInputLevelServiceBean.java @@ -104,7 +104,7 @@ public void delete(DataverseFieldTypeInputLevel dataverseFieldTypeInputLevel) { cache.invalidate(); } - public void deleteFacetsFor(Dataverse d) { + public void deleteDataverseFieldTypeInputLevelFor(Dataverse d) { em.createNamedQuery("DataverseFieldTypeInputLevel.removeByOwnerId") .setParameter("ownerId", d.getId()) .executeUpdate(); @@ -117,4 +117,13 @@ public void create(DataverseFieldTypeInputLevel dataverseFieldTypeInputLevel) { em.persist(dataverseFieldTypeInputLevel); } + public DataverseFieldTypeInputLevel save(DataverseFieldTypeInputLevel inputLevel) { + if (inputLevel.getId() == null) { + em.persist(inputLevel); + return inputLevel; + } else { + return em.merge(inputLevel); + } + } + } diff --git a/src/main/java/edu/harvard/iq/dataverse/DataversePage.java b/src/main/java/edu/harvard/iq/dataverse/DataversePage.java index 351d304bad3..4eb9828253d 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DataversePage.java +++ b/src/main/java/edu/harvard/iq/dataverse/DataversePage.java @@ -627,44 +627,17 @@ public String save() { if (dataverse.isMetadataBlockRoot() && (mdb.isSelected() || mdb.isRequired())) { selectedBlocks.add(mdb); for (DatasetFieldType dsft : mdb.getDatasetFieldTypes()) { - // currently we don't allow input levels for setting an optional field as conditionally required - // so we skip looking at parents (which get set automatically with their children) - if (!dsft.isHasChildren() && dsft.isRequiredDV()) { - boolean addRequiredInputLevels = false; - boolean parentAlreadyAdded = false; + if (!dsft.isChild()) { + // Save input level for parent field + saveInputLevels(listDFTIL, dsft, dataverse); - if (!dsft.isHasParent() && dsft.isInclude()) { - addRequiredInputLevels = !dsft.isRequired(); - } - if (dsft.isHasParent() && dsft.getParentDatasetFieldType().isInclude()) { - addRequiredInputLevels = !dsft.isRequired() || !dsft.getParentDatasetFieldType().isRequired(); - } - - if (addRequiredInputLevels) { - listDFTIL.add(new DataverseFieldTypeInputLevel(dsft, dataverse,true, true)); - - //also add the parent as required (if it hasn't been added already) - // todo: review needed .equals() methods, then change this to use a Set, in order to simplify code - if (dsft.isHasParent()) { - DataverseFieldTypeInputLevel parentToAdd = new DataverseFieldTypeInputLevel(dsft.getParentDatasetFieldType(), dataverse, true, true); - for (DataverseFieldTypeInputLevel dataverseFieldTypeInputLevel : listDFTIL) { - if (dataverseFieldTypeInputLevel.getDatasetFieldType().getId() == parentToAdd.getDatasetFieldType().getId()) { - parentAlreadyAdded = true; - break; - } - } - if (!parentAlreadyAdded) { - // Only add the parent once. There's a UNIQUE (dataverse_id, datasetfieldtype_id) - // constraint on the dataversefieldtypeinputlevel table we need to avoid. - listDFTIL.add(parentToAdd); - } - } + // Handle child fields + if (dsft.isHasChildren()) { + for (DatasetFieldType child : dsft.getChildDatasetFieldTypes()) { + saveInputLevels(listDFTIL, child, dataverse); + } } } - if ((!dsft.isHasParent() && !dsft.isInclude()) - || (dsft.isHasParent() && !dsft.getParentDatasetFieldType().isInclude())) { - listDFTIL.add(new DataverseFieldTypeInputLevel(dsft, dataverse,false, false)); - } } } } @@ -1030,27 +1003,11 @@ private void refreshAllMetadataBlocks() { for (DatasetFieldType dsft : mdb.getDatasetFieldTypes()) { if (!dsft.isChild()) { - DataverseFieldTypeInputLevel dsfIl = dataverseFieldTypeInputLevelService.findByDataverseIdDatasetFieldTypeId(dataverseIdForInputLevel, dsft.getId()); - if (dsfIl != null) { - dsft.setRequiredDV(dsfIl.isRequired()); - dsft.setInclude(dsfIl.isInclude()); - } else { - dsft.setRequiredDV(dsft.isRequired()); - dsft.setInclude(true); - } + loadInputLevels(dsft, dataverseIdForInputLevel); dsft.setOptionSelectItems(resetSelectItems(dsft)); if (dsft.isHasChildren()) { for (DatasetFieldType child : dsft.getChildDatasetFieldTypes()) { - DataverseFieldTypeInputLevel dsfIlChild = dataverseFieldTypeInputLevelService.findByDataverseIdDatasetFieldTypeId(dataverseIdForInputLevel, child.getId()); - if (dsfIlChild != null) { - child.setRequiredDV(dsfIlChild.isRequired()); - child.setInclude(dsfIlChild.isInclude()); - } else { - // in the case of conditionally required (child = true, parent = false) - // we set this to false; i.e this is the default "don't override" value - child.setRequiredDV(child.isRequired() && dsft.isRequired()); - child.setInclude(true); - } + loadInputLevels(child, dataverseIdForInputLevel); child.setOptionSelectItems(resetSelectItems(child)); } } @@ -1061,6 +1018,21 @@ private void refreshAllMetadataBlocks() { setAllMetadataBlocks(retList); } + private void loadInputLevels(DatasetFieldType dsft, Long dataverseIdForInputLevel) { + DataverseFieldTypeInputLevel dsfIl = dataverseFieldTypeInputLevelService + .findByDataverseIdDatasetFieldTypeId(dataverseIdForInputLevel, dsft.getId()); + + if (dsfIl != null) { + dsft.setRequiredDV(dsfIl.isRequired()); + dsft.setInclude(dsfIl.isInclude()); + dsft.setLocalDisplayOnCreate(dsfIl.getDisplayOnCreate()); + } else { + // If there is no input level, use the default values + dsft.setRequiredDV(dsft.isRequired()); + dsft.setInclude(true); + } + } + public void validateAlias(FacesContext context, UIComponent toValidate, Object value) { if (!StringUtils.isEmpty((String) value)) { String alias = (String) value; @@ -1337,4 +1309,57 @@ public Set> getPidProviderOptions() { } return options; } + + public void updateDisplayOnCreate(Long mdbId, Long dsftId, boolean currentValue) { + for (MetadataBlock mdb : allMetadataBlocks) { + if (mdb.getId().equals(mdbId)) { + for (DatasetFieldType dsft : mdb.getDatasetFieldTypes()) { + if (dsft.getId().equals(dsftId)) { + // Update value in memory + dsft.setLocalDisplayOnCreate(!currentValue); + + // Update or create input level + DataverseFieldTypeInputLevel existingLevel = dataverseFieldTypeInputLevelService + .findByDataverseIdDatasetFieldTypeId(dataverse.getId(), dsftId); + + if (existingLevel != null) { + existingLevel.setDisplayOnCreate(!currentValue); + dataverseFieldTypeInputLevelService.save(existingLevel); + } else { + DataverseFieldTypeInputLevel newLevel = new DataverseFieldTypeInputLevel( + dsft, + dataverse, + dsft.isRequiredDV(), + true, // default include + !currentValue // new value of displayOnCreate + ); + dataverseFieldTypeInputLevelService.save(newLevel); + } + } + } + } + } + } + + private void saveInputLevels(List listDFTIL, DatasetFieldType dsft, Dataverse dataverse) { + // If the field already has an input level, update it + DataverseFieldTypeInputLevel existingLevel = dataverseFieldTypeInputLevelService + .findByDataverseIdDatasetFieldTypeId(dataverse.getId(), dsft.getId()); + + if (existingLevel != null) { + existingLevel.setDisplayOnCreate(dsft.getLocalDisplayOnCreate()); + existingLevel.setInclude(dsft.isInclude()); + existingLevel.setRequired(dsft.isRequiredDV()); + listDFTIL.add(existingLevel); + } else if (dsft.isInclude() || (dsft.getLocalDisplayOnCreate()!=null) || dsft.isRequiredDV()) { + // Only create new input level if there is any specific configuration + listDFTIL.add(new DataverseFieldTypeInputLevel( + dsft, + dataverse, + dsft.isRequiredDV(), + dsft.isInclude(), + dsft.getLocalDisplayOnCreate() + )); + } + } } diff --git a/src/main/java/edu/harvard/iq/dataverse/DataverseRoleServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DataverseRoleServiceBean.java index 78d5eaf3414..b751841da74 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DataverseRoleServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/DataverseRoleServiceBean.java @@ -23,7 +23,6 @@ import jakarta.persistence.EntityManager; import jakarta.persistence.PersistenceContext; import jakarta.persistence.TypedQuery; -//import jakarta.validation.constraints.NotNull; /** * @@ -40,6 +39,9 @@ public class DataverseRoleServiceBean implements java.io.Serializable { @EJB RoleAssigneeServiceBean roleAssigneeService; + + @EJB + DataverseServiceBean dataverseService; @EJB IndexServiceBean indexService; @EJB @@ -48,22 +50,23 @@ public class DataverseRoleServiceBean implements java.io.Serializable { IndexAsync indexAsync; public DataverseRole save(DataverseRole aRole) { - if (aRole.getId() == null) { + if (aRole.getId() == null) { // persist a new Role em.persist(aRole); - /** - * @todo Why would getId be null? Should we call - * indexDefinitionPoint here too? A: it's null for new roles. - */ - return aRole; - } else { - DataverseRole merged = em.merge(aRole); - /** - * @todo update permissionModificationTime here. - */ - IndexResponse indexDefinitionPountResult = indexDefinitionPoint(merged.getOwner()); - logger.info("aRole getId was not null. Indexing result: " + indexDefinitionPountResult); - return merged; + } else { // update an existing Role + aRole = em.merge(aRole); + } + + DvObject owner = aRole.getOwner(); + if(owner == null) { // Builtin Role + owner = dataverseService.findByAlias("root"); + } + + if(owner != null) { // owner may be null if a role is created before the root collection as in setup-all.sh + IndexResponse indexDefinitionPointResult = indexDefinitionPoint(owner); + logger.info("Indexing result: " + indexDefinitionPointResult); } + + return aRole; } public RoleAssignment save(RoleAssignment assignment) { diff --git a/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java index 91b15f77111..f89e707cc03 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java @@ -267,14 +267,18 @@ public Dataverse findByAlias(String anAlias) { return null; } } - - public boolean hasData( Dataverse dv ) { - TypedQuery amountQry = em.createNamedQuery("Dataverse.ownedObjectsById", Long.class) - .setParameter("id", dv.getId()); - - return (amountQry.getSingleResult()>0); - } - + + public boolean hasData(Dataverse dataverse) { + return (getChildCount(dataverse) > 0); + } + + public Long getChildCount(Dataverse dataverse) { + TypedQuery amountQry = em.createNamedQuery("Dataverse.ownedObjectsById", Long.class) + .setParameter("id", dataverse.getId()); + + return amountQry.getSingleResult(); + } + public boolean isRootDataverseExists() { long count = em.createQuery("SELECT count(dv) FROM Dataverse dv WHERE dv.owner.id=null", Long.class).getSingleResult(); return (count == 1); @@ -948,16 +952,21 @@ public String getCollectionDatasetSchema(String dataverseAlias, Map childrenRequired = new ArrayList<>(); List childrenAllowed = new ArrayList<>(); @@ -967,11 +976,12 @@ public String getCollectionDatasetSchema(String dataverseAlias, Map> map = new HashMap<>(); map.put("required", childrenRequired); map.put("allowed", childrenAllowed); schemaChildMap.put(dsft.getName(), map); } + if(dsft.isRequiredDV()){ requiredDSFT.add(dsft); } } } - } String reqMDBNames = ""; List hasReqFields = new ArrayList<>(); String retval = datasetSchemaPreface; + + // Build list of metadata blocks with required fields for (MetadataBlock mdb : selectedBlocks) { for (DatasetFieldType dsft : requiredDSFT) { if (dsft.getMetadataBlock().equals(mdb)) { @@ -1006,9 +1019,11 @@ public String getCollectionDatasetSchema(String dataverseAlias, Map0){ + if (countMDB > 0) { retval += ","; } retval += getCustomMDBSchema(mdb, requiredDSFT); @@ -1016,44 +1031,56 @@ public String getCollectionDatasetSchema(String dataverseAlias, Map requiredDSFT){ String retval = ""; boolean mdbHasReqField = false; int numReq = 0; List requiredThisMDB = new ArrayList<>(); + List allFieldsThisMDB = new ArrayList<>(mdb.getDatasetFieldTypes()); - for (DatasetFieldType dsft : requiredDSFT ){ - + // First collect all required fields for this metadata block + for (DatasetFieldType dsft : requiredDSFT) { if(dsft.getMetadataBlock().equals(mdb)){ numReq++; mdbHasReqField = true; requiredThisMDB.add(dsft); } } - if (mdbHasReqField){ - retval += startOfMDB.replace("blockName", mdb.getName()); - - retval += minItemsTemplate.replace("numMinItems", Integer.toString(requiredThisMDB.size())); - int count = 0; - for (DatasetFieldType dsft:requiredThisMDB ){ - count++; - String reqValImp = reqValTemplate.replace("reqFieldTypeName", dsft.getName()); - if (count < requiredThisMDB.size()){ - retval += reqValImp + "\n"; - } else { - reqValImp = StringUtils.substring(reqValImp, 0, reqValImp.length() - 1); - retval += reqValImp+ "\n"; - retval += endOfReqVal; - } - } + + // Start building the schema for this metadata block + retval += startOfMDB.replace("blockName", mdb.getName()); + // Add minItems constraint only if there are required fields + if (mdbHasReqField) { + retval += minItemsTemplate.replace("numMinItems", Integer.toString(requiredThisMDB.size())); + + // Add contains validation for each required field + int count = 0; + for (DatasetFieldType dsft : requiredThisMDB) { + count++; + String reqValImp = reqValTemplate.replace("reqFieldTypeName", dsft.getName()); + if (count < requiredThisMDB.size()) { + retval += reqValImp + "\n"; + } else { + reqValImp = StringUtils.substring(reqValImp, 0, reqValImp.length() - 1); + retval += reqValImp + "\n"; + retval += endOfReqVal; + } + } + } else { + // If no required fields, just close the items definition + retval += "\n \"items\": {\n" + + " \"$ref\": \"#/$defs/field\"\n" + + " }\n" + + " }\n" + + " },\n" + + " \"required\": [\"fields\"]\n" + + " }"; } return retval; @@ -1131,6 +1158,9 @@ static String getBaseSchemaStringFromFile(String pathToJsonFile) { " },\n" + " \"typeName\": {\n" + " \"type\": \"string\"\n" + + " },\n" + + " \"displayOnCreate\": {\n" + + " \"type\": \"boolean\"\n" + " }\n" + " }\n" + " }\n" + @@ -1219,4 +1249,12 @@ public void disableStorageQuota(StorageQuota storageQuota) { em.flush(); } } + + /** + * Returns the total number of Dataverses + * @return the number of dataverse in the database + */ + public long getDataverseCount() { + return em.createNamedQuery("Dataverse.countAll", Long.class).getSingleResult(); + } } diff --git a/src/main/java/edu/harvard/iq/dataverse/DataverseSession.java b/src/main/java/edu/harvard/iq/dataverse/DataverseSession.java index e8d76e1825e..607bc9d5a47 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DataverseSession.java +++ b/src/main/java/edu/harvard/iq/dataverse/DataverseSession.java @@ -209,19 +209,17 @@ public String getLocaleTitle() { public void initLocale() { - if(FacesContext.getCurrentInstance() == null) { - localeCode = "en"; - } - else if (FacesContext.getCurrentInstance().getViewRoot() == null ) { - localeCode = FacesContext.getCurrentInstance().getExternalContext().getRequestLocale().getLanguage(); - } - else if (FacesContext.getCurrentInstance().getViewRoot().getLocale().getLanguage().equals("en_US")) { - localeCode = "en"; - } - else { - localeCode = FacesContext.getCurrentInstance().getViewRoot().getLocale().getLanguage(); + + localeCode = "en"; + if (FacesContext.getCurrentInstance() != null) { + if (FacesContext.getCurrentInstance().getViewRoot() == null) { + localeCode = FacesContext.getCurrentInstance().getExternalContext().getRequestLocale().getLanguage(); + } else if (FacesContext.getCurrentInstance().getViewRoot().getLocale().getLanguage().equals("en_US")) { + localeCode = "en"; + } else { + localeCode = FacesContext.getCurrentInstance().getViewRoot().getLocale().getLanguage(); + } } - logger.fine("init: locale set to "+localeCode); } diff --git a/src/main/java/edu/harvard/iq/dataverse/DvObject.java b/src/main/java/edu/harvard/iq/dataverse/DvObject.java index 5dab43fbdbd..7bb93ea6dde 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DvObject.java +++ b/src/main/java/edu/harvard/iq/dataverse/DvObject.java @@ -143,12 +143,14 @@ public String visit(DataFile df) { @Column(insertable = false, updatable = false) private String dtype; /* - * Add DOI related fields + * Add PID related fields */ private String protocol; private String authority; + private String separator; + @Temporal(value = TemporalType.TIMESTAMP) private Date globalIdCreateTime; @@ -323,6 +325,16 @@ public void setAuthority(String authority) { globalId=null; } + public String getSeparator() { + return separator; + } + + public void setSeparator(String separator) { + this.separator = separator; + //Remove cached value + globalId=null; + } + public Date getGlobalIdCreateTime() { return globalIdCreateTime; } @@ -353,11 +365,13 @@ public void setGlobalId( GlobalId pid ) { if ( pid == null ) { setProtocol(null); setAuthority(null); + setSeparator(null); setIdentifier(null); } else { //These reset globalId=null setProtocol(pid.getProtocol()); setAuthority(pid.getAuthority()); + setSeparator(pid.getSeparator()); setIdentifier(pid.getIdentifier()); } } diff --git a/src/main/java/edu/harvard/iq/dataverse/EjbDataverseEngine.java b/src/main/java/edu/harvard/iq/dataverse/EjbDataverseEngine.java index 0561fed8a97..0f211dc6713 100644 --- a/src/main/java/edu/harvard/iq/dataverse/EjbDataverseEngine.java +++ b/src/main/java/edu/harvard/iq/dataverse/EjbDataverseEngine.java @@ -4,6 +4,7 @@ import edu.harvard.iq.dataverse.actionlogging.ActionLogServiceBean; import edu.harvard.iq.dataverse.authorization.AuthenticationServiceBean; import edu.harvard.iq.dataverse.authorization.providers.builtin.BuiltinUserServiceBean; +import edu.harvard.iq.dataverse.dataverse.featured.DataverseFeaturedItemServiceBean; import edu.harvard.iq.dataverse.util.cache.CacheFactoryBean; import edu.harvard.iq.dataverse.engine.DataverseEngine; import edu.harvard.iq.dataverse.authorization.Permission; @@ -184,7 +185,10 @@ public class EjbDataverseEngine { ConfirmEmailServiceBean confirmEmailService; @EJB - StorageUseServiceBean storageUseService; + StorageUseServiceBean storageUseService; + + @EJB + DataverseFeaturedItemServiceBean dataverseFeaturedItemServiceBean; @EJB EjbDataverseEngineInner innerEngine; @@ -522,6 +526,11 @@ public DatasetFieldServiceBean dsField() { return dsField; } + @Override + public DataverseFeaturedItemServiceBean dataverseFeaturedItems() { + return dataverseFeaturedItemServiceBean; + } + @Override public StorageUseServiceBean storageUse() { return storageUseService; diff --git a/src/main/java/edu/harvard/iq/dataverse/ExternalIdentifier.java b/src/main/java/edu/harvard/iq/dataverse/ExternalIdentifier.java index 8c4fb6b1325..ef4bcb312c1 100644 --- a/src/main/java/edu/harvard/iq/dataverse/ExternalIdentifier.java +++ b/src/main/java/edu/harvard/iq/dataverse/ExternalIdentifier.java @@ -4,17 +4,18 @@ import java.util.regex.Pattern; public enum ExternalIdentifier { - ORCID("ORCID", "https://orcid.org/%s", "^\\d{4}-\\d{4}-\\d{4}-(\\d{4}|\\d{3}X)$"), - ISNI("ISNI", "http://www.isni.org/isni/%s", "^\\d*$"), - LCNA("LCNA", "http://id.loc.gov/authorities/names/%s", "^[a-z]+\\d+$"), - VIAF("VIAF", "https://viaf.org/viaf/%s", "^\\d*$"), + ORCID("ORCID", "https://orcid.org/%s", "^(https:\\/\\/orcid\\.org\\/)?\\d{4}-\\d{4}-\\d{4}-(\\d{4}|\\d{3}X)$"), + ISNI("ISNI", "http://www.isni.org/isni/%s", "^(http:\\/\\/www\\.isni\\.org\\/isni\\/)?(\\d{16}|\\d{15}X)$"), + LCNA("LCNA", "http://id.loc.gov/authorities/names/%s", "^(http:\\/\\/id\\.loc\\.gov\\/authorities\\/names\\/)?[a-z]+\\d+$"), + VIAF("VIAF", "https://viaf.org/viaf/%s", "^(https:\\/\\/viaf\\.org\\/viaf\\/)?\\d*$"), // GND regex from https://www.wikidata.org/wiki/Property:P227 - GND("GND", "https://d-nb.info/gnd/%s", "^1[01]?\\d{7}[0-9X]|[47]\\d{6}-\\d|[1-9]\\d{0,7}-[0-9X]|3\\d{7}[0-9X]$"), + GND("GND", "https://d-nb.info/gnd/%s", "^(https:\\/\\/d-nb\\.info\\/gnd\\/)?(1[01]?\\d{7}[0-9X]|[47]\\d{6}-\\d|[1-9]\\d{0,7}-[0-9X]|3\\d{7}[0-9X])$"), // note: DAI is missing from this list, because it doesn't have resolvable URL - ResearcherID("ResearcherID", "https://publons.com/researcher/%s/", "^[A-Z\\d][A-Z\\d-]+[A-Z\\d]$"), - ScopusID("ScopusID", "https://www.scopus.com/authid/detail.uri?authorId=%s", "^\\d*$"), - //Requiring ROR to be URL form as we use it where there is no id type field and matching any 9 digit number starting with 0 seems a bit aggressive - ROR("ROR", "https://ror.org/%s", "^(https:\\/\\/ror.org\\/)0[a-hj-km-np-tv-z|0-9]{6}[0-9]{2}$"); + ResearcherID("ResearcherID", "https://publons.com/researcher/%s/", "^([A-Z\\d][A-Z\\d-]+[A-Z\\d]|(https:\\/\\/publons\\.com\\/researcher\\/)?[A-Z\\d][A-Z\\d-]+[A-Z\\d]\\/)$"), + ScopusID("ScopusID", "https://www.scopus.com/authid/detail.uri?authorId=%s", "^(https:\\/\\/www\\.scopus\\.com\\/authid\\/detail\\.uri\\?authorId=)?\\d*$"), + // ROR regex from https://ror.readme.io/docs/identifier + ROR("ROR", "https://ror.org/%s", "^(https:\\/\\/ror\\.org\\/)?0[a-hj-km-np-tv-z|0-9]{6}[0-9]{2}$"), + DAI("DAI", "info:eu-repo/dai/nl/%s", "^(info:eu-repo\\/dai\\/nl\\/)?[\\d]?\\d{8}[0-9X]$"); private String name; private String template; @@ -55,6 +56,9 @@ public Pattern getPattern() { } public String format(String idValue) { + if(idValue.startsWith(template.substring(0,template.indexOf("%s")))) { + return idValue; + } return String.format(template, idValue); } } diff --git a/src/main/java/edu/harvard/iq/dataverse/FileDownloadHelper.java b/src/main/java/edu/harvard/iq/dataverse/FileDownloadHelper.java index 80cf3db8d53..02f986519a2 100644 --- a/src/main/java/edu/harvard/iq/dataverse/FileDownloadHelper.java +++ b/src/main/java/edu/harvard/iq/dataverse/FileDownloadHelper.java @@ -72,13 +72,23 @@ public FileDownloadHelper() { // file downloads and multiple (batch) downloads - since both use the same // terms/etc. popup. public void writeGuestbookAndStartDownload(GuestbookResponse guestbookResponse, boolean isGlobusTransfer) { + logger.fine("inside FileDownloadHelper.writeGuestbookAndStartDownload() " + (isGlobusTransfer ? "Globus Transfer" : "NOT a Globus Transfer")); PrimeFaces.current().executeScript("PF('guestbookAndTermsPopup').hide()"); guestbookResponse.setEventType(GuestbookResponse.DOWNLOAD); // Note that this method is only ever called from the file-download-popup - // meaning we know for the fact that we DO want to save this // guestbookResponse permanently in the database. - if(isGlobusTransfer) { - globusService.writeGuestbookAndStartTransfer(guestbookResponse, true); + // Do keep in mind that "true" in writeGuestbookAndStartTransfer() below + // would mean "DO SKIP writing the guestbookResponse", and "false" means + // "DO write ..." + if(isGlobusTransfer) { + // Note that *single-file* Globus transfers are NOT handled here. + // Instead they are coming in through this method with isGlobusTransfer=false, + // and then picked up by the fileDownloadService, below, which in turn + // recognizes them as Globus types via guestbookResponse.getFileFormat() == "GlobusTransfer" + // and treats them as such... I'm not super clear as to why they can't + // be handled here instead... Don't ask. + globusService.writeGuestbookAndStartTransfer(guestbookResponse, false); } else { if (guestbookResponse.getSelectedFileIds() != null) { // this is a batch (multiple file) download. diff --git a/src/main/java/edu/harvard/iq/dataverse/FileMetadataVersionsHelper.java b/src/main/java/edu/harvard/iq/dataverse/FileMetadataVersionsHelper.java new file mode 100644 index 00000000000..cf5a536d219 --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/FileMetadataVersionsHelper.java @@ -0,0 +1,274 @@ +package edu.harvard.iq.dataverse; + +import edu.harvard.iq.dataverse.authorization.Permission; +import edu.harvard.iq.dataverse.engine.command.DataverseRequest; +import edu.harvard.iq.dataverse.util.StringUtil; +import jakarta.ejb.EJB; +import jakarta.ejb.Stateless; +import jakarta.json.*; + +import java.util.*; +import java.util.logging.Logger; +import java.util.stream.Collectors; + +import static edu.harvard.iq.dataverse.util.json.NullSafeJsonBuilder.jsonObjectBuilder; + +@Stateless +public class FileMetadataVersionsHelper { + private static final Logger logger = Logger.getLogger(FileMetadataVersionsHelper.class.getCanonicalName()); + + @EJB + DataFileServiceBean datafileService; + @EJB + DatasetVersionServiceBean datasetVersionService; + @EJB + PermissionServiceBean permissionService; + + // Groups that are single element groups and therefore not arrays. + private static final List SINGLE_ELEMENT_GROUPS = List.of("File Access"); + + public List loadFileVersionList(DataverseRequest req, FileMetadata fileMetadata) { + List allfiles = allRelatedFiles(fileMetadata); + List retList = new ArrayList<>(); + boolean hasPermission = permissionService.requestOn(req, fileMetadata.getDatasetVersion().getDataset()).has(Permission.ViewUnpublishedDataset); + for (DatasetVersion versionLoop : fileMetadata.getDatasetVersion().getDataset().getVersions()) { + boolean foundFmd = false; + if (versionLoop.isReleased() || versionLoop.isDeaccessioned() || hasPermission) { + foundFmd = false; + //TODO: Improve this code to get the FileMetadata directly from the list of allfiles without the need to double loop + for (DataFile df : allfiles) { + FileMetadata fmd = datafileService.findFileMetadataByDatasetVersionIdAndDataFileId(versionLoop.getId(), df.getId()); + if (fmd != null) { + fmd.setContributorNames(datasetVersionService.getContributorsNames(versionLoop)); + FileVersionDifference fvd = new FileVersionDifference(fmd, getPreviousFileMetadata(fileMetadata, fmd), true); + fmd.setFileVersionDifference(fvd); + retList.add(fmd); + foundFmd = true; + break; + } + } + // no File metadata found make dummy one + if (!foundFmd) { + FileMetadata dummy = new FileMetadata(); + dummy.setDatasetVersion(versionLoop); + dummy.setDataFile(null); + FileVersionDifference fvd = new FileVersionDifference(dummy, getPreviousFileMetadata(fileMetadata, versionLoop), true); + dummy.setFileVersionDifference(fvd); + retList.add(dummy); + } + } + } + return retList; + } + + public JsonObjectBuilder jsonDataFileVersions(FileMetadata fileMetadata) { + JsonObjectBuilder job = jsonObjectBuilder(); + if (fileMetadata.getDatasetVersion() != null) { + job.add("datasetVersion", fileMetadata.getDatasetVersion().getFriendlyVersionNumber()); + if (fileMetadata.getDatasetVersion().getVersionNumber() != null) { + job + .add("versionNumber", fileMetadata.getDatasetVersion().getVersionNumber()) + .add("versionMinorNumber", fileMetadata.getDatasetVersion().getMinorVersionNumber()); + } + + job + .add("isDraft", fileMetadata.getDatasetVersion().isDraft()) + .add("isReleased", fileMetadata.getDatasetVersion().isReleased()) + .add("isDeaccessioned", fileMetadata.getDatasetVersion().isDeaccessioned()) + .add("versionState", fileMetadata.getDatasetVersion().getVersionState().name()) + .add("summary", fileMetadata.getDatasetVersion().getVersionNote()) + .add("contributors", fileMetadata.getContributorNames()) + ; + } + if (fileMetadata.getDataFile() != null) { + job.add("datafileId", fileMetadata.getDataFile().getId()); + job.add("persistentId", (fileMetadata.getDataFile().getGlobalId() != null ? fileMetadata.getDataFile().getGlobalId().asString() : "")); + if (fileMetadata.getDataFile().getPublicationDate() != null) { + job.add("publishedDate", fileMetadata.getDataFile().getPublicationDate().toString()); + } + } + FileVersionDifference fvd = fileMetadata.getFileVersionDifference(); + if (fvd != null) { + List groups = fvd.getDifferenceSummaryGroups(); + JsonObjectBuilder fileDifferenceSummary = jsonObjectBuilder(); + + if (fileMetadata.getDatasetVersion().isDeaccessioned() && fileMetadata.getDatasetVersion().getVersionNote() != null) { + fileDifferenceSummary.add("deaccessionedReason", fileMetadata.getDatasetVersion().getVersionNote()); + } + String fileAction = getFileAction(fvd.getOriginalFileMetadata(), fvd.getNewFileMetadata()); + if (fileAction != null) { + fileDifferenceSummary.add("file", fileAction); + } + + if (groups != null && !groups.isEmpty()) { + List sortedGroups = groups.stream() + .sorted(Comparator.comparing(FileVersionDifference.FileDifferenceSummaryGroup::getName)) + .collect(Collectors.toList()); + String groupName = null; + final JsonArrayBuilder groupsArrayBuilder = Json.createArrayBuilder(); + final JsonObjectBuilder groupsObjectBuilder = jsonObjectBuilder(); + Map itemCounts = new HashMap<>(); + + for (FileVersionDifference.FileDifferenceSummaryGroup group : sortedGroups) { + if (!StringUtil.isEmpty(group.getName())) { + // if the group name changed then add its data to the fileDifferenceSummary and reset list for next group + if (groupName != null && groupName.compareTo(group.getName()) != 0) { + addJsonGroupObject(fileDifferenceSummary, groupName, groupsObjectBuilder.build(), groupsArrayBuilder.build(), itemCounts); + // Note: groupsArrayBuilder.build() also clears the data within it + itemCounts.clear(); + } + groupName = group.getName(); + + group.getFileDifferenceSummaryItems().forEach(item -> { + JsonObjectBuilder itemObjectBuilder = jsonObjectBuilder(); + if (item.getName().isEmpty()) { + // 'groupName': {'Added'=#, 'Changed'=#, ...} + // accumulate the counts since we can't make a separate array item + itemCounts.merge("Added", item.getAdded(), Integer::sum); + itemCounts.merge("Changed", item.getChanged(), Integer::sum); + itemCounts.merge("Deleted", item.getDeleted(), Integer::sum); + itemCounts.merge("Replaced", item.getReplaced(), Integer::sum); + } else if (SINGLE_ELEMENT_GROUPS.contains(group.getName())) { + // 'groupName': 'getNameValue' + groupsObjectBuilder.add(group.getName(), group.getFileDifferenceSummaryItems().get(0).getName()); + } else { + // 'groupName': [{name='', action=''}, {name='', action=''}] + String action = item.getAdded() > 0 ? "Added" : item.getChanged() > 0 ? "Changed" : + item.getDeleted() > 0 ? "Deleted" : item.getReplaced() > 0 ? "Replaced" : ""; + itemObjectBuilder.add("name", item.getName()); + if (!action.isEmpty()) { + itemObjectBuilder.add("action", action); + } + groupsArrayBuilder.add(itemObjectBuilder.build()); + } + }); + } + } + // process last group + addJsonGroupObject(fileDifferenceSummary, groupName, groupsObjectBuilder.build(), groupsArrayBuilder.build(), itemCounts); + } + JsonObject fileDifferenceSummaryObject = fileDifferenceSummary.build(); + if (!fileDifferenceSummaryObject.isEmpty()) { + job.add("fileDifferenceSummary", fileDifferenceSummaryObject); + } + } + return job; + } + + //TODO: this could use some refactoring to cut down on the number of for loops! + private FileMetadata getPreviousFileMetadata(FileMetadata fileMetadata, FileMetadata fmdIn){ + + DataFile dfPrevious = datafileService.findPreviousFile(fmdIn.getDataFile()); + DatasetVersion dvPrevious = null; + boolean gotCurrent = false; + for (DatasetVersion dvloop: fileMetadata.getDatasetVersion().getDataset().getVersions()) { + if(gotCurrent){ + dvPrevious = dvloop; + break; + } + if(dvloop.equals(fmdIn.getDatasetVersion())){ + gotCurrent = true; + } + } + + List allfiles = allRelatedFiles(fileMetadata); + + if (dvPrevious != null && dvPrevious.getFileMetadatasSorted() != null) { + for (FileMetadata fmdTest : dvPrevious.getFileMetadatasSorted()) { + for (DataFile fileTest : allfiles) { + if (fmdTest.getDataFile().equals(fileTest)) { + return fmdTest; + } + } + } + } + + Long dfId = fmdIn.getDataFile().getId(); + if (dfPrevious != null){ + dfId = dfPrevious.getId(); + } + Long versionId = null; + if (dvPrevious !=null){ + versionId = dvPrevious.getId(); + } + + FileMetadata fmd = datafileService.findFileMetadataByDatasetVersionIdAndDataFileId(versionId, dfId); + + return fmd; + } + + //TODO: this could use some refactoring to cut down on the number of for loops! + private FileMetadata getPreviousFileMetadata(FileMetadata fileMetadata, DatasetVersion currentversion) { + List allfiles = allRelatedFiles(fileMetadata); + boolean foundCurrent = false; + DatasetVersion priorVersion = null; + for (DatasetVersion versionLoop : fileMetadata.getDatasetVersion().getDataset().getVersions()) { + if (foundCurrent) { + priorVersion = versionLoop; + break; + } + if (versionLoop.equals(currentversion)) { + foundCurrent = true; + } + + } + if (priorVersion != null && priorVersion.getFileMetadatasSorted() != null) { + for (FileMetadata fmdTest : priorVersion.getFileMetadatasSorted()) { + for (DataFile fileTest : allfiles) { + if (fmdTest.getDataFile().equals(fileTest)) { + return fmdTest; + } + } + } + } + return null; + } + private List allRelatedFiles(FileMetadata fileMetadata) { + List dataFiles = new ArrayList<>(); + DataFile dataFileToTest = fileMetadata.getDataFile(); + Long rootDataFileId = dataFileToTest.getRootDataFileId(); + if (rootDataFileId < 0) { + dataFiles.add(dataFileToTest); + } else { + dataFiles.addAll(datafileService.findAllRelatedByRootDatafileId(rootDataFileId)); + } + return dataFiles; + } + + private String getFileAction(FileMetadata originalFileMetadata, FileMetadata newFileMetadata) { + if (newFileMetadata.getDataFile() != null && originalFileMetadata == null) { + return "Added"; + } else if (newFileMetadata.getDataFile() == null && originalFileMetadata != null) { + return "Deleted"; + } else if (originalFileMetadata != null && + newFileMetadata.getDataFile() != null && originalFileMetadata.getDataFile() != null &&!originalFileMetadata.getDataFile().equals(newFileMetadata.getDataFile())) { + return "Replaced"; + } else { + return null; + } + } + + private void addJsonGroupObject(JsonObjectBuilder jsonObjectBuilder, String key, JsonObject jsonObjectValue, JsonArray jsonArrayValue, Map itemCounts) { + if (key != null && !key.isEmpty()) { + String sanitizedKey = key.replaceAll("\\s+", ""); + if (itemCounts.isEmpty()) { + if (jsonArrayValue.isEmpty()) { + // add the object + jsonObjectBuilder.add(sanitizedKey, jsonObjectValue.getValue("/"+key)); + } else { + // add the array + jsonObjectBuilder.add(sanitizedKey, jsonArrayValue); + } + } else { + // add the accumulated totals + JsonObjectBuilder accumulatedTotalsObjectBuilder = jsonObjectBuilder(); + itemCounts.forEach((k, v) -> { + if (v != 0) { + accumulatedTotalsObjectBuilder.add(k, v); + } + }); + jsonObjectBuilder.add(sanitizedKey, accumulatedTotalsObjectBuilder.build()); + } + } + } +} diff --git a/src/main/java/edu/harvard/iq/dataverse/FilePage.java b/src/main/java/edu/harvard/iq/dataverse/FilePage.java index 5717da38f29..6ad934b0640 100644 --- a/src/main/java/edu/harvard/iq/dataverse/FilePage.java +++ b/src/main/java/edu/harvard/iq/dataverse/FilePage.java @@ -16,6 +16,7 @@ import edu.harvard.iq.dataverse.dataaccess.DataAccess; import edu.harvard.iq.dataverse.dataaccess.StorageIO; import edu.harvard.iq.dataverse.engine.command.Command; +import edu.harvard.iq.dataverse.engine.command.DataverseRequest; import edu.harvard.iq.dataverse.engine.command.exception.CommandException; import edu.harvard.iq.dataverse.engine.command.exception.IllegalCommandException; import edu.harvard.iq.dataverse.engine.command.impl.CreateNewDatasetCommand; @@ -59,17 +60,13 @@ import jakarta.ejb.EJB; import jakarta.ejb.EJBException; import jakarta.faces.application.FacesMessage; -import jakarta.faces.component.UIComponent; import jakarta.faces.context.FacesContext; -import jakarta.faces.validator.ValidatorException; import jakarta.faces.view.ViewScoped; import jakarta.inject.Inject; import jakarta.inject.Named; -import jakarta.json.JsonArray; -import jakarta.json.JsonObject; -import jakarta.json.JsonValue; import jakarta.validation.ConstraintViolation; +import org.omnifaces.util.Faces; import org.primefaces.PrimeFaces; import org.primefaces.component.tabview.TabView; import org.primefaces.event.TabChangeEvent; @@ -156,6 +153,8 @@ public class FilePage implements java.io.Serializable { @Inject RetentionServiceBean retentionService; + @Inject + FileMetadataVersionsHelper fileMetadataVersionsHelper; private static final Logger logger = Logger.getLogger(FilePage.class.getCanonicalName()); @@ -689,115 +688,12 @@ public void tabChanged(TabChangeEvent event) { TabView tv = (TabView) event.getComponent(); this.activeTabIndex = tv.getActiveIndex(); if (this.activeTabIndex == 1 || this.activeTabIndex == 2 ) { - setFileMetadatasForTab(loadFileMetadataTabList()); + setFileMetadatasForTab(fileMetadataVersionsHelper.loadFileVersionList(new DataverseRequest(session.getUser(), Faces.getRequest()), fileMetadata)); } else { setFileMetadatasForTab( new ArrayList<>()); } } - - - private List loadFileMetadataTabList() { - List allfiles = allRelatedFiles(); - List retList = new ArrayList<>(); - for (DatasetVersion versionLoop : fileMetadata.getDatasetVersion().getDataset().getVersions()) { - boolean foundFmd = false; - - if (versionLoop.isReleased() || versionLoop.isDeaccessioned() || permissionService.on(fileMetadata.getDatasetVersion().getDataset()).has(Permission.ViewUnpublishedDataset)) { - foundFmd = false; - for (DataFile df : allfiles) { - FileMetadata fmd = datafileService.findFileMetadataByDatasetVersionIdAndDataFileId(versionLoop.getId(), df.getId()); - if (fmd != null) { - fmd.setContributorNames(datasetVersionService.getContributorsNames(versionLoop)); - FileVersionDifference fvd = new FileVersionDifference(fmd, getPreviousFileMetadata(fmd)); - fmd.setFileVersionDifference(fvd); - retList.add(fmd); - foundFmd = true; - break; - } - } - //no File metadata found make dummy one - if (!foundFmd) { - FileMetadata dummy = new FileMetadata(); - dummy.setDatasetVersion(versionLoop); - dummy.setDataFile(null); - FileVersionDifference fvd = new FileVersionDifference(dummy, getPreviousFileMetadata(versionLoop)); - dummy.setFileVersionDifference(fvd); - retList.add(dummy); - } - } - } - return retList; - } - - private FileMetadata getPreviousFileMetadata(DatasetVersion currentversion) { - List allfiles = allRelatedFiles(); - boolean foundCurrent = false; - DatasetVersion priorVersion = null; - for (DatasetVersion versionLoop : fileMetadata.getDatasetVersion().getDataset().getVersions()) { - if (foundCurrent) { - priorVersion = versionLoop; - break; - } - if (versionLoop.equals(currentversion)) { - foundCurrent = true; - } - - } - if (priorVersion != null && priorVersion.getFileMetadatasSorted() != null) { - for (FileMetadata fmdTest : priorVersion.getFileMetadatasSorted()) { - for (DataFile fileTest : allfiles) { - if (fmdTest.getDataFile().equals(fileTest)) { - return fmdTest; - } - } - } - } - return null; - - } - - private FileMetadata getPreviousFileMetadata(FileMetadata fmdIn){ - - DataFile dfPrevious = datafileService.findPreviousFile(fmdIn.getDataFile()); - DatasetVersion dvPrevious = null; - boolean gotCurrent = false; - for (DatasetVersion dvloop: fileMetadata.getDatasetVersion().getDataset().getVersions()){ - if(gotCurrent){ - dvPrevious = dvloop; - break; - } - if(dvloop.equals(fmdIn.getDatasetVersion())){ - gotCurrent = true; - } - } - - List allfiles = allRelatedFiles(); - - if (dvPrevious != null && dvPrevious.getFileMetadatasSorted() != null) { - for (FileMetadata fmdTest : dvPrevious.getFileMetadatasSorted()) { - for (DataFile fileTest : allfiles) { - if (fmdTest.getDataFile().equals(fileTest)) { - return fmdTest; - } - } - } - } - - Long dfId = fmdIn.getDataFile().getId(); - if (dfPrevious != null){ - dfId = dfPrevious.getId(); - } - Long versionId = null; - if (dvPrevious !=null){ - versionId = dvPrevious.getId(); - } - - FileMetadata fmd = datafileService.findFileMetadataByDatasetVersionIdAndDataFileId(versionId, dfId); - - return fmd; - } - public List getFileMetadatasForTab() { return fileMetadatasForTab; } diff --git a/src/main/java/edu/harvard/iq/dataverse/GlobalId.java b/src/main/java/edu/harvard/iq/dataverse/GlobalId.java index 1c8783c5bd5..058a6269b57 100644 --- a/src/main/java/edu/harvard/iq/dataverse/GlobalId.java +++ b/src/main/java/edu/harvard/iq/dataverse/GlobalId.java @@ -63,6 +63,10 @@ public String getAuthority() { return authority; } + public String getSeparator() { + return separator; + } + public String getIdentifier() { return identifier; } diff --git a/src/main/java/edu/harvard/iq/dataverse/HarvestingClientsPage.java b/src/main/java/edu/harvard/iq/dataverse/HarvestingClientsPage.java index f008db1403f..1effd137e0e 100644 --- a/src/main/java/edu/harvard/iq/dataverse/HarvestingClientsPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/HarvestingClientsPage.java @@ -78,7 +78,7 @@ public class HarvestingClientsPage implements java.io.Serializable { private Long dataverseId = null; private HarvestingClient selectedClient; private boolean setListTruncated = false; - + //private static final String solrDocIdentifierDataset = "dataset_"; public enum PageMode { @@ -242,6 +242,7 @@ public void editClient(HarvestingClient harvestingClient) { setSelectedClient(harvestingClient); this.newNickname = harvestingClient.getName(); + this.sourceName = harvestingClient.getSourceName(); this.newHarvestingUrl = harvestingClient.getHarvestingUrl(); this.customHeader = harvestingClient.getCustomHttpHeaders(); this.initialSettingsValidated = false; @@ -323,10 +324,9 @@ public void deleteClient() { } public void createClient(ActionEvent ae) { - - HarvestingClient newHarvestingClient = new HarvestingClient(); // will be set as type OAI by default - - newHarvestingClient.setName(newNickname); + + // will be set as type OAI by default + HarvestingClient newHarvestingClient = fillHarvestingClient(new HarvestingClient()); if (getSelectedDestinationDataverse() == null) { JsfHelper.JH.addMessage(FacesMessage.SEVERITY_ERROR,BundleUtil.getStringFromBundle("harvest.create.error")); @@ -338,35 +338,6 @@ public void createClient(ActionEvent ae) { } getSelectedDestinationDataverse().getHarvestingClientConfigs().add(newHarvestingClient); - newHarvestingClient.setHarvestingUrl(newHarvestingUrl); - newHarvestingClient.setCustomHttpHeaders(customHeader); - if (!StringUtils.isEmpty(newOaiSet)) { - newHarvestingClient.setHarvestingSet(newOaiSet); - } - newHarvestingClient.setMetadataPrefix(newMetadataFormat); - newHarvestingClient.setHarvestStyle(newHarvestingStyle); - - if (isNewHarvestingScheduled()) { - newHarvestingClient.setScheduled(true); - - if (isNewHarvestingScheduledWeekly()) { - newHarvestingClient.setSchedulePeriod(HarvestingClient.SCHEDULE_PERIOD_WEEKLY); - if (getWeekDayNumber() == null) { - // create a "week day is required..." error message, etc. - // but we may be better off not even giving them an opportunity - // to leave the field blank - ? - } - newHarvestingClient.setScheduleDayOfWeek(getWeekDayNumber()); - } else { - newHarvestingClient.setSchedulePeriod(HarvestingClient.SCHEDULE_PERIOD_DAILY); - } - - if (getHourOfDay() == null) { - // see the comment above, about the day of week. same here. - } - newHarvestingClient.setScheduleHourOfDay(getHourOfDay()); - } - // make default archive url (used to generate links pointing back to the // archival sources, when harvested datasets are displayed in search results), // from the harvesting url: @@ -412,51 +383,9 @@ public void createClient(ActionEvent ae) { // this saves an existing client that the user has edited: public void saveClient(ActionEvent ae) { - - HarvestingClient harvestingClient = getSelectedClient(); - - if (harvestingClient == null) { - // TODO: - // tell the user somehow that the client cannot be saved, and advise - // them to save the settings they have entered. - // as of now - we will show an error message, but only after the - // edit form has been closed. - } - - // nickname is not editable for existing clients: - //harvestingClient.setName(newNickname); - harvestingClient.setHarvestingUrl(newHarvestingUrl); - harvestingClient.setCustomHttpHeaders(customHeader); - harvestingClient.setHarvestingSet(newOaiSet); - harvestingClient.setMetadataPrefix(newMetadataFormat); - harvestingClient.setHarvestStyle(newHarvestingStyle); - - if (isNewHarvestingScheduled()) { - harvestingClient.setScheduled(true); - - if (isNewHarvestingScheduledWeekly()) { - harvestingClient.setSchedulePeriod(HarvestingClient.SCHEDULE_PERIOD_WEEKLY); - if (getWeekDayNumber() == null) { - // create a "week day is required..." error message, etc. - // but we may be better off not even giving them an opportunity - // to leave the field blank - ? - } - harvestingClient.setScheduleDayOfWeek(getWeekDayNumber()); - } else { - harvestingClient.setSchedulePeriod(HarvestingClient.SCHEDULE_PERIOD_DAILY); - } - - if (getHourOfDay() == null) { - // see the comment above, about the day of week. same here. - } - harvestingClient.setScheduleHourOfDay(getHourOfDay()); - } else { - harvestingClient.setScheduled(false); - } - - // will try to save it now: - try { + HarvestingClient harvestingClient = fillHarvestingClient(getSelectedClient()); + harvestingClient = engineService.submit( new UpdateHarvestingClientCommand(dvRequestService.getDataverseRequest(), harvestingClient)); configuredHarvestingClients = harvestingClientService.getAllHarvestingClients(); @@ -477,9 +406,50 @@ public void saveClient(ActionEvent ae) { } setPageMode(PageMode.VIEW); - + } - + + /** + * Based on a new harvestingClient instance or an existing one, it will update basics fields with new UI fields values + * @param harvestingClient new or existing harvestingClient to update + * @return harvestingClient with updated values + */ + private HarvestingClient fillHarvestingClient(HarvestingClient harvestingClient) { + // update nickname if it's a new object otherwise is not editable for existing clients + if(harvestingClient.getId() == null) { + harvestingClient.setName(newNickname); + } + harvestingClient.setSourceName(sourceName); + harvestingClient.setHarvestingUrl(newHarvestingUrl); + harvestingClient.setCustomHttpHeaders(customHeader); + if (!StringUtils.isEmpty(newOaiSet)) { + harvestingClient.setHarvestingSet(newOaiSet); + } + harvestingClient.setMetadataPrefix(newMetadataFormat); + harvestingClient.setHarvestStyle(newHarvestingStyle); + + harvestingClient.setScheduled(isNewHarvestingScheduled()); + if (isNewHarvestingScheduled()) { + if (isNewHarvestingScheduledWeekly()) { + harvestingClient.setSchedulePeriod(HarvestingClient.SCHEDULE_PERIOD_WEEKLY); + if (getWeekDayNumber() == null) { + // create a "week day is required..." error message, etc. + // but we may be better off not even giving them an opportunity + // to leave the field blank - ? + } + harvestingClient.setScheduleDayOfWeek(getWeekDayNumber()); + } else { + harvestingClient.setSchedulePeriod(HarvestingClient.SCHEDULE_PERIOD_DAILY); + } + + if (getHourOfDay() == null) { + // see the comment above, about the day of week. same here. + } + harvestingClient.setScheduleHourOfDay(getHourOfDay()); + } + return harvestingClient; + } + public void validateMetadataFormat(FacesContext context, UIComponent toValidate, Object rawValue) { String value = (String) rawValue; UIInput input = (UIInput) toValidate; @@ -717,6 +687,7 @@ public void backToStepThree() { UIInput selectedDataverseMenu; private String newNickname = ""; + private String sourceName = ""; private String newHarvestingUrl = ""; private String customHeader = null; private boolean initialSettingsValidated = false; @@ -741,6 +712,7 @@ public void backToStepThree() { public void initNewClient(ActionEvent ae) { //this.selectedClient = new HarvestingClient(); this.newNickname = ""; + this.sourceName = ""; this.newHarvestingUrl = ""; this.customHeader = null; this.initialSettingsValidated = false; @@ -842,6 +814,14 @@ public int getHarvestingScheduleRadio() { public void setHarvestingScheduleRadio(int harvestingScheduleRadio) { this.harvestingScheduleRadio = harvestingScheduleRadio; } + + public String getSourceName() { + return sourceName; + } + + public void setSourceName(String sourceName) { + this.sourceName = sourceName; + } public boolean isNewHarvestingScheduled() { return this.harvestingScheduleRadio != harvestingScheduleRadioNone; diff --git a/src/main/java/edu/harvard/iq/dataverse/MetadataBlock.java b/src/main/java/edu/harvard/iq/dataverse/MetadataBlock.java index 0fd7c2efbc7..c5a6d03c5d7 100644 --- a/src/main/java/edu/harvard/iq/dataverse/MetadataBlock.java +++ b/src/main/java/edu/harvard/iq/dataverse/MetadataBlock.java @@ -102,7 +102,8 @@ public void setDatasetFieldTypes(List datasetFieldTypes) { public boolean isDisplayOnCreate() { for (DatasetFieldType dsfType : datasetFieldTypes) { - if (dsfType.isDisplayOnCreate()) { + boolean shouldDisplayOnCreate = dsfType.shouldDisplayOnCreate(); + if (shouldDisplayOnCreate) { return true; } } diff --git a/src/main/java/edu/harvard/iq/dataverse/MetadataBlockServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/MetadataBlockServiceBean.java index 1e2a34f5472..9a5a1060c50 100644 --- a/src/main/java/edu/harvard/iq/dataverse/MetadataBlockServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/MetadataBlockServiceBean.java @@ -8,6 +8,7 @@ import jakarta.persistence.TypedQuery; import jakarta.persistence.criteria.*; +import java.util.Comparator; import java.util.List; /** @@ -52,31 +53,62 @@ public MetadataBlock findByName(String name) { public List listMetadataBlocksDisplayedOnCreate(Dataverse ownerDataverse) { CriteriaBuilder criteriaBuilder = em.getCriteriaBuilder(); CriteriaQuery criteriaQuery = criteriaBuilder.createQuery(MetadataBlock.class); - Root metadataBlockRoot = criteriaQuery.from(MetadataBlock.class); - Join datasetFieldTypeJoin = metadataBlockRoot.join("datasetFieldTypes"); + Root dataverseRoot = criteriaQuery.from(Dataverse.class); + + // Join metadataBlocks from Dataverse + Join metadataBlockJoin = dataverseRoot.join("metadataBlocks"); + + // Join datasetFieldTypes from MetadataBlock + Join datasetFieldTypeJoin = metadataBlockJoin.join("datasetFieldTypes"); + Predicate displayOnCreatePredicate = criteriaBuilder.isTrue(datasetFieldTypeJoin.get("displayOnCreate")); + Predicate requiredPredicate = criteriaBuilder.isTrue(datasetFieldTypeJoin.get("required")); if (ownerDataverse != null) { - Root dataverseRoot = criteriaQuery.from(Dataverse.class); - Join datasetFieldTypeInputLevelJoin = dataverseRoot.join("dataverseFieldTypeInputLevels", JoinType.LEFT); + // Ensure we filter for the specific Dataverse + Predicate dataversePredicate = criteriaBuilder.equal(dataverseRoot.get("id"), ownerDataverse.getId()); + + // Join DataverseFieldTypeInputLevel (LEFT JOIN) + Join datasetFieldTypeInputLevelJoin = + dataverseRoot.join("dataverseFieldTypeInputLevels", JoinType.LEFT); + + // Check if input level explicitly defines displayOnCreate + Predicate inputLevelDisplayPredicate = criteriaBuilder.and( + criteriaBuilder.equal(datasetFieldTypeInputLevelJoin.get("datasetFieldType"), datasetFieldTypeJoin), + criteriaBuilder.isTrue(datasetFieldTypeInputLevelJoin.get("displayOnCreate")) + ); - Predicate requiredPredicate = criteriaBuilder.and( - datasetFieldTypeInputLevelJoin.get("datasetFieldType").in(metadataBlockRoot.get("datasetFieldTypes")), - criteriaBuilder.isTrue(datasetFieldTypeInputLevelJoin.get("required"))); + // Check if input level explicitly defines required + Predicate inputLevelRequiredPredicate = criteriaBuilder.and( + criteriaBuilder.equal(datasetFieldTypeInputLevelJoin.get("datasetFieldType"), datasetFieldTypeJoin), + criteriaBuilder.isTrue(datasetFieldTypeInputLevelJoin.get("required")) + ); - Predicate unionPredicate = criteriaBuilder.or(displayOnCreatePredicate, requiredPredicate); + Predicate finalDisplayPredicate = criteriaBuilder.or(inputLevelDisplayPredicate, displayOnCreatePredicate); + Predicate finalRequiredPredicate = criteriaBuilder.or(inputLevelRequiredPredicate, requiredPredicate); - criteriaQuery.where(criteriaBuilder.and( - criteriaBuilder.equal(dataverseRoot.get("id"), ownerDataverse.getId()), - metadataBlockRoot.in(dataverseRoot.get("metadataBlocks")), - unionPredicate - )); + criteriaQuery.where( + dataversePredicate, + criteriaBuilder.or(finalDisplayPredicate, finalRequiredPredicate) + ); } else { - criteriaQuery.where(displayOnCreatePredicate); + // When ownerDataverse is null, we need to include fields that are either displayOnCreate=true OR required=true + // We also need to ensure that fields from linked metadata blocks are included + Predicate linkedFieldsPredicate = criteriaBuilder.and( + criteriaBuilder.isNotNull(datasetFieldTypeJoin.get("id")), + criteriaBuilder.or(displayOnCreatePredicate, requiredPredicate) + ); + + criteriaQuery.where(linkedFieldsPredicate); } - criteriaQuery.select(metadataBlockRoot).distinct(true); - TypedQuery typedQuery = em.createQuery(criteriaQuery); - return typedQuery.getResultList(); + criteriaQuery.select(metadataBlockJoin).distinct(true); + + List result = em.createQuery(criteriaQuery).getResultList(); + + // Order by id + result.sort(Comparator.comparing(MetadataBlock::getId)); + + return result; } } diff --git a/src/main/java/edu/harvard/iq/dataverse/PermissionServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/PermissionServiceBean.java index a389cbc735b..2f727987537 100644 --- a/src/main/java/edu/harvard/iq/dataverse/PermissionServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/PermissionServiceBean.java @@ -2,13 +2,14 @@ import edu.harvard.iq.dataverse.authorization.AuthenticationServiceBean; import edu.harvard.iq.dataverse.authorization.DataverseRole; +import edu.harvard.iq.dataverse.authorization.groups.impl.ipaddress.ip.IPv4Address; +import edu.harvard.iq.dataverse.authorization.groups.impl.ipaddress.ip.IPv6Address; +import edu.harvard.iq.dataverse.authorization.groups.impl.ipaddress.ip.IpAddress; import edu.harvard.iq.dataverse.authorization.providers.builtin.BuiltinUserServiceBean; import edu.harvard.iq.dataverse.authorization.users.GuestUser; import edu.harvard.iq.dataverse.authorization.Permission; import edu.harvard.iq.dataverse.authorization.RoleAssignee; -import edu.harvard.iq.dataverse.authorization.groups.Group; import edu.harvard.iq.dataverse.authorization.groups.GroupServiceBean; -import edu.harvard.iq.dataverse.authorization.groups.GroupUtil; import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser; import edu.harvard.iq.dataverse.authorization.users.User; import edu.harvard.iq.dataverse.engine.command.Command; @@ -37,7 +38,6 @@ import java.util.Collections; import java.util.HashMap; import java.util.LinkedList; -import java.util.logging.Level; import java.util.stream.Collectors; import static java.util.stream.Collectors.toList; import jakarta.persistence.Query; @@ -100,6 +100,70 @@ public class PermissionServiceBean { @Inject DatasetVersionFilesServiceBean datasetVersionFilesServiceBean; + private static final String LIST_ALL_DATAVERSES_USER_HAS_PERMISSION = """ + WITH grouplist AS ( + SELECT explicitgroup_authenticateduser.explicitgroup_id as id FROM explicitgroup_authenticateduser + WHERE explicitgroup_authenticateduser.containedauthenticatedusers_id = @USERID + ) + + SELECT * FROM DATAVERSE WHERE id IN ( + SELECT definitionpoint_id + FROM roleassignment + WHERE roleassignment.assigneeidentifier IN ( + SELECT CONCAT('&explicit/', explicitgroup.groupalias) as assignee + FROM explicitgroup + WHERE explicitgroup.id IN ( + ( + SELECT explicitgroup.id id + FROM explicitgroup + WHERE EXISTS (SELECT id FROM grouplist WHERE id = explicitgroup.id) + ) UNION ( + SELECT explicitgroup_explicitgroup.containedexplicitgroups_id id + FROM explicitgroup_explicitgroup + WHERE EXISTS (SELECT id FROM grouplist WHERE id = explicitgroup_explicitgroup.explicitgroup_id) + AND EXISTS (SELECT id FROM dataverserole + WHERE dataverserole.id = roleassignment.role_id AND (dataverserole.permissionbits & @PERMISSIONBIT !=0)) + ) + ) + ) UNION ( + SELECT definitionpoint_id + FROM roleassignment + WHERE roleassignment.assigneeidentifier = ( + SELECT CONCAT('@', authenticateduser.useridentifier) + FROM authenticateduser + WHERE authenticateduser.id = @USERID) + AND EXISTS (SELECT id FROM dataverserole + WHERE dataverserole.id = roleassignment.role_id AND (dataverserole.permissionbits & @PERMISSIONBIT !=0)) + ) UNION ( + SELECT definitionpoint_id + FROM roleassignment + WHERE roleassignment.assigneeidentifier = ':authenticated-users' + AND EXISTS (SELECT id FROM dataverserole + WHERE dataverserole.id = roleassignment.role_id AND (dataverserole.permissionbits & @PERMISSIONBIT !=0)) + ) UNION ( + SELECT definitionpoint_id + FROM roleassignment + WHERE roleassignment.assigneeidentifier IN ( + SELECT CONCAT('&shib/', persistedglobalgroup.persistedgroupalias) as assignee + FROM persistedglobalgroup + WHERE dtype = 'ShibGroup' + AND EXISTS (SELECT id FROM dataverserole WHERE dataverserole.id = roleassignment.role_id AND (dataverserole.permissionbits & @PERMISSIONBIT !=0)) + ) + ) UNION ( + SELECT definitionpoint_id + FROM roleassignment + WHERE roleassignment.assigneeidentifier IN ( + SELECT CONCAT('&ip/', persistedglobalgroup.persistedgroupalias) as assignee + FROM persistedglobalgroup + LEFT OUTER JOIN ipv4range ON persistedglobalgroup.id = ipv4range.owner_id + LEFT OUTER JOIN ipv6range ON persistedglobalgroup.id = ipv6range.owner_id + WHERE dtype = 'IpGroup' + AND EXISTS (SELECT id FROM dataverserole WHERE dataverserole.id = roleassignment.role_id AND (dataverserole.permissionbits & @PERMISSIONBIT !=0)) + AND @IPRANGESQL + ) + ) + ) + """; /** * A request-level permission query (e.g includes IP ras). */ @@ -553,36 +617,6 @@ public RequestPermissionQuery request(DataverseRequest req) { return new RequestPermissionQuery(null, req); } - /** - * Go from (User, Permission) to a list of Dataverse objects that the user - has the permission on. - * - * @param user - * @param permission - * @return The list of dataverses {@code user} has permission - {@code permission} on. - */ - public List getDataversesUserHasPermissionOn(AuthenticatedUser user, Permission permission) { - Set groups = groupService.groupsFor(user); - String identifiers = GroupUtil.getAllIdentifiersForUser(user, groups); - /** - * @todo Are there any strings in identifiers that would break this SQL - * query? - */ - String query = "SELECT id FROM dvobject WHERE dtype = 'Dataverse' and id in (select definitionpoint_id from roleassignment where assigneeidentifier in (" + identifiers + "));"; - logger.log(Level.FINE, "query: {0}", query); - Query nativeQuery = em.createNativeQuery(query); - List dataverseIdsToCheck = nativeQuery.getResultList(); - List dataversesUserHasPermissionOn = new LinkedList<>(); - for (int dvIdAsInt : dataverseIdsToCheck) { - Dataverse dataverse = dataverseService.find(Long.valueOf(dvIdAsInt)); - if (userOn(user, dataverse).has(permission)) { - dataversesUserHasPermissionOn.add(dataverse); - } - } - return dataversesUserHasPermissionOn; - } - public List getUsersWithPermissionOn(Permission permission, DvObject dvo) { List usersHasPermissionOn = new LinkedList<>(); Set ras = roleService.rolesAssignments(dvo); @@ -888,4 +922,46 @@ private boolean hasUnrestrictedReleasedFiles(DatasetVersion targetDatasetVersion Long result = em.createQuery(criteriaQuery).getSingleResult(); return result > 0; } + + public List findPermittedCollections(DataverseRequest request, AuthenticatedUser user, Permission permission) { + return findPermittedCollections(request, user, 1 << permission.ordinal()); + } + public List findPermittedCollections(DataverseRequest request, AuthenticatedUser user, int permissionBit) { + if (user != null) { + // IP Group - Only check IP if a User is calling for themself + String ipRangeSQL = "FALSE"; + if (request != null + && request.getAuthenticatedUser() != null + && request.getSourceAddress() != null + && request.getAuthenticatedUser().getUserIdentifier().equalsIgnoreCase(user.getUserIdentifier())) { + IpAddress ip = request.getSourceAddress(); + if (ip instanceof IPv4Address) { + IPv4Address ipv4 = (IPv4Address) ip; + ipRangeSQL = ipv4.toBigInteger() + " BETWEEN ipv4range.bottomaslong AND ipv4range.topaslong"; + } else if (ip instanceof IPv6Address) { + IPv6Address ipv6 = (IPv6Address) ip; + long[] vals = ipv6.toLongArray(); + if (vals.length == 4) { + ipRangeSQL = """ + (@0 BETWEEN ipv6range.bottoma AND ipv6range.topa + AND @1 BETWEEN ipv6range.bottomb AND ipv6range.topb + AND @2 BETWEEN ipv6range.bottomc AND ipv6range.topc + AND @3 BETWEEN ipv6range.bottomd AND ipv6range.topd) + """; + for (int i = 0; i < vals.length; i++) { + ipRangeSQL = ipRangeSQL.replace("@" + i, String.valueOf(vals[i])); + } + } + } + } + + String sqlCode = LIST_ALL_DATAVERSES_USER_HAS_PERMISSION + .replace("@USERID", String.valueOf(user.getId())) + .replace("@PERMISSIONBIT", String.valueOf(permissionBit)) + .replace("@IPRANGESQL", ipRangeSQL); + return em.createNativeQuery(sqlCode, Dataverse.class).getResultList(); + } + return null; + } } + diff --git a/src/main/java/edu/harvard/iq/dataverse/TemplatePage.java b/src/main/java/edu/harvard/iq/dataverse/TemplatePage.java index 44070dcbb41..279944beaa7 100644 --- a/src/main/java/edu/harvard/iq/dataverse/TemplatePage.java +++ b/src/main/java/edu/harvard/iq/dataverse/TemplatePage.java @@ -166,9 +166,13 @@ private void updateDatasetFieldInputLevels(){ } for (DatasetField dsf: template.getFlatDatasetFields()){ - DataverseFieldTypeInputLevel dsfIl = dataverseFieldTypeInputLevelService.findByDataverseIdDatasetFieldTypeId(dvIdForInputLevel, dsf.getDatasetFieldType().getId()); - if (dsfIl != null){ + DataverseFieldTypeInputLevel dsfIl = dataverseFieldTypeInputLevelService.findByDataverseIdDatasetFieldTypeId( + dvIdForInputLevel, + dsf.getDatasetFieldType().getId() + ); + if (dsfIl != null) { dsf.setInclude(dsfIl.isInclude()); + dsf.getDatasetFieldType().setLocalDisplayOnCreate(dsfIl.getDisplayOnCreate()); } else { dsf.setInclude(true); } diff --git a/src/main/java/edu/harvard/iq/dataverse/ValidateVersionNote.java b/src/main/java/edu/harvard/iq/dataverse/ValidateDeaccessionNote.java similarity index 78% rename from src/main/java/edu/harvard/iq/dataverse/ValidateVersionNote.java rename to src/main/java/edu/harvard/iq/dataverse/ValidateDeaccessionNote.java index c8d64d4a642..d6bf2b857d6 100644 --- a/src/main/java/edu/harvard/iq/dataverse/ValidateVersionNote.java +++ b/src/main/java/edu/harvard/iq/dataverse/ValidateDeaccessionNote.java @@ -22,17 +22,17 @@ @Target({TYPE, ANNOTATION_TYPE}) @Retention(RUNTIME) -@Constraint(validatedBy = {DatasetVersionNoteValidator.class}) +@Constraint(validatedBy = {DatasetDeaccessionNoteValidator.class}) @Documented -public @interface ValidateVersionNote { +public @interface ValidateDeaccessionNote { - String message() default "Failed Validation for DatasetVersionNote"; + String message() default "Failed Validation for DatasetsDeaccessionNote"; Class[] groups() default {}; Class[] payload() default {}; - String versionNote(); + String deaccessionNote(); String versionState(); diff --git a/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java b/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java index 3257a3cc7ac..6d54f3b6971 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java @@ -14,14 +14,11 @@ import edu.harvard.iq.dataverse.dataset.DatasetTypeServiceBean; import edu.harvard.iq.dataverse.engine.command.Command; import edu.harvard.iq.dataverse.engine.command.DataverseRequest; -import edu.harvard.iq.dataverse.engine.command.exception.CommandException; -import edu.harvard.iq.dataverse.engine.command.exception.IllegalCommandException; -import edu.harvard.iq.dataverse.engine.command.exception.PermissionException; +import edu.harvard.iq.dataverse.engine.command.exception.*; import edu.harvard.iq.dataverse.engine.command.impl.GetDraftDatasetVersionCommand; import edu.harvard.iq.dataverse.engine.command.impl.GetLatestAccessibleDatasetVersionCommand; import edu.harvard.iq.dataverse.engine.command.impl.GetLatestPublishedDatasetVersionCommand; import edu.harvard.iq.dataverse.engine.command.impl.GetSpecificPublishedDatasetVersionCommand; -import edu.harvard.iq.dataverse.engine.command.exception.RateLimitCommandException; import edu.harvard.iq.dataverse.externaltools.ExternalToolServiceBean; import edu.harvard.iq.dataverse.license.LicenseServiceBean; import edu.harvard.iq.dataverse.pidproviders.PidUtil; @@ -56,6 +53,7 @@ import java.net.URI; import java.util.Arrays; import java.util.Collections; +import java.util.Map; import java.util.UUID; import java.util.concurrent.Callable; import java.util.logging.Level; @@ -631,10 +629,24 @@ protected T execCommand( Command cmd ) throws WrappedResponse { * sometimes?) doesn't have much information in it: * * "User @jsmith is not permitted to perform requested action." + * + * Update (11/11/2024): + * + * An {@code isDetailedMessageRequired} flag has been added to {@code PermissionException} to selectively return more + * specific error messages when the generic message (e.g. "User :guest is not permitted to perform requested action") + * lacks sufficient context. This approach aims to provide valuable permission-related details in cases where it + * could help users better understand their permission issues without exposing unnecessary internal information. */ - throw new WrappedResponse(error(Response.Status.UNAUTHORIZED, - "User " + cmd.getRequest().getUser().getIdentifier() + " is not permitted to perform requested action.") ); - + if (ex.isDetailedMessageRequired()) { + throw new WrappedResponse(error(Response.Status.UNAUTHORIZED, ex.getMessage())); + } else { + throw new WrappedResponse(error(Response.Status.UNAUTHORIZED, + "User " + cmd.getRequest().getUser().getIdentifier() + " is not permitted to perform requested action.")); + } + } catch (InvalidFieldsCommandException ex) { + throw new WrappedResponse(ex, badRequest(ex.getMessage(), ex.getFieldErrors())); + } catch (InvalidCommandArgumentsException ex) { + throw new WrappedResponse(ex, error(Status.BAD_REQUEST, ex.getMessage())); } catch (CommandException ex) { Logger.getLogger(AbstractApiBean.class.getName()).log(Level.SEVERE, "Error while executing command " + cmd, ex); throw new WrappedResponse(ex, error(Status.INTERNAL_SERVER_ERROR, ex.getMessage())); @@ -809,6 +821,30 @@ protected Response badRequest( String msg ) { return error( Status.BAD_REQUEST, msg ); } + protected Response badRequest(String msg, Map fieldErrors) { + return Response.status(Status.BAD_REQUEST) + .entity(NullSafeJsonBuilder.jsonObjectBuilder() + .add("status", ApiConstants.STATUS_ERROR) + .add("message", msg) + .add("fieldErrors", Json.createObjectBuilder(fieldErrors).build()) + .build() + ) + .type(MediaType.APPLICATION_JSON_TYPE) + .build(); + } + + /** + * In short, your password is fine but you don't have permission. + * + * "The 403 (Forbidden) status code indicates that the server understood the + * request but refuses to authorize it. A server that wishes to make public + * why the request has been forbidden can describe that reason in the + * response payload (if any). + * + * If authentication credentials were provided in the request, the server + * considers them insufficient to grant access." -- + * https://datatracker.ietf.org/doc/html/rfc7231#section-6.5.3 + */ protected Response forbidden( String msg ) { return error( Status.FORBIDDEN, msg ); } @@ -830,9 +866,17 @@ protected Response permissionError( PermissionException pe ) { } protected Response permissionError( String message ) { - return unauthorized( message ); + return forbidden( message ); } + /** + * In short, bad password. + * + * "The 401 (Unauthorized) status code indicates that the request has not + * been applied because it lacks valid authentication credentials for the + * target resource." -- + * https://datatracker.ietf.org/doc/html/rfc7235#section-3.1 + */ protected Response unauthorized( String message ) { return error( Status.UNAUTHORIZED, message ); } diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Access.java b/src/main/java/edu/harvard/iq/dataverse/api/Access.java index 16ac884180b..2a27c89eaaa 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Access.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Access.java @@ -6,32 +6,7 @@ package edu.harvard.iq.dataverse.api; -import edu.harvard.iq.dataverse.AuxiliaryFile; -import edu.harvard.iq.dataverse.AuxiliaryFileServiceBean; -import edu.harvard.iq.dataverse.DataCitation; -import edu.harvard.iq.dataverse.DataFile; -import edu.harvard.iq.dataverse.FileAccessRequest; -import edu.harvard.iq.dataverse.FileMetadata; -import edu.harvard.iq.dataverse.DataFileServiceBean; -import edu.harvard.iq.dataverse.Dataset; -import edu.harvard.iq.dataverse.DatasetVersion; -import edu.harvard.iq.dataverse.DatasetVersionServiceBean; -import edu.harvard.iq.dataverse.DatasetServiceBean; -import edu.harvard.iq.dataverse.Dataverse; -import edu.harvard.iq.dataverse.DataverseRequestServiceBean; -import edu.harvard.iq.dataverse.DataverseRoleServiceBean; -import edu.harvard.iq.dataverse.DataverseServiceBean; -import edu.harvard.iq.dataverse.DataverseSession; -import edu.harvard.iq.dataverse.DataverseTheme; -import edu.harvard.iq.dataverse.FileDownloadServiceBean; -import edu.harvard.iq.dataverse.GuestbookResponse; -import edu.harvard.iq.dataverse.GuestbookResponseServiceBean; -import edu.harvard.iq.dataverse.PermissionServiceBean; -import edu.harvard.iq.dataverse.PermissionsWrapper; -import edu.harvard.iq.dataverse.RoleAssignment; -import edu.harvard.iq.dataverse.UserNotification; -import edu.harvard.iq.dataverse.UserNotificationServiceBean; -import edu.harvard.iq.dataverse.ThemeWidgetFragment; +import edu.harvard.iq.dataverse.*; import static edu.harvard.iq.dataverse.api.Datasets.handleVersion; @@ -52,18 +27,12 @@ import edu.harvard.iq.dataverse.dataaccess.ImageThumbConverter; import edu.harvard.iq.dataverse.datavariable.DataVariable; import edu.harvard.iq.dataverse.datavariable.VariableServiceBean; +import edu.harvard.iq.dataverse.dataverse.featured.DataverseFeaturedItem; +import edu.harvard.iq.dataverse.dataverse.featured.DataverseFeaturedItemServiceBean; import edu.harvard.iq.dataverse.engine.command.Command; import edu.harvard.iq.dataverse.engine.command.DataverseRequest; import edu.harvard.iq.dataverse.engine.command.exception.CommandException; -import edu.harvard.iq.dataverse.engine.command.impl.AssignRoleCommand; -import edu.harvard.iq.dataverse.engine.command.impl.GetDatasetCommand; -import edu.harvard.iq.dataverse.engine.command.impl.GetDraftDatasetVersionCommand; -import edu.harvard.iq.dataverse.engine.command.impl.GetLatestAccessibleDatasetVersionCommand; -import edu.harvard.iq.dataverse.engine.command.impl.GetLatestPublishedDatasetVersionCommand; -import edu.harvard.iq.dataverse.engine.command.impl.GetSpecificPublishedDatasetVersionCommand; -import edu.harvard.iq.dataverse.engine.command.impl.RequestAccessCommand; -import edu.harvard.iq.dataverse.engine.command.impl.RevokeRoleCommand; -import edu.harvard.iq.dataverse.engine.command.impl.UpdateDatasetVersionCommand; +import edu.harvard.iq.dataverse.engine.command.impl.*; import edu.harvard.iq.dataverse.export.DDIExportServiceBean; import edu.harvard.iq.dataverse.makedatacount.MakeDataCountLoggingServiceBean; import edu.harvard.iq.dataverse.makedatacount.MakeDataCountLoggingServiceBean.MakeDataCountEntry; @@ -88,7 +57,6 @@ import java.util.Arrays; import java.util.Date; import java.util.List; -import java.util.Properties; import java.util.logging.Level; import jakarta.inject.Inject; import jakarta.json.Json; @@ -133,7 +101,6 @@ import org.eclipse.microprofile.openapi.annotations.Operation; import org.eclipse.microprofile.openapi.annotations.media.Content; -import org.eclipse.microprofile.openapi.annotations.media.Schema; import org.eclipse.microprofile.openapi.annotations.parameters.RequestBody; import org.eclipse.microprofile.openapi.annotations.responses.APIResponse; import org.eclipse.microprofile.openapi.annotations.responses.APIResponses; @@ -199,6 +166,8 @@ public class Access extends AbstractApiBean { PermissionsWrapper permissionsWrapper; @Inject MakeDataCountLoggingServiceBean mdcLogService; + @Inject + DataverseFeaturedItemServiceBean dataverseFeaturedItemServiceBean; //@EJB @@ -2015,4 +1984,24 @@ private URI handleCustomZipDownload(User user, String customZipServiceUrl, Strin } return redirectUri; } + + @GET + @AuthRequired + @Produces({"image/png"}) + @Path("dataverseFeaturedItemImage/{itemId}") + public InputStream getDataverseFeatureItemImage(@Context ContainerRequestContext crc, @PathParam("itemId") Long itemId) { + DataverseFeaturedItem dataverseFeaturedItem; + try { + dataverseFeaturedItem = execCommand(new GetDataverseFeaturedItemCommand(createDataverseRequest(getRequestUser(crc)), dataverseFeaturedItemServiceBean.findById(itemId))); + } catch (WrappedResponse wr) { + logger.warning("Cannot locate a dataverse featured item with id " + itemId); + return null; + } + try { + return dataverseFeaturedItemServiceBean.getImageFileAsInputStream(dataverseFeaturedItem); + } catch (IOException e) { + logger.warning("Error while obtaining the input stream for the image file associated with the dataverse featured item with id " + itemId); + return null; + } + } } diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Admin.java b/src/main/java/edu/harvard/iq/dataverse/api/Admin.java index 152bcf5066e..2d850cc092f 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Admin.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Admin.java @@ -1,11 +1,30 @@ package edu.harvard.iq.dataverse.api; -import edu.harvard.iq.dataverse.*; +import edu.harvard.iq.dataverse.BannerMessage; +import edu.harvard.iq.dataverse.BannerMessageServiceBean; +import edu.harvard.iq.dataverse.BannerMessageText; +import edu.harvard.iq.dataverse.DataFile; +import edu.harvard.iq.dataverse.DataFileServiceBean; +import edu.harvard.iq.dataverse.Dataset; +import edu.harvard.iq.dataverse.DatasetServiceBean; +import edu.harvard.iq.dataverse.DatasetVersion; +import edu.harvard.iq.dataverse.DatasetVersionServiceBean; +import edu.harvard.iq.dataverse.Dataverse; +import edu.harvard.iq.dataverse.DataverseRequestServiceBean; +import edu.harvard.iq.dataverse.DataverseServiceBean; +import edu.harvard.iq.dataverse.DataverseSession; +import edu.harvard.iq.dataverse.DvObject; +import edu.harvard.iq.dataverse.DvObjectServiceBean; +import edu.harvard.iq.dataverse.FileMetadata; import edu.harvard.iq.dataverse.api.auth.AuthRequired; import edu.harvard.iq.dataverse.settings.JvmSettings; import edu.harvard.iq.dataverse.util.StringUtil; import edu.harvard.iq.dataverse.util.json.NullSafeJsonBuilder; import edu.harvard.iq.dataverse.validation.EMailValidator; +import edu.harvard.iq.dataverse.EjbDataverseEngine; +import edu.harvard.iq.dataverse.Template; +import edu.harvard.iq.dataverse.TemplateServiceBean; +import edu.harvard.iq.dataverse.UserServiceBean; import edu.harvard.iq.dataverse.actionlogging.ActionLogRecord; import edu.harvard.iq.dataverse.api.dto.RoleDTO; import edu.harvard.iq.dataverse.authorization.AuthenticatedUserDisplayInfo; @@ -49,7 +68,8 @@ import java.io.InputStream; import java.io.StringReader; import java.nio.charset.StandardCharsets; -import java.util.*; +import java.util.Collections; +import java.util.Map; import java.util.Map.Entry; import java.util.function.Predicate; import java.util.logging.Level; @@ -65,6 +85,7 @@ import org.apache.commons.io.IOUtils; +import java.util.List; import edu.harvard.iq.dataverse.authorization.AuthTestDataServiceBean; import edu.harvard.iq.dataverse.authorization.AuthenticationProvidersRegistrationServiceBean; import edu.harvard.iq.dataverse.authorization.DataverseRole; @@ -101,7 +122,9 @@ import static edu.harvard.iq.dataverse.util.json.JsonPrinter.json; import static edu.harvard.iq.dataverse.util.json.JsonPrinter.rolesToJson; import static edu.harvard.iq.dataverse.util.json.JsonPrinter.toJsonArray; - +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Date; import jakarta.inject.Inject; import jakarta.json.JsonArray; import jakarta.persistence.Query; @@ -109,6 +132,7 @@ import jakarta.ws.rs.WebApplicationException; import jakarta.ws.rs.core.StreamingOutput; import java.nio.file.Paths; +import java.util.TreeMap; /** * Where the secure, setup API calls live. @@ -990,6 +1014,22 @@ public Response createNewBuiltinRole(RoleDTO roleDto) { actionLogSvc.log(alr); } } + @Path("roles/{id}") + @PUT + public Response updateBuiltinRole(RoleDTO roleDto, @PathParam("id") long roleId) { + ActionLogRecord alr = new ActionLogRecord(ActionLogRecord.ActionType.Admin, "updateBuiltInRole") + .setInfo(roleDto.getAlias() + ":" + roleDto.getDescription()); + try { + DataverseRole role = roleDto.updateRoleFromDTO(rolesSvc.find(roleId)); + return ok(json(rolesSvc.save(role))); + } catch (Exception e) { + alr.setActionResult(ActionLogRecord.Result.InternalError); + alr.setInfo(alr.getInfo() + "// " + e.getMessage()); + return error(Response.Status.INTERNAL_SERVER_ERROR, e.getMessage()); + } finally { + actionLogSvc.log(alr); + } + } @Path("roles") @GET diff --git a/src/main/java/edu/harvard/iq/dataverse/api/DatasetFieldServiceApi.java b/src/main/java/edu/harvard/iq/dataverse/api/DatasetFieldServiceApi.java index 907295ad848..f29387aeb56 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/DatasetFieldServiceApi.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/DatasetFieldServiceApi.java @@ -42,6 +42,7 @@ import java.util.logging.Logger; import jakarta.persistence.NoResultException; import jakarta.persistence.TypedQuery; +import jakarta.ws.rs.QueryParam; import jakarta.ws.rs.core.Response.Status; import java.io.BufferedInputStream; @@ -545,4 +546,19 @@ public static String getDataverseLangDirectory() { return dataverseLangDirectory; } + /** + * Set setDisplayOnCreate for a DatasetFieldType. + */ + @POST + @Path("/setDisplayOnCreate") + public Response setDisplayOnCreate(@QueryParam("datasetFieldType") String datasetFieldTypeIn, @QueryParam("setDisplayOnCreate") Boolean setDisplayOnCreateIn) { + DatasetFieldType dft = datasetFieldService.findByName(datasetFieldTypeIn); + if (dft == null) { + return error(Status.NOT_FOUND, "Cound not find a DatasetFieldType by looking up " + datasetFieldTypeIn); + } + dft.setDisplayOnCreate(setDisplayOnCreateIn); + DatasetFieldType saved = datasetFieldService.save(dft); + return ok("DisplayOnCreate for DatasetFieldType " + saved.getName() + " is now " + saved.isDisplayOnCreate()); + } + } diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index 2ec10816acc..2d5b5848f71 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -3,6 +3,7 @@ import com.amazonaws.services.s3.model.PartETag; import edu.harvard.iq.dataverse.*; import edu.harvard.iq.dataverse.DatasetLock.Reason; +import edu.harvard.iq.dataverse.DatasetVersion.VersionState; import edu.harvard.iq.dataverse.actionlogging.ActionLogRecord; import edu.harvard.iq.dataverse.api.auth.AuthRequired; import edu.harvard.iq.dataverse.api.dto.RoleAssignmentDTO; @@ -69,7 +70,6 @@ import org.apache.commons.lang3.StringUtils; import org.eclipse.microprofile.openapi.annotations.Operation; import org.eclipse.microprofile.openapi.annotations.media.Content; -import org.eclipse.microprofile.openapi.annotations.media.Schema; import org.eclipse.microprofile.openapi.annotations.parameters.RequestBody; import org.eclipse.microprofile.openapi.annotations.responses.APIResponse; import org.eclipse.microprofile.openapi.annotations.tags.Tag; @@ -99,12 +99,14 @@ import static edu.harvard.iq.dataverse.api.ApiConstants.*; import edu.harvard.iq.dataverse.engine.command.exception.IllegalCommandException; +import edu.harvard.iq.dataverse.engine.command.exception.PermissionException; import edu.harvard.iq.dataverse.dataset.DatasetType; import edu.harvard.iq.dataverse.dataset.DatasetTypeServiceBean; import static edu.harvard.iq.dataverse.util.json.JsonPrinter.*; import static edu.harvard.iq.dataverse.util.json.NullSafeJsonBuilder.jsonObjectBuilder; import static jakarta.ws.rs.core.Response.Status.BAD_REQUEST; import static jakarta.ws.rs.core.Response.Status.NOT_FOUND; +import static jakarta.ws.rs.core.Response.Status.FORBIDDEN; @Path("datasets") public class Datasets extends AbstractApiBean { @@ -119,7 +121,10 @@ public class Datasets extends AbstractApiBean { @EJB DataverseServiceBean dataverseService; - + + @EJB + GuestbookResponseServiceBean guestbookResponseService; + @EJB GlobusServiceBean globusService; @@ -421,15 +426,16 @@ public Response useDefaultCitationDate(@Context ContainerRequestContext crc, @Pa @GET @AuthRequired @Path("{id}/versions") - public Response listVersions(@Context ContainerRequestContext crc, @PathParam("id") String id, @QueryParam("excludeFiles") Boolean excludeFiles, @QueryParam("limit") Integer limit, @QueryParam("offset") Integer offset) { + public Response listVersions(@Context ContainerRequestContext crc, @PathParam("id") String id, @QueryParam("excludeFiles") Boolean excludeFiles,@QueryParam("excludeMetadataBlocks") Boolean excludeMetadataBlocks, @QueryParam("limit") Integer limit, @QueryParam("offset") Integer offset) { return response( req -> { Dataset dataset = findDatasetOrDie(id); Boolean deepLookup = excludeFiles == null ? true : !excludeFiles; + Boolean includeMetadataBlocks = excludeMetadataBlocks == null ? true : !excludeMetadataBlocks; return ok( execCommand( new ListVersionsCommand(req, dataset, offset, limit, deepLookup) ) .stream() - .map( d -> json(d, deepLookup) ) + .map( d -> json(d, deepLookup, includeMetadataBlocks) ) .collect(toJsonArray())); }, getRequestUser(crc)); } @@ -441,6 +447,7 @@ public Response getVersion(@Context ContainerRequestContext crc, @PathParam("id") String datasetId, @PathParam("versionId") String versionId, @QueryParam("excludeFiles") Boolean excludeFiles, + @QueryParam("excludeMetadataBlocks") Boolean excludeMetadataBlocks, @QueryParam("includeDeaccessioned") boolean includeDeaccessioned, @QueryParam("returnOwners") boolean returnOwners, @Context UriInfo uriInfo, @@ -466,11 +473,12 @@ public Response getVersion(@Context ContainerRequestContext crc, if (excludeFiles == null ? true : !excludeFiles) { requestedDatasetVersion = datasetversionService.findDeep(requestedDatasetVersion.getId()); } + Boolean includeMetadataBlocks = excludeMetadataBlocks == null ? true : !excludeMetadataBlocks; JsonObjectBuilder jsonBuilder = json(requestedDatasetVersion, null, - excludeFiles == null ? true : !excludeFiles, - returnOwners); + excludeFiles == null ? true : !excludeFiles, + returnOwners, includeMetadataBlocks); return ok(jsonBuilder); }, getRequestUser(crc)); @@ -556,6 +564,41 @@ public Response getVersionFileCounts(@Context ContainerRequestContext crc, }, getRequestUser(crc)); } + @GET + @AuthRequired + @Path("{id}/download/count") + public Response getDownloadCountByDatasetId(@Context ContainerRequestContext crc, + @PathParam("id") String datasetId, + @QueryParam("includeMDC") Boolean includeMDC) { + Long id; + Long count; + LocalDate date = includeMDC == null || !includeMDC ? getMDCStartDate() : null; + try { + Dataset ds = findDatasetOrDie(datasetId); + id = ds.getId(); + count = guestbookResponseService.getDownloadCountByDatasetId(id, date); + } catch (WrappedResponse wr) { + return wr.getResponse(); + } + JsonObjectBuilder job = Json.createObjectBuilder() + .add("id", id) + .add("downloadCount", count); + if (date != null) { + job.add("MDCStartDate" , date.format(DateTimeFormatter.ofPattern("yyyy-MM-dd"))); + } + return Response.ok(job.build()) + .type(MediaType.APPLICATION_JSON) + .build(); + } + private LocalDate getMDCStartDate() { + String date = settingsService.getValueForKey(SettingsServiceBean.Key.MDCStartDate); + LocalDate ld=null; + if(date!=null) { + ld = LocalDate.parse(date); + } + return ld; + } + @GET @AuthRequired @Path("{id}/dirindex") @@ -1184,8 +1227,14 @@ private String validateDatasetFieldValues(List fields) { if (dsf.getDatasetFieldType().isAllowMultiples() && dsf.getControlledVocabularyValues().isEmpty() && dsf.getDatasetFieldCompoundValues().isEmpty() && dsf.getDatasetFieldValues().isEmpty()) { error.append("Empty multiple value for field: ").append(dsf.getDatasetFieldType().getDisplayName()).append(" "); - } else if (!dsf.getDatasetFieldType().isAllowMultiples() && dsf.getSingleValue().getValue().isEmpty()) { - error.append("Empty value for field: ").append(dsf.getDatasetFieldType().getDisplayName()).append(" "); + } else if (!dsf.getDatasetFieldType().isAllowMultiples()) { + if (dsf.getDatasetFieldType().isControlledVocabulary() && dsf.getSingleControlledVocabularyValue().getStrValue().isEmpty()) { + error.append("Empty cvoc value for field: ").append(dsf.getDatasetFieldType().getDisplayName()).append(" "); + } else if (dsf.getDatasetFieldType().isCompound() && dsf.getDatasetFieldCompoundValues().isEmpty()) { + error.append("Empty compound value for field: ").append(dsf.getDatasetFieldType().getDisplayName()).append(" "); + } else if (!dsf.getDatasetFieldType().isControlledVocabulary() && !dsf.getDatasetFieldType().isCompound() && dsf.getSingleValue().getValue().isEmpty()) { + error.append("Empty value for field: ").append(dsf.getDatasetFieldType().getDisplayName()).append(" "); + } } } @@ -3036,6 +3085,62 @@ public Response getCompareVersions(@Context ContainerRequestContext crc, @PathPa return wr.getResponse(); } } + + @GET + @AuthRequired + @Path("{id}/versions/compareSummary") + public Response getCompareVersionsSummary(@Context ContainerRequestContext crc, @PathParam("id") String id, + @Context UriInfo uriInfo, @Context HttpHeaders headers) { + try { + Dataset dataset = findDatasetOrDie(id); + User user = getRequestUser(crc); + JsonArrayBuilder differenceSummaries = Json.createArrayBuilder(); + + for (DatasetVersion dv : dataset.getVersions()) { + //only get summaries of draft is user may view unpublished + + if (dv.isPublished() || permissionService.hasPermissionsFor(user, dv.getDataset(), + EnumSet.of(Permission.ViewUnpublishedDataset))) { + + JsonObjectBuilder versionBuilder = new NullSafeJsonBuilder(); + versionBuilder.add("id", dv.getId()); + versionBuilder.add("versionNumber", dv.getFriendlyVersionNumber()); + DatasetVersionDifference dvdiff = dv.getDefaultVersionDifference(); + if (dvdiff == null) { + if (dv.isReleased()) { + if (dv.getPriorVersionState() == null) { + versionBuilder.add("summary", "firstPublished"); + } + if (dv.getPriorVersionState() != null && dv.getPriorVersionState().equals(VersionState.DEACCESSIONED)) { + versionBuilder.add("summary", "previousVersionDeaccessioned"); + } + } + if (dv.isDraft()) { + if (dv.getPriorVersionState() == null) { + versionBuilder.add("summary", "firstDraft"); + } + if (dv.getPriorVersionState() != null && dv.getPriorVersionState().equals(VersionState.DEACCESSIONED)) { + versionBuilder.add("summary", "previousVersionDeaccessioned"); + } + } + if (dv.isDeaccessioned()) { + versionBuilder.add("summary", "versionDeaccessioned"); + } + + } else { + versionBuilder.add("summary", dvdiff.getSummaryDifferenceAsJson()); + } + + versionBuilder.add("contributors", datasetversionService.getContributorsNames(dv)); + versionBuilder.add("publishedOn", !dv.isDraft() ? dv.getPublicationDateAsString() : ""); + differenceSummaries.add(versionBuilder); + } + } + return ok(differenceSummaries); + } catch (WrappedResponse wr) { + return wr.getResponse(); + } + } private static Set getDatasetFilenames(Dataset dataset) { Set files = new HashSet<>(); @@ -4018,7 +4123,7 @@ public Response requestGlobusUpload(@Context ContainerRequestContext crc, @PathP case 400: return badRequest("Unable to grant permission"); case 409: - return conflict("Permission already exists"); + return conflict("Permission already exists or no more permissions allowed"); default: return error(null, "Unexpected error when granting permission"); } @@ -4389,7 +4494,7 @@ public Response requestGlobusDownload(@Context ContainerRequestContext crc, @Pat case 400: return badRequest("Unable to grant permission"); case 409: - return conflict("Permission already exists"); + return conflict("Permission already exists or no more permissions allowed"); default: return error(null, "Unexpected error when granting permission"); } @@ -4443,8 +4548,17 @@ public Response monitorGlobusDownload(@Context ContainerRequestContext crc, @Pat return wr.getResponse(); } + JsonObject jsonObject = null; + try { + jsonObject = JsonUtil.getJsonObject(jsonData); + } catch (Exception ex) { + logger.warning("Globus download monitoring: error parsing json: " + jsonData + " " + ex.getMessage()); + return badRequest("Error parsing json body"); + + } + // Async Call - globusService.globusDownload(jsonData, dataset, authUser); + globusService.globusDownload(jsonObject, dataset, authUser); return ok("Async call to Globus Download started"); @@ -4927,23 +5041,68 @@ public Response getPreviewUrlDatasetVersionCitation(@PathParam("previewUrlToken" } DatasetVersion dsv = privateUrlService.getDraftDatasetVersionFromToken(previewUrlToken); return (dsv == null || dsv.getId() == null) ? notFound("Dataset version not found") - : ok(dsv.getCitation(true, privateUrlUser.hasAnonymizedAccess())); + : ok(dsv.getCitation(DataCitation.Format.Internal, true, privateUrlUser.hasAnonymizedAccess())); } @GET @AuthRequired @Path("{id}/versions/{versionId}/citation") - public Response getDatasetVersionCitation(@Context ContainerRequestContext crc, - @PathParam("id") String datasetId, - @PathParam("versionId") String versionId, - @QueryParam("includeDeaccessioned") boolean includeDeaccessioned, - @Context UriInfo uriInfo, - @Context HttpHeaders headers) { + public Response getDatasetVersionInternalCitation(@Context ContainerRequestContext crc, + @PathParam("id") String datasetId, @PathParam("versionId") String versionId, + @QueryParam("includeDeaccessioned") boolean includeDeaccessioned, @Context UriInfo uriInfo, + @Context HttpHeaders headers) { + try { + return ok(getDatasetVersionCitationAsString(crc, datasetId, versionId, DataCitation.Format.Internal, includeDeaccessioned, + uriInfo, headers)); + } catch (WrappedResponse wr) { + return wr.getResponse(); + } + } + + /** + * Returns one of the DataCitation.Format types as a raw file download (not wrapped in our ok json) + * @param crc + * @param datasetId + * @param versionId + * @param formatString + * @param includeDeaccessioned + * @param uriInfo + * @param headers + * @return + */ + @GET + @AuthRequired + @Path("{id}/versions/{versionId}/citation/{format}") + public Response getDatasetVersionCitation(@Context ContainerRequestContext crc, @PathParam("id") String datasetId, + @PathParam("versionId") String versionId, @PathParam("format") String formatString, + @QueryParam("includeDeaccessioned") boolean includeDeaccessioned, @Context UriInfo uriInfo, + @Context HttpHeaders headers) { + + DataCitation.Format format; + try { + format = DataCitation.Format.valueOf(formatString); + } catch (IllegalArgumentException e) { + return badRequest(BundleUtil.getStringFromBundle("datasets.api.citation.invalidFormat")); + } + try { + //ToDo - add ContentDisposition to support downloading with a file name + return Response.ok().type(DataCitation.getCitationFormatMediaType(format, true)).entity( + getDatasetVersionCitationAsString(crc, datasetId, versionId, format, includeDeaccessioned, uriInfo, headers)) + .build(); + } catch (WrappedResponse wr) { + return wr.getResponse(); + } + } + + public String getDatasetVersionCitationAsString(ContainerRequestContext crc, String datasetId, String versionId, + DataCitation.Format format, boolean includeDeaccessioned, UriInfo uriInfo, HttpHeaders headers) + throws IllegalArgumentException, WrappedResponse { boolean checkFilePerms = false; - return response(req -> ok( - getDatasetVersionOrDie(req, versionId, findDatasetOrDie(datasetId), uriInfo, headers, - includeDeaccessioned, checkFilePerms).getCitation(true, false)), - getRequestUser(crc)); + + DataverseRequest req = createDataverseRequest(getRequestUser(crc)); + DatasetVersion dsv = getDatasetVersionOrDie(req, versionId, findDatasetOrDie(datasetId), uriInfo, headers, + includeDeaccessioned, checkFilePerms); + return dsv.getCitation(format, true, false); } @POST @@ -4957,11 +5116,11 @@ public Response deaccessionDataset(@Context ContainerRequestContext crc, @PathPa DatasetVersion datasetVersion = getDatasetVersionOrDie(req, versionId, findDatasetOrDie(datasetId), uriInfo, headers); try { JsonObject jsonObject = JsonUtil.getJsonObject(jsonBody); - datasetVersion.setVersionNote(jsonObject.getString("deaccessionReason")); + datasetVersion.setDeaccessionNote(jsonObject.getString("deaccessionReason")); String deaccessionForwardURL = jsonObject.getString("deaccessionForwardURL", null); if (deaccessionForwardURL != null) { try { - datasetVersion.setArchiveNote(deaccessionForwardURL); + datasetVersion.setDeaccessionLink(deaccessionForwardURL); } catch (IllegalArgumentException iae) { return badRequest(BundleUtil.getStringFromBundle("datasets.api.deaccessionDataset.invalid.forward.url", List.of(iae.getMessage()))); } @@ -5193,14 +5352,10 @@ public Response resetPidGenerator(@Context ContainerRequestContext crc, @PathPar @Path("datasetTypes") public Response getDatasetTypes() { JsonArrayBuilder jab = Json.createArrayBuilder(); - List datasetTypes = datasetTypeSvc.listAll(); - for (DatasetType datasetType : datasetTypes) { - JsonObjectBuilder job = Json.createObjectBuilder(); - job.add("id", datasetType.getId()); - job.add("name", datasetType.getName()); - jab.add(job); + for (DatasetType datasetType : datasetTypeSvc.listAll()) { + jab.add(datasetType.toJson()); } - return ok(jab.build()); + return ok(jab); } @GET @@ -5315,4 +5470,186 @@ public Response deleteDatasetType(@Context ContainerRequestContext crc, @PathPar } } + @AuthRequired + @PUT + @Path("datasetTypes/{idOrName}") + public Response updateDatasetTypeLinksWithMetadataBlocks(@Context ContainerRequestContext crc, @PathParam("idOrName") String idOrName, String jsonBody) { + DatasetType datasetType = null; + if (StringUtils.isNumeric(idOrName)) { + try { + long id = Long.parseLong(idOrName); + datasetType = datasetTypeSvc.getById(id); + } catch (NumberFormatException ex) { + return error(NOT_FOUND, "Could not find a dataset type with id " + idOrName); + } + } else { + datasetType = datasetTypeSvc.getByName(idOrName); + } + JsonArrayBuilder datasetTypesBefore = Json.createArrayBuilder(); + for (MetadataBlock metadataBlock : datasetType.getMetadataBlocks()) { + datasetTypesBefore.add(metadataBlock.getName()); + } + JsonArrayBuilder datasetTypesAfter = Json.createArrayBuilder(); + List metadataBlocksToSave = new ArrayList<>(); + if (jsonBody != null && !jsonBody.isEmpty()) { + JsonArray json = JsonUtil.getJsonArray(jsonBody); + for (JsonString jsonValue : json.getValuesAs(JsonString.class)) { + String name = jsonValue.getString(); + MetadataBlock metadataBlock = metadataBlockSvc.findByName(name); + if (metadataBlock != null) { + metadataBlocksToSave.add(metadataBlock); + datasetTypesAfter.add(name); + } else { + String availableBlocks = metadataBlockSvc.listMetadataBlocks().stream().map(MetadataBlock::getName).collect(Collectors.joining(", ")); + return badRequest("Metadata block not found: " + name + ". Available metadata blocks: " + availableBlocks); + } + } + } + try { + execCommand(new UpdateDatasetTypeLinksToMetadataBlocksCommand(createDataverseRequest(getRequestUser(crc)), datasetType, metadataBlocksToSave)); + return ok(Json.createObjectBuilder() + .add("linkedMetadataBlocks", Json.createObjectBuilder() + .add("before", datasetTypesBefore) + .add("after", datasetTypesAfter)) + ); + + } catch (WrappedResponse ex) { + return ex.getResponse(); + } + } + + @PUT + @AuthRequired + @Path("{id}/deleteFiles") + @Consumes(MediaType.APPLICATION_JSON) + public Response deleteDatasetFiles(@Context ContainerRequestContext crc, @PathParam("id") String id, + JsonArray fileIds) { + try { + getRequestAuthenticatedUserOrDie(crc); + } catch (WrappedResponse ex) { + return ex.getResponse(); + } + return response(req -> { + Dataset dataset = findDatasetOrDie(id); + // Convert JsonArray to List + List fileIdList = new ArrayList<>(); + for (JsonValue value : fileIds) { + fileIdList.add(((JsonNumber) value).longValue()); + } + // Find the files to be deleted + List filesToDelete = dataset.getOrCreateEditVersion().getFileMetadatas().stream() + .filter(fileMetadata -> fileIdList.contains(fileMetadata.getDataFile().getId())) + .collect(Collectors.toList()); + + if (filesToDelete.isEmpty()) { + return badRequest("No files found with the provided IDs."); + } + + if (filesToDelete.size() != fileIds.size()) { + return badRequest( + "Some files listed are not present in the latest dataset version and cannot be deleted."); + } + try { + + UpdateDatasetVersionCommand update_cmd = new UpdateDatasetVersionCommand(dataset, req, filesToDelete); + + commandEngine.submit(update_cmd); + for (FileMetadata fm : filesToDelete) { + DataFile dataFile = fm.getDataFile(); + boolean deletePhysicalFile = !dataFile.isReleased(); + if (deletePhysicalFile) { + try { + fileService.finalizeFileDelete(dataFile.getId(), + fileService.getPhysicalFileToDelete(dataFile)); + } catch (IOException ioex) { + logger.warning("Failed to delete the physical file associated with the deleted datafile id=" + + dataFile.getId() + ", storage location: " + + fileService.getPhysicalFileToDelete(dataFile)); + } + } + } + } catch (PermissionException ex) { + return error(FORBIDDEN, "You do not have permission to delete files ont this dataset."); + } catch (CommandException ex) { + return error(BAD_REQUEST, + "File deletes failed for dataset ID " + id + " (CommandException): " + ex.getMessage()); + } catch (EJBException ex) { + return error(jakarta.ws.rs.core.Response.Status.INTERNAL_SERVER_ERROR, + "File deletes failed for dataset ID " + id + "(EJBException): " + ex.getMessage()); + } + return ok(fileIds.size() + " files deleted successfully"); + + }, getRequestUser(crc)); + } + +@GET + @AuthRequired + @Path("{id}/versions/{versionId}/versionNote") + public Response getVersionCreationNote(@Context ContainerRequestContext crc, @PathParam("id") String datasetId, @PathParam("versionId") String versionId, @Context UriInfo uriInfo, @Context HttpHeaders headers) throws WrappedResponse { + + return response(req -> { + DatasetVersion datasetVersion = getDatasetVersionOrDie(req, versionId, findDatasetOrDie(datasetId), uriInfo, headers); + String note = datasetVersion.getVersionNote(); + if(note == null) { + return ok(Json.createObjectBuilder()); + } + return ok(note); + }, getRequestUser(crc)); + } + + @PUT + @AuthRequired + @Path("{id}/versions/{versionId}/versionNote") + public Response addVersionNote(@Context ContainerRequestContext crc, @PathParam("id") String datasetId, @PathParam("versionId") String versionId, String note, @Context UriInfo uriInfo, @Context HttpHeaders headers) throws WrappedResponse { + if (!FeatureFlags.VERSION_NOTE.enabled()) { + return notFound(BundleUtil.getStringFromBundle("datasets.api.addVersionNote.notEnabled")); + } + if (!DS_VERSION_DRAFT.equals(versionId)) { + try { + AuthenticatedUser user = getRequestAuthenticatedUserOrDie(crc); + + if (!user.isSuperuser()) { + return forbidden(BundleUtil.getStringFromBundle("datasets.api.addVersionNote.forbidden")); + } + return response(req -> { + DatasetVersion datasetVersion = getDatasetVersionOrDie(req, versionId, findDatasetOrDie(datasetId), uriInfo, headers); + datasetVersion.setVersionNote(note); + execCommand(new UpdatePublishedDatasetVersionCommand(req, datasetVersion)); + return ok("Note added to version " + datasetVersion.getFriendlyVersionNumber()); + }, getRequestUser(crc)); + } catch (WrappedResponse ex) { + return ex.getResponse(); + } + } + return response(req -> { + DatasetVersion datasetVersion = findDatasetOrDie(datasetId).getOrCreateEditVersion(); + datasetVersion.setVersionNote(note); + execCommand(new UpdateDatasetVersionCommand(datasetVersion.getDataset(), req)); + + return ok("Note added"); + }, getRequestUser(crc)); + } + + @DELETE + @AuthRequired + @Path("{id}/versions/{versionId}/versionNote") + public Response deleteVersionNote(@Context ContainerRequestContext crc, @PathParam("id") String datasetId, @PathParam("versionId") String versionId, @Context UriInfo uriInfo, @Context HttpHeaders headers) throws WrappedResponse { + if(!FeatureFlags.VERSION_NOTE.enabled()) { + return notFound(BundleUtil.getStringFromBundle("datasets.api.addVersionNote.notEnabled")); + } + if (!DS_VERSION_DRAFT.equals(versionId)) { + AuthenticatedUser user = getRequestAuthenticatedUserOrDie(crc); + if (!user.isSuperuser()) { + return forbidden(BundleUtil.getStringFromBundle("datasets.api.addVersionNote.forbidden")); + } + } + return response(req -> { + DatasetVersion datasetVersion = getDatasetVersionOrDie(req, versionId, findDatasetOrDie(datasetId), uriInfo, headers); + datasetVersion.setVersionNote(null); + execCommand(new UpdateDatasetVersionCommand(datasetVersion.getDataset(), req)); + + return ok("Note deleted"); + }, getRequestUser(crc)); + } + } diff --git a/src/main/java/edu/harvard/iq/dataverse/api/DataverseFeaturedItems.java b/src/main/java/edu/harvard/iq/dataverse/api/DataverseFeaturedItems.java new file mode 100644 index 00000000000..a77ea000415 --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/api/DataverseFeaturedItems.java @@ -0,0 +1,69 @@ +package edu.harvard.iq.dataverse.api; + +import edu.harvard.iq.dataverse.api.auth.AuthRequired; +import edu.harvard.iq.dataverse.api.dto.UpdatedDataverseFeaturedItemDTO; +import edu.harvard.iq.dataverse.dataverse.featured.DataverseFeaturedItem; +import edu.harvard.iq.dataverse.dataverse.featured.DataverseFeaturedItemServiceBean; +import edu.harvard.iq.dataverse.engine.command.impl.*; +import edu.harvard.iq.dataverse.util.BundleUtil; +import jakarta.ejb.Stateless; +import jakarta.inject.Inject; +import jakarta.ws.rs.*; +import jakarta.ws.rs.container.ContainerRequestContext; +import jakarta.ws.rs.core.Context; +import jakarta.ws.rs.core.MediaType; +import jakarta.ws.rs.core.Response; +import org.glassfish.jersey.media.multipart.FormDataContentDisposition; +import org.glassfish.jersey.media.multipart.FormDataParam; + +import java.io.InputStream; +import java.text.MessageFormat; + +import static edu.harvard.iq.dataverse.util.json.JsonPrinter.json; + +@Stateless +@Path("dataverseFeaturedItems") +public class DataverseFeaturedItems extends AbstractApiBean { + + @Inject + DataverseFeaturedItemServiceBean dataverseFeaturedItemServiceBean; + + @DELETE + @AuthRequired + @Path("{id}") + public Response deleteFeaturedItem(@Context ContainerRequestContext crc, @PathParam("id") Long id) { + try { + DataverseFeaturedItem dataverseFeaturedItem = dataverseFeaturedItemServiceBean.findById(id); + if (dataverseFeaturedItem == null) { + throw new WrappedResponse(error(Response.Status.NOT_FOUND, MessageFormat.format(BundleUtil.getStringFromBundle("dataverseFeaturedItems.errors.notFound"), id))); + } + execCommand(new DeleteDataverseFeaturedItemCommand(createDataverseRequest(getRequestUser(crc)), dataverseFeaturedItem)); + return ok(MessageFormat.format(BundleUtil.getStringFromBundle("dataverseFeaturedItems.delete.successful"), id)); + } catch (WrappedResponse e) { + return e.getResponse(); + } + } + + @PUT + @AuthRequired + @Consumes(MediaType.MULTIPART_FORM_DATA) + @Path("{id}") + public Response updateFeaturedItem(@Context ContainerRequestContext crc, + @PathParam("id") Long id, + @FormDataParam("content") String content, + @FormDataParam("displayOrder") int displayOrder, + @FormDataParam("keepFile") boolean keepFile, + @FormDataParam("file") InputStream imageFileInputStream, + @FormDataParam("file") FormDataContentDisposition contentDispositionHeader) { + try { + DataverseFeaturedItem dataverseFeaturedItem = dataverseFeaturedItemServiceBean.findById(id); + if (dataverseFeaturedItem == null) { + throw new WrappedResponse(error(Response.Status.NOT_FOUND, MessageFormat.format(BundleUtil.getStringFromBundle("dataverseFeaturedItems.errors.notFound"), id))); + } + UpdatedDataverseFeaturedItemDTO updatedDataverseFeaturedItemDTO = UpdatedDataverseFeaturedItemDTO.fromFormData(content, displayOrder, keepFile, imageFileInputStream, contentDispositionHeader); + return ok(json(execCommand(new UpdateDataverseFeaturedItemCommand(createDataverseRequest(getRequestUser(crc)), dataverseFeaturedItem, updatedDataverseFeaturedItemDTO)))); + } catch (WrappedResponse e) { + return e.getResponse(); + } + } +} diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java index f864a5a9d1c..3564a07d984 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java @@ -1,5 +1,7 @@ package edu.harvard.iq.dataverse.api; +import com.google.common.collect.Lists; +import com.google.api.client.util.ArrayMap; import edu.harvard.iq.dataverse.*; import edu.harvard.iq.dataverse.api.auth.AuthRequired; import edu.harvard.iq.dataverse.api.datadeposit.SwordServiceBean; @@ -15,7 +17,10 @@ import edu.harvard.iq.dataverse.authorization.groups.impl.explicit.ExplicitGroupServiceBean; import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser; import edu.harvard.iq.dataverse.authorization.users.User; +import edu.harvard.iq.dataverse.dataset.DatasetType; import edu.harvard.iq.dataverse.dataverse.DataverseUtil; +import edu.harvard.iq.dataverse.dataverse.featured.DataverseFeaturedItem; +import edu.harvard.iq.dataverse.dataverse.featured.DataverseFeaturedItemServiceBean; import edu.harvard.iq.dataverse.engine.command.DataverseRequest; import edu.harvard.iq.dataverse.engine.command.impl.*; import edu.harvard.iq.dataverse.pidproviders.PidProvider; @@ -33,7 +38,7 @@ import edu.harvard.iq.dataverse.util.json.JsonPrinter; import edu.harvard.iq.dataverse.util.json.JsonUtil; -import java.io.StringReader; +import java.io.*; import java.util.*; import java.util.logging.Level; import java.util.logging.Logger; @@ -59,8 +64,6 @@ import jakarta.ws.rs.core.Response; import jakarta.ws.rs.core.Response.Status; -import java.io.IOException; -import java.io.OutputStream; import java.text.MessageFormat; import java.text.SimpleDateFormat; import java.util.stream.Collectors; @@ -68,6 +71,10 @@ import jakarta.ws.rs.WebApplicationException; import jakarta.ws.rs.core.Context; import jakarta.ws.rs.core.StreamingOutput; +import org.glassfish.jersey.media.multipart.FormDataBodyPart; +import org.glassfish.jersey.media.multipart.FormDataContentDisposition; +import org.glassfish.jersey.media.multipart.FormDataParam; + import javax.xml.stream.XMLStreamException; /** @@ -111,7 +118,10 @@ public class Dataverses extends AbstractApiBean { @EJB PermissionServiceBean permissionService; - + + @EJB + DataverseFeaturedItemServiceBean dataverseFeaturedItemServiceBean; + @POST @AuthRequired public Response addRoot(@Context ContainerRequestContext crc, String body) { @@ -195,7 +205,7 @@ public Response updateDataverse(@Context ContainerRequestContext crc, String bod List facets = parseFacets(body); AuthenticatedUser u = getRequestAuthenticatedUserOrDie(crc); - dataverse = execCommand(new UpdateDataverseCommand(dataverse, facets, null, createDataverseRequest(u), inputLevels, metadataBlocks, updatedDataverseDTO, true)); + dataverse = execCommand(new UpdateDataverseCommand(dataverse, facets, null, createDataverseRequest(u), inputLevels, metadataBlocks, updatedDataverseDTO)); return ok(json(dataverse)); } catch (WrappedResponse ww) { @@ -221,31 +231,60 @@ private DataverseDTO parseAndValidateUpdateDataverseRequestBody(String body) thr } } + /* + return null - ignore + return empty list - delete and inherit from parent + return non-empty list - update + */ private List parseInputLevels(String body, Dataverse dataverse) throws WrappedResponse { JsonObject metadataBlocksJson = getMetadataBlocksJson(body); - if (metadataBlocksJson == null) { - return null; + JsonArray inputLevelsArray = metadataBlocksJson != null ? metadataBlocksJson.getJsonArray("inputLevels") : null; + + if (metadataBlocksJson != null && metadataBlocksJson.containsKey("inheritMetadataBlocksFromParent") && metadataBlocksJson.getBoolean("inheritMetadataBlocksFromParent")) { + return Lists.newArrayList(); // delete } - JsonArray inputLevelsArray = metadataBlocksJson.getJsonArray("inputLevels"); - return inputLevelsArray != null ? parseInputLevels(inputLevelsArray, dataverse) : null; + return parseInputLevels(inputLevelsArray, dataverse); } + /* + return null - ignore + return empty list - delete and inherit from parent + return non-empty list - update + */ private List parseMetadataBlocks(String body) throws WrappedResponse { JsonObject metadataBlocksJson = getMetadataBlocksJson(body); - if (metadataBlocksJson == null) { - return null; + JsonArray metadataBlocksArray = metadataBlocksJson != null ? metadataBlocksJson.getJsonArray("metadataBlockNames") : null; + + if (metadataBlocksArray != null && metadataBlocksJson.containsKey("inheritMetadataBlocksFromParent") && metadataBlocksJson.getBoolean("inheritMetadataBlocksFromParent")) { + String errorMessage = MessageFormat.format(BundleUtil.getStringFromBundle("dataverse.metadatablocks.error.containslistandinheritflag"), "metadataBlockNames", "inheritMetadataBlocksFromParent"); + throw new WrappedResponse(badRequest(errorMessage)); + } + if (metadataBlocksJson != null && metadataBlocksJson.containsKey("inheritMetadataBlocksFromParent") && metadataBlocksJson.getBoolean("inheritMetadataBlocksFromParent")) { + return Lists.newArrayList(); // delete and inherit from parent } - JsonArray metadataBlocksArray = metadataBlocksJson.getJsonArray("metadataBlockNames"); - return metadataBlocksArray != null ? parseNewDataverseMetadataBlocks(metadataBlocksArray) : null; + + return parseNewDataverseMetadataBlocks(metadataBlocksArray); } + /* + return null - ignore + return empty list - delete and inherit from parent + return non-empty list - update + */ private List parseFacets(String body) throws WrappedResponse { JsonObject metadataBlocksJson = getMetadataBlocksJson(body); - if (metadataBlocksJson == null) { - return null; + JsonArray facetsArray = metadataBlocksJson != null ? metadataBlocksJson.getJsonArray("facetIds") : null; + + if (facetsArray != null && metadataBlocksJson.containsKey("inheritFacetsFromParent") && metadataBlocksJson.getBoolean("inheritFacetsFromParent")) { + String errorMessage = MessageFormat.format(BundleUtil.getStringFromBundle("dataverse.metadatablocks.error.containslistandinheritflag"), "facetIds", "inheritFacetsFromParent"); + throw new WrappedResponse(badRequest(errorMessage)); + } + + if (metadataBlocksJson != null && metadataBlocksJson.containsKey("inheritFacetsFromParent") && metadataBlocksJson.getBoolean("inheritFacetsFromParent")) { + return Lists.newArrayList(); // delete and inherit from parent } - JsonArray facetsArray = metadataBlocksJson.getJsonArray("facetIds"); - return facetsArray != null ? parseFacets(facetsArray) : null; + + return parseFacets(facetsArray); } private JsonObject getMetadataBlocksJson(String body) { @@ -277,6 +316,9 @@ private Response handleEJBException(EJBException ex, String action) { } private List parseNewDataverseMetadataBlocks(JsonArray metadataBlockNamesArray) throws WrappedResponse { + if (metadataBlockNamesArray == null) { + return null; + } List selectedMetadataBlocks = new ArrayList<>(); for (JsonString metadataBlockName : metadataBlockNamesArray.getValuesAs(JsonString.class)) { MetadataBlock metadataBlock = metadataBlockSvc.findByName(metadataBlockName.getString()); @@ -670,12 +712,12 @@ private Dataset parseDataset(String datasetJson) throws WrappedResponse { @GET @AuthRequired @Path("{identifier}") - public Response getDataverse(@Context ContainerRequestContext crc, @PathParam("identifier") String idtf, @QueryParam("returnOwners") boolean returnOwners) { - return response(req -> ok( - json(execCommand(new GetDataverseCommand(req, findDataverseOrDie(idtf))), - settingsService.isTrueForKey(SettingsServiceBean.Key.ExcludeEmailFromExport, false), - returnOwners - )), getRequestUser(crc)); + public Response getDataverse(@Context ContainerRequestContext crc, @PathParam("identifier") String idtf, @QueryParam("returnOwners") boolean returnOwners, @QueryParam("returnChildCount") boolean returnChildCount) { + return response(req -> { + Dataverse dataverse = execCommand(new GetDataverseCommand(req, findDataverseOrDie(idtf))); + boolean hideEmail = settingsService.isTrueForKey(SettingsServiceBean.Key.ExcludeEmailFromExport, false); + return ok(json(dataverse, hideEmail, returnOwners, returnChildCount ? dataverseService.getChildCount(dataverse) : null)); + }, getRequestUser(crc)); } @DELETE @@ -711,7 +753,7 @@ public Response updateAttribute(@Context ContainerRequestContext crc, @PathParam } private Object formatAttributeValue(String attribute, String value) throws WrappedResponse { - if (attribute.equals("filePIDsEnabled")) { + if (List.of("filePIDsEnabled","requireFilesToPublishDataset").contains(attribute)) { return parseBooleanOrDie(value); } return value; @@ -745,6 +787,9 @@ public Response updateInputLevels(@Context ContainerRequestContext crc, @PathPar } private List parseInputLevels(JsonArray inputLevelsArray, Dataverse dataverse) throws WrappedResponse { + if (inputLevelsArray == null) { + return null; + } List newInputLevels = new ArrayList<>(); for (JsonValue value : inputLevelsArray) { JsonObject inputLevel = (JsonObject) value; @@ -758,19 +803,26 @@ private List parseInputLevels(JsonArray inputLevel boolean required = inputLevel.getBoolean("required"); boolean include = inputLevel.getBoolean("include"); + Boolean displayOnCreate = null; + if(inputLevel.containsKey("displayOnCreate")) { + displayOnCreate = inputLevel.getBoolean("displayOnCreate", false); + } if (required && !include) { String errorMessage = MessageFormat.format(BundleUtil.getStringFromBundle("dataverse.inputlevels.error.cannotberequiredifnotincluded"), datasetFieldTypeName); throw new WrappedResponse(badRequest(errorMessage)); } - newInputLevels.add(new DataverseFieldTypeInputLevel(datasetFieldType, dataverse, required, include)); + newInputLevels.add(new DataverseFieldTypeInputLevel(datasetFieldType, dataverse, required, include, displayOnCreate)); } return newInputLevels; } private List parseFacets(JsonArray facetsArray) throws WrappedResponse { + if (facetsArray == null) { + return null; + } List facets = new LinkedList<>(); for (JsonString facetId : facetsArray.getValuesAs(JsonString.class)) { DatasetFieldType dsfType = findDatasetFieldType(facetId.getString()); @@ -801,17 +853,20 @@ public Response deleteDataverseLinkingDataverse(@Context ContainerRequestContext public Response listMetadataBlocks(@Context ContainerRequestContext crc, @PathParam("identifier") String dvIdtf, @QueryParam("onlyDisplayedOnCreate") boolean onlyDisplayedOnCreate, - @QueryParam("returnDatasetFieldTypes") boolean returnDatasetFieldTypes) { + @QueryParam("returnDatasetFieldTypes") boolean returnDatasetFieldTypes, + @QueryParam("datasetType") String datasetTypeIn) { try { Dataverse dataverse = findDataverseOrDie(dvIdtf); + DatasetType datasetType = datasetTypeSvc.getByName(datasetTypeIn); final List metadataBlocks = execCommand( new ListMetadataBlocksCommand( createDataverseRequest(getRequestUser(crc)), dataverse, - onlyDisplayedOnCreate + onlyDisplayedOnCreate, + datasetType ) ); - return ok(json(metadataBlocks, returnDatasetFieldTypes, onlyDisplayedOnCreate, dataverse)); + return ok(json(metadataBlocks, returnDatasetFieldTypes, onlyDisplayedOnCreate, dataverse, datasetType)); } catch (WrappedResponse we) { return we.getResponse(); } @@ -1729,4 +1784,131 @@ public Response getUserPermissionsOnDataverse(@Context ContainerRequestContext c jsonObjectBuilder.add("canDeleteDataverse", permissionService.userOn(requestUser, dataverse).has(Permission.DeleteDataverse)); return ok(jsonObjectBuilder); } + + @POST + @AuthRequired + @Consumes(MediaType.MULTIPART_FORM_DATA) + @Path("{identifier}/featuredItems") + public Response createFeaturedItem(@Context ContainerRequestContext crc, + @PathParam("identifier") String dvIdtf, + @FormDataParam("content") String content, + @FormDataParam("displayOrder") int displayOrder, + @FormDataParam("file") InputStream imageFileInputStream, + @FormDataParam("file") FormDataContentDisposition contentDispositionHeader) { + Dataverse dataverse; + try { + dataverse = findDataverseOrDie(dvIdtf); + } catch (WrappedResponse wr) { + return wr.getResponse(); + } + NewDataverseFeaturedItemDTO newDataverseFeaturedItemDTO = NewDataverseFeaturedItemDTO.fromFormData(content, displayOrder, imageFileInputStream, contentDispositionHeader); + try { + DataverseFeaturedItem dataverseFeaturedItem = execCommand(new CreateDataverseFeaturedItemCommand( + createDataverseRequest(getRequestUser(crc)), + dataverse, + newDataverseFeaturedItemDTO + )); + return ok(json(dataverseFeaturedItem)); + } catch (WrappedResponse e) { + return e.getResponse(); + } + } + + @GET + @AuthRequired + @Path("{identifier}/featuredItems") + public Response listFeaturedItems(@Context ContainerRequestContext crc, @PathParam("identifier") String dvIdtf) { + try { + Dataverse dataverse = findDataverseOrDie(dvIdtf); + List featuredItems = execCommand(new ListDataverseFeaturedItemsCommand(createDataverseRequest(getRequestUser(crc)), dataverse)); + return ok(jsonDataverseFeaturedItems(featuredItems)); + } catch (WrappedResponse e) { + return e.getResponse(); + } + } + + @PUT + @AuthRequired + @Consumes(MediaType.MULTIPART_FORM_DATA) + @Path("{dataverseId}/featuredItems") + public Response updateFeaturedItems( + @Context ContainerRequestContext crc, + @PathParam("dataverseId") String dvIdtf, + @FormDataParam("id") List ids, + @FormDataParam("content") List contents, + @FormDataParam("displayOrder") List displayOrders, + @FormDataParam("keepFile") List keepFiles, + @FormDataParam("fileName") List fileNames, + @FormDataParam("file") List files) { + try { + if (ids == null || contents == null || displayOrders == null || keepFiles == null || fileNames == null) { + throw new WrappedResponse(error(Response.Status.BAD_REQUEST, + BundleUtil.getStringFromBundle("dataverse.update.featuredItems.error.missingInputParams"))); + } + + int size = ids.size(); + if (contents.size() != size || displayOrders.size() != size || keepFiles.size() != size || fileNames.size() != size) { + throw new WrappedResponse(error(Response.Status.BAD_REQUEST, + BundleUtil.getStringFromBundle("dataverse.update.featuredItems.error.inputListsSizeMismatch"))); + } + + Dataverse dataverse = findDataverseOrDie(dvIdtf); + List newItems = new ArrayList<>(); + Map itemsToUpdate = new HashMap<>(); + + for (int i = 0; i < contents.size(); i++) { + String fileName = fileNames.get(i); + InputStream fileInputStream = null; + FormDataContentDisposition contentDisposition = null; + + if (files != null) { + Optional matchingFile = files.stream() + .filter(file -> file.getFormDataContentDisposition().getFileName().equals(fileName)) + .findFirst(); + + if (matchingFile.isPresent()) { + fileInputStream = matchingFile.get().getValueAs(InputStream.class); + contentDisposition = matchingFile.get().getFormDataContentDisposition(); + } + } + + if (ids.get(i) == 0) { + newItems.add(NewDataverseFeaturedItemDTO.fromFormData( + contents.get(i), displayOrders.get(i), fileInputStream, contentDisposition)); + } else { + DataverseFeaturedItem existingItem = dataverseFeaturedItemServiceBean.findById(ids.get(i)); + if (existingItem == null) { + throw new WrappedResponse(error(Response.Status.NOT_FOUND, + MessageFormat.format(BundleUtil.getStringFromBundle("dataverseFeaturedItems.errors.notFound"), ids.get(i)))); + } + itemsToUpdate.put(existingItem, UpdatedDataverseFeaturedItemDTO.fromFormData( + contents.get(i), displayOrders.get(i), keepFiles.get(i), fileInputStream, contentDisposition)); + } + } + + List featuredItems = execCommand(new UpdateDataverseFeaturedItemsCommand( + createDataverseRequest(getRequestUser(crc)), + dataverse, + newItems, + itemsToUpdate + )); + + return ok(jsonDataverseFeaturedItems(featuredItems)); + } catch (WrappedResponse wr) { + return wr.getResponse(); + } + } + + @DELETE + @AuthRequired + @Path("{identifier}/featuredItems") + public Response deleteFeaturedItems(@Context ContainerRequestContext crc, @PathParam("identifier") String dvIdtf) { + try { + Dataverse dataverse = findDataverseOrDie(dvIdtf); + execCommand(new UpdateDataverseFeaturedItemsCommand(createDataverseRequest(getRequestUser(crc)), dataverse, new ArrayList<>(), new ArrayMap<>())); + return ok(BundleUtil.getStringFromBundle("dataverse.delete.featuredItems.success")); + } catch (WrappedResponse e) { + return e.getResponse(); + } + } } diff --git a/src/main/java/edu/harvard/iq/dataverse/api/ExternalToolsApi.java b/src/main/java/edu/harvard/iq/dataverse/api/ExternalToolsApi.java new file mode 100644 index 00000000000..92139d86caf --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/api/ExternalToolsApi.java @@ -0,0 +1,58 @@ +package edu.harvard.iq.dataverse.api; + +import edu.harvard.iq.dataverse.api.auth.AuthRequired; +import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser; +import jakarta.inject.Inject; +import jakarta.ws.rs.DELETE; +import jakarta.ws.rs.GET; +import jakarta.ws.rs.POST; +import jakarta.ws.rs.Path; +import jakarta.ws.rs.PathParam; +import jakarta.ws.rs.container.ContainerRequestContext; +import jakarta.ws.rs.core.Context; +import jakarta.ws.rs.core.Response; + +@Path("externalTools") +public class ExternalToolsApi extends AbstractApiBean { + + @Inject + ExternalTools externalTools; + + @GET + public Response getExternalTools() { + return externalTools.getExternalTools(); + } + + @GET + @Path("{id}") + public Response getExternalTool(@PathParam("id") long externalToolIdFromUser) { + return externalTools.getExternalTool(externalToolIdFromUser); + } + + @POST + @AuthRequired + public Response addExternalTool(@Context ContainerRequestContext crc, String manifest) { + Response notAuthorized = authorize(crc); + return notAuthorized == null ? externalTools.addExternalTool(manifest) : notAuthorized; + } + + @DELETE + @AuthRequired + @Path("{id}") + public Response deleteExternalTool(@Context ContainerRequestContext crc, @PathParam("id") long externalToolIdFromUser) { + Response notAuthorized = authorize(crc); + return notAuthorized == null ? externalTools.deleteExternalTool(externalToolIdFromUser) : notAuthorized; + } + + private Response authorize(ContainerRequestContext crc) { + try { + AuthenticatedUser user = getRequestAuthenticatedUserOrDie(crc); + if (!user.isSuperuser()) { + return error(Response.Status.FORBIDDEN, "Superusers only."); + } + } catch (WrappedResponse ex) { + return error(Response.Status.FORBIDDEN, "Superusers only."); + } + return null; + } +} diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Files.java b/src/main/java/edu/harvard/iq/dataverse/api/Files.java index 633d420c527..89e4e6d7f97 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Files.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Files.java @@ -46,6 +46,7 @@ import java.util.List; import java.util.logging.Level; import java.util.logging.Logger; + import jakarta.ejb.EJB; import jakarta.ejb.EJBException; import jakarta.inject.Inject; @@ -67,7 +68,6 @@ import org.eclipse.microprofile.openapi.annotations.Operation; import org.eclipse.microprofile.openapi.annotations.media.Content; -import org.eclipse.microprofile.openapi.annotations.media.Schema; import org.eclipse.microprofile.openapi.annotations.parameters.RequestBody; import org.eclipse.microprofile.openapi.annotations.responses.APIResponse; import org.eclipse.microprofile.openapi.annotations.tags.Tag; @@ -102,6 +102,8 @@ public class Files extends AbstractApiBean { GuestbookResponseServiceBean guestbookResponseService; @Inject DataFileServiceBean dataFileServiceBean; + @Inject + FileMetadataVersionsHelper fileMetadataVersionsHelper; private static final Logger logger = Logger.getLogger(Files.class.getName()); @@ -464,12 +466,16 @@ public Response updateFileMetadata(@Context ContainerRequestContext crc, @FormDa String pathPlusFilename = IngestUtil.getPathAndFileNameToCheck(incomingLabel, incomingDirectoryLabel, existingLabel, existingDirectoryLabel); // We remove the current file from the list we'll check for duplicates. // Instead, the current file is passed in as pathPlusFilename. + // the original test fails for published datasets/new draft because the filemetadata + // lacks an id for the "equals" test. Changing test to datafile for #11208 List fmdListMinusCurrentFile = new ArrayList<>(); + for (FileMetadata fileMetadata : fmdList) { - if (!fileMetadata.equals(df.getFileMetadata())) { + if (!fileMetadata.getDataFile().equals(df)) { fmdListMinusCurrentFile.add(fileMetadata); } } + if (IngestUtil.conflictsWithExistingFilenames(pathPlusFilename, fmdListMinusCurrentFile)) { return error(BAD_REQUEST, BundleUtil.getStringFromBundle("files.api.metadata.update.duplicateFile", Arrays.asList(pathPlusFilename))); } @@ -980,4 +986,30 @@ public Response getFileCitationByVersion(@Context ContainerRequestContext crc, @ } } + @GET + @AuthRequired + @Path("{id}/versionDifferences") + @Produces(MediaType.APPLICATION_JSON) + public Response getFileVersionsList(@Context ContainerRequestContext crc, @PathParam("id") String fileIdOrPersistentId) { + try { + DataverseRequest req = createDataverseRequest(getRequestUser(crc)); + final DataFile df = execCommand(new GetDataFileCommand(req, findDataFileOrDie(fileIdOrPersistentId))); + FileMetadata fm = df.getFileMetadata(); + if (fm == null) { + return notFound(BundleUtil.getStringFromBundle("files.api.fileNotFound")); + } + List fileMetadataList = fileMetadataVersionsHelper.loadFileVersionList(req, fm); + JsonArrayBuilder jab = Json.createArrayBuilder(); + for (FileMetadata fileMetadata : fileMetadataList) { + jab.add(fileMetadataVersionsHelper.jsonDataFileVersions(fileMetadata).build()); + } + return Response.ok() + .entity(Json.createObjectBuilder() + .add("status", STATUS_OK) + .add("data", jab.build()).build() + ).build(); + } catch (WrappedResponse ex) { + return ex.getResponse(); + } + } } diff --git a/src/main/java/edu/harvard/iq/dataverse/api/HarvestingClients.java b/src/main/java/edu/harvard/iq/dataverse/api/HarvestingClients.java index dfc9f48dd1a..e4300099244 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/HarvestingClients.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/HarvestingClients.java @@ -278,7 +278,10 @@ public Response modifyHarvestingClient(@Context ContainerRequestContext crc, Str // Go through the supported editable fields and update the client accordingly: // TODO: We may want to reevaluate whether we really want/need *all* // of these fields to be editable. - + + if (newHarvestingClient.getSourceName() != null) { + harvestingClient.setSourceName(newHarvestingClient.getSourceName()); + } if (newHarvestingClient.getHarvestingUrl() != null) { harvestingClient.setHarvestingUrl(newHarvestingClient.getHarvestingUrl()); } diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Index.java b/src/main/java/edu/harvard/iq/dataverse/api/Index.java index c30a77acb58..bc9a8ae692b 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Index.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Index.java @@ -44,6 +44,7 @@ import java.lang.reflect.Field; import java.util.ArrayList; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.concurrent.ExecutionException; import java.util.concurrent.Future; @@ -451,11 +452,11 @@ public Response clearOrphans(@QueryParam("sync") String sync) { public String getSolrSchema() { StringBuilder sb = new StringBuilder(); - - for (DatasetFieldType datasetField : datasetFieldService.findAllOrderedByName()) { + Map cvocTermUriMap = datasetFieldSvc.getCVocConf(true); + for (DatasetFieldType datasetFieldType : datasetFieldService.findAllOrderedByName()) { //ToDo - getSolrField() creates/returns a new object - just get it once and re-use - String nameSearchable = datasetField.getSolrField().getNameSearchable(); - SolrField.SolrType solrType = datasetField.getSolrField().getSolrType(); + String nameSearchable = datasetFieldType.getSolrField().getNameSearchable(); + SolrField.SolrType solrType = datasetFieldType.getSolrField().getSolrType(); String type = solrType.getType(); if (solrType.equals(SolrField.SolrType.EMAIL)) { /** @@ -474,7 +475,7 @@ public String getSolrSchema() { */ logger.info("email type detected (" + nameSearchable + ") See also https://github.com/IQSS/dataverse/issues/759"); } - String multivalued = datasetField.getSolrField().isAllowedToBeMultivalued().toString(); + String multivalued = Boolean.toString(datasetFieldType.getSolrField().isAllowedToBeMultivalued() || cvocTermUriMap.containsKey(datasetFieldType.getId())); // sb.append(" \n"); } diff --git a/src/main/java/edu/harvard/iq/dataverse/api/MakeDataCountApi.java b/src/main/java/edu/harvard/iq/dataverse/api/MakeDataCountApi.java index 306b863c9e4..ca8f59a71be 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/MakeDataCountApi.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/MakeDataCountApi.java @@ -235,6 +235,9 @@ public Response getProcessingState(@PathParam("yearMonth") String yearMonth) { output.add("yearMonth", mdcps.getYearMonth()); output.add("state", mdcps.getState().name()); output.add("stateChangeTimestamp", mdcps.getStateChangeTime().toString()); + if (mdcps.getServer() != null && !mdcps.getServer().isBlank()) { + output.add("server", mdcps.getServer()); + } return ok(output); } else { return error(Status.NOT_FOUND, "Could not find an existing process state for " + yearMonth); @@ -243,10 +246,10 @@ public Response getProcessingState(@PathParam("yearMonth") String yearMonth) { @POST @Path("{yearMonth}/processingState") - public Response updateProcessingState(@PathParam("yearMonth") String yearMonth, @QueryParam("state") String state) { + public Response updateProcessingState(@PathParam("yearMonth") String yearMonth, @QueryParam("state") String state, @QueryParam("server") String server) { MakeDataCountProcessState mdcps; try { - mdcps = makeDataCountProcessStateService.setMakeDataCountProcessState(yearMonth, state); + mdcps = makeDataCountProcessStateService.setMakeDataCountProcessState(yearMonth, state, server); } catch (Exception e) { return badRequest(e.getMessage()); } @@ -255,6 +258,9 @@ public Response updateProcessingState(@PathParam("yearMonth") String yearMonth, output.add("yearMonth", mdcps.getYearMonth()); output.add("state", mdcps.getState().name()); output.add("stateChangeTimestamp", mdcps.getStateChangeTime().toString()); + if ( mdcps.getServer() != null) { + output.add("server", mdcps.getServer()); + } return ok(output); } diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Pids.java b/src/main/java/edu/harvard/iq/dataverse/api/Pids.java index 4ad57bceb58..32d942f972c 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Pids.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Pids.java @@ -10,6 +10,8 @@ import edu.harvard.iq.dataverse.settings.JvmSettings; import edu.harvard.iq.dataverse.util.BundleUtil; import java.util.Arrays; +import java.util.logging.Logger; + import jakarta.ejb.Stateless; import jakarta.json.Json; import jakarta.json.JsonArray; @@ -40,6 +42,7 @@ @Path("pids") public class Pids extends AbstractApiBean { + private static final Logger logger = Logger.getLogger(Pids.class.getName()); @GET @AuthRequired @Produces(MediaType.APPLICATION_JSON) @@ -143,6 +146,7 @@ public Response getPidProviders(@Context ContainerRequestContext crc) throws Wra return ok(PidUtil.getProviders()); } + @GET @AuthRequired // The :.+ suffix allows PIDs with a / char to be entered w/o escaping @@ -166,5 +170,4 @@ public Response getPidProviderId(@Context ContainerRequestContext crc, @PathPara } } } - } diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Search.java b/src/main/java/edu/harvard/iq/dataverse/api/Search.java index f86f9f446fa..bfae753d591 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Search.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Search.java @@ -73,6 +73,7 @@ public Response search( @QueryParam("metadata_fields") List metadataFields, @QueryParam("geo_point") String geoPointRequested, @QueryParam("geo_radius") String geoRadiusRequested, + @QueryParam("show_type_counts") boolean showTypeCounts, @Context HttpServletResponse response ) { @@ -210,6 +211,52 @@ public Response search( } value.add("count_in_response", solrSearchResults.size()); + + // we want to show the missing dvobject types with count = 0 + // per https://github.com/IQSS/dataverse/issues/11127 + + if (showTypeCounts) { + JsonObjectBuilder objectTypeCounts = Json.createObjectBuilder(); + if (!solrQueryResponse.getTypeFacetCategories().isEmpty()) { + boolean filesMissing = true; + boolean datasetsMissing = true; + boolean dataversesMissing = true; + for (FacetCategory facetCategory : solrQueryResponse.getTypeFacetCategories()) { + for (FacetLabel facetLabel : facetCategory.getFacetLabel()) { + objectTypeCounts.add(facetLabel.getName(), facetLabel.getCount()); + if (facetLabel.getName().equals((SearchConstants.UI_DATAVERSES))) { + dataversesMissing = false; + } + if (facetLabel.getName().equals((SearchConstants.UI_DATASETS))) { + datasetsMissing = false; + } + if (facetLabel.getName().equals((SearchConstants.UI_FILES))) { + filesMissing = false; + } + } + } + + if (solrQueryResponse.getTypeFacetCategories().size() < 3) { + if (dataversesMissing) { + objectTypeCounts.add(SearchConstants.UI_DATAVERSES, 0); + } + if (datasetsMissing) { + objectTypeCounts.add(SearchConstants.UI_DATASETS, 0); + } + if (filesMissing) { + objectTypeCounts.add(SearchConstants.UI_FILES, 0); + } + } + + } + if (showTypeCounts && solrQueryResponse.getTypeFacetCategories().isEmpty()) { + objectTypeCounts.add(SearchConstants.UI_DATAVERSES, 0); + objectTypeCounts.add(SearchConstants.UI_DATASETS, 0); + objectTypeCounts.add(SearchConstants.UI_FILES, 0); + } + + value.add("total_count_per_object_type", objectTypeCounts); + } /** * @todo Returning the fq might be useful as a troubleshooting aid * but we don't want to expose the raw dataverse database ids in diff --git a/src/main/java/edu/harvard/iq/dataverse/api/SendFeedbackAPI.java b/src/main/java/edu/harvard/iq/dataverse/api/SendFeedbackAPI.java new file mode 100644 index 00000000000..3bffcd042a3 --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/api/SendFeedbackAPI.java @@ -0,0 +1,126 @@ +package edu.harvard.iq.dataverse.api; + +import edu.harvard.iq.dataverse.*; +import edu.harvard.iq.dataverse.api.auth.AuthRequired; +import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser; +import edu.harvard.iq.dataverse.authorization.users.User; +import edu.harvard.iq.dataverse.branding.BrandingUtil; +import edu.harvard.iq.dataverse.engine.command.impl.CheckRateLimitForDatasetFeedbackCommand; +import edu.harvard.iq.dataverse.feedback.Feedback; +import edu.harvard.iq.dataverse.feedback.FeedbackUtil; +import edu.harvard.iq.dataverse.util.BundleUtil; +import edu.harvard.iq.dataverse.util.cache.CacheFactoryBean; +import edu.harvard.iq.dataverse.util.json.JsonUtil; +import edu.harvard.iq.dataverse.validation.EMailValidator; +import jakarta.ejb.EJB; +import jakarta.json.*; +import jakarta.mail.internet.InternetAddress; +import jakarta.ws.rs.Consumes; +import jakarta.ws.rs.POST; +import jakarta.ws.rs.Path; +import jakarta.ws.rs.container.ContainerRequestContext; +import jakarta.ws.rs.core.Context; +import jakarta.ws.rs.core.Response; + +import java.text.MessageFormat; +import java.util.logging.Logger; + +@Path("sendfeedback") +public class SendFeedbackAPI extends AbstractApiBean { + private static final Logger logger = Logger.getLogger(SendFeedbackAPI.class.getCanonicalName()); + @EJB + MailServiceBean mailService; + @EJB + CacheFactoryBean cacheFactory; + /** + * This method mimics the contact form and sends an email to the contacts of the + * specified Collection/Dataset/DataFile, optionally ccing the support email + * address, or to the support email address when there is no target object. + **/ + @POST + @AuthRequired + public Response submitFeedback(@Context ContainerRequestContext crc, String jsonString) { + try { + JsonObject jsonObject = JsonUtil.getJsonObject(jsonString); + if (!jsonObject.containsKey("subject") || !jsonObject.containsKey("body")) { + return badRequest(BundleUtil.getStringFromBundle("sendfeedback.body.error.missingRequiredFields")); + } + + JsonNumber jsonNumber = jsonObject.containsKey("targetId") ? jsonObject.getJsonNumber("targetId") : null; + // idtf will hold the "targetId" or the "identifier". If neither is set then this is a general feedback to support + String idtf = jsonNumber != null ? jsonNumber.toString() : jsonObject.containsKey("identifier") ? jsonObject.getString("identifier") : null; + DvObject feedbackTarget = null; + + if (jsonNumber != null) { + feedbackTarget = dvObjSvc.findDvObject(jsonNumber.longValue()); + } else if (idtf != null) { + if (feedbackTarget == null) { + feedbackTarget = dataverseSvc.findByAlias(idtf); + } + if (feedbackTarget == null) { + feedbackTarget = dvObjSvc.findByGlobalId(idtf, DvObject.DType.Dataset); + } + if (feedbackTarget == null) { + feedbackTarget = dvObjSvc.findByGlobalId(idtf, DvObject.DType.DataFile); + } + } + + // feedbackTarget and idtf are both null this is a support feedback and is ok + if (feedbackTarget == null && idtf != null) { + return error(Response.Status.BAD_REQUEST, BundleUtil.getStringFromBundle("sendfeedback.request.error.targetNotFound")); + } + // Check for rate limit exceeded. + if (!cacheFactory.checkRate(getRequestUser(crc), new CheckRateLimitForDatasetFeedbackCommand(null, feedbackTarget))) { + return error(Response.Status.TOO_MANY_REQUESTS, BundleUtil.getStringFromBundle("sendfeedback.request.rateLimited")); + } + + DataverseSession dataverseSession = null; + String userMessage = sanitizeBody(jsonObject.getString("body")); + InternetAddress systemAddress = mailService.getSupportAddress().orElse(null); + String userEmail = getEmail(jsonObject, crc); + String messageSubject = jsonObject.getString("subject"); + String baseUrl = systemConfig.getDataverseSiteUrl(); + String installationBrandName = BrandingUtil.getInstallationBrandName(); + String supportTeamName = BrandingUtil.getSupportTeamName(systemAddress); + JsonArrayBuilder jab = Json.createArrayBuilder(); + Feedback feedback = FeedbackUtil.gatherFeedback(feedbackTarget, dataverseSession, messageSubject, userMessage, systemAddress, userEmail, baseUrl, installationBrandName, supportTeamName, SendFeedbackDialog.ccSupport(feedbackTarget)); + jab.add(feedback.toLimitedJsonObjectBuilder()); + mailService.sendMail(feedback.getFromEmail(), feedback.getToEmail(), feedback.getCcEmail(), feedback.getSubject(), feedback.getBody()); + return ok(jab); + } catch (WrappedResponse resp) { + return resp.getResponse(); + } catch (JsonException je) { + return error(Response.Status.BAD_REQUEST, "Invalid JSON; error message: " + je.getMessage()); + } + } + + private String getEmail(JsonObject jsonObject, ContainerRequestContext crc) throws WrappedResponse { + String fromEmail = jsonObject.containsKey("fromEmail") ? jsonObject.getString("fromEmail") : ""; + if (fromEmail.isBlank() && crc != null) { + User user = getRequestUser(crc); + if (user instanceof AuthenticatedUser) { + fromEmail = ((AuthenticatedUser) user).getEmail(); + } + } + if (fromEmail == null || fromEmail.isBlank()) { + throw new WrappedResponse(badRequest(BundleUtil.getStringFromBundle("sendfeedback.fromEmail.error.missing"))); + } + if (!EMailValidator.isEmailValid(fromEmail)) { + throw new WrappedResponse(badRequest(MessageFormat.format(BundleUtil.getStringFromBundle("sendfeedback.fromEmail.error.invalid"), fromEmail))); + } + return fromEmail; + } + private String sanitizeBody (String body) throws WrappedResponse { + // remove malicious html + String sanitizedBody = body == null ? "" : body.replaceAll("\\<.*?>", ""); + + long limit = systemConfig.getContactFeedbackMessageSizeLimit(); + if (limit > 0 && sanitizedBody.length() > limit) { + throw new WrappedResponse(badRequest(MessageFormat.format(BundleUtil.getStringFromBundle("sendfeedback.body.error.exceedsLength"), sanitizedBody.length(), limit))); + } else if (sanitizedBody.length() == 0) { + throw new WrappedResponse(badRequest(BundleUtil.getStringFromBundle("sendfeedback.body.error.isEmpty"))); + } + + return sanitizedBody; + } +} diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Users.java b/src/main/java/edu/harvard/iq/dataverse/api/Users.java index ecf7839e616..77e08bf6ceb 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Users.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Users.java @@ -8,29 +8,33 @@ import edu.harvard.iq.dataverse.api.auth.AuthRequired; import edu.harvard.iq.dataverse.authorization.users.ApiToken; import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser; +import edu.harvard.iq.dataverse.authorization.users.GuestUser; import edu.harvard.iq.dataverse.authorization.users.User; -import edu.harvard.iq.dataverse.engine.command.impl.ChangeUserIdentifierCommand; -import edu.harvard.iq.dataverse.engine.command.impl.GetUserTracesCommand; -import edu.harvard.iq.dataverse.engine.command.impl.MergeInAccountCommand; -import edu.harvard.iq.dataverse.engine.command.impl.RevokeAllRolesCommand; +import edu.harvard.iq.dataverse.engine.command.impl.*; +import edu.harvard.iq.dataverse.settings.FeatureFlags; +import edu.harvard.iq.dataverse.util.BundleUtil; import edu.harvard.iq.dataverse.util.FileUtil; +import static edu.harvard.iq.dataverse.api.auth.AuthUtil.extractBearerTokenFromHeaderParam; import static edu.harvard.iq.dataverse.util.json.JsonPrinter.json; +import java.text.MessageFormat; import java.util.Arrays; import java.util.List; +import java.util.Optional; import java.util.logging.Level; import java.util.logging.Logger; + +import edu.harvard.iq.dataverse.util.json.JsonParseException; +import edu.harvard.iq.dataverse.util.json.JsonUtil; import jakarta.ejb.Stateless; import jakarta.json.JsonArray; +import jakarta.json.JsonObject; import jakarta.json.JsonObjectBuilder; +import jakarta.json.stream.JsonParsingException; import jakarta.ws.rs.*; import jakarta.ws.rs.container.ContainerRequestContext; -import jakarta.ws.rs.core.Context; -import jakarta.ws.rs.core.MediaType; -import jakarta.ws.rs.core.Request; -import jakarta.ws.rs.core.Response; -import jakarta.ws.rs.core.Variant; +import jakarta.ws.rs.core.*; /** * @@ -266,4 +270,47 @@ public Response getTracesElement(@Context ContainerRequestContext crc, @Context } } + @GET + @AuthRequired + @Path("{identifier}/allowedCollections/{permission}") + @Produces("application/json") + public Response getUserPermittedCollections(@Context ContainerRequestContext crc, @Context Request req, @PathParam("identifier") String identifier, @PathParam("permission") String permission) { + AuthenticatedUser authenticatedUser = null; + try { + authenticatedUser = getRequestAuthenticatedUserOrDie(crc); + if (!authenticatedUser.getUserIdentifier().equalsIgnoreCase(identifier) && !authenticatedUser.isSuperuser()) { + return error(Response.Status.FORBIDDEN, "This API call can be used by Users getting there own permitted collections or by superusers."); + } + } catch (WrappedResponse ex) { + return error(Response.Status.UNAUTHORIZED, "Authentication is required."); + } + try { + AuthenticatedUser userToQuery = authSvc.getAuthenticatedUser(identifier); + JsonObjectBuilder jsonObj = execCommand(new GetUserPermittedCollectionsCommand(createDataverseRequest(getRequestUser(crc)), userToQuery, permission)); + return ok(jsonObj); + } catch (WrappedResponse ex) { + return ex.getResponse(); + } + } + + @POST + @Path("register") + public Response registerOIDCUser(String body) { + if (!FeatureFlags.API_BEARER_AUTH.enabled()) { + return error(Response.Status.INTERNAL_SERVER_ERROR, BundleUtil.getStringFromBundle("users.api.errors.bearerAuthFeatureFlagDisabled")); + } + Optional bearerToken = extractBearerTokenFromHeaderParam(httpRequest.getHeader(HttpHeaders.AUTHORIZATION)); + if (bearerToken.isEmpty()) { + return error(Response.Status.BAD_REQUEST, BundleUtil.getStringFromBundle("users.api.errors.bearerTokenRequired")); + } + try { + JsonObject userJson = JsonUtil.getJsonObject(body); + execCommand(new RegisterOIDCUserCommand(createDataverseRequest(GuestUser.get()), bearerToken.get(), jsonParser().parseUserDTO(userJson))); + } catch (JsonParseException | JsonParsingException e) { + return error(Response.Status.BAD_REQUEST, MessageFormat.format(BundleUtil.getStringFromBundle("users.api.errors.jsonParseToUserDTO"), e.getMessage())); + } catch (WrappedResponse e) { + return e.getResponse(); + } + return ok(BundleUtil.getStringFromBundle("users.api.userRegistered")); + } } diff --git a/src/main/java/edu/harvard/iq/dataverse/api/auth/ApiKeyAuthMechanism.java b/src/main/java/edu/harvard/iq/dataverse/api/auth/ApiKeyAuthMechanism.java index 0dd8a28baca..fbb0b484b58 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/auth/ApiKeyAuthMechanism.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/auth/ApiKeyAuthMechanism.java @@ -9,6 +9,7 @@ import jakarta.inject.Inject; import jakarta.ws.rs.container.ContainerRequestContext; + import java.util.logging.Logger; /** @@ -49,7 +50,7 @@ public User findUserFromRequest(ContainerRequestContext containerRequestContext) authUser = userSvc.updateLastApiUseTime(authUser); return authUser; } - throw new WrappedAuthErrorResponse(RESPONSE_MESSAGE_BAD_API_KEY); + throw new WrappedUnauthorizedAuthErrorResponse(RESPONSE_MESSAGE_BAD_API_KEY); } private String getRequestApiKey(ContainerRequestContext containerRequestContext) { @@ -59,7 +60,7 @@ private String getRequestApiKey(ContainerRequestContext containerRequestContext) return headerParamApiKey != null ? headerParamApiKey : queryParamApiKey; } - private void checkAnonymizedAccessToRequestPath(String requestPath, PrivateUrlUser privateUrlUser) throws WrappedAuthErrorResponse { + private void checkAnonymizedAccessToRequestPath(String requestPath, PrivateUrlUser privateUrlUser) throws WrappedUnauthorizedAuthErrorResponse { if (!privateUrlUser.hasAnonymizedAccess()) { return; } @@ -67,7 +68,7 @@ private void checkAnonymizedAccessToRequestPath(String requestPath, PrivateUrlUs // to download the file or image thumbs if (!(requestPath.startsWith(ACCESS_DATAFILE_PATH_PREFIX) && !requestPath.substring(ACCESS_DATAFILE_PATH_PREFIX.length()).contains("/"))) { logger.info("Anonymized access request for " + requestPath); - throw new WrappedAuthErrorResponse(RESPONSE_MESSAGE_BAD_API_KEY); + throw new WrappedUnauthorizedAuthErrorResponse(RESPONSE_MESSAGE_BAD_API_KEY); } } } diff --git a/src/main/java/edu/harvard/iq/dataverse/api/auth/AuthUtil.java b/src/main/java/edu/harvard/iq/dataverse/api/auth/AuthUtil.java new file mode 100644 index 00000000000..36cd7c7f1df --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/api/auth/AuthUtil.java @@ -0,0 +1,24 @@ +package edu.harvard.iq.dataverse.api.auth; + +import java.util.Optional; + +public class AuthUtil { + + private static final String BEARER_AUTH_SCHEME = "Bearer"; + + /** + * Extracts the Bearer token from the provided HTTP Authorization header value. + *

+ * Validates that the header value starts with the "Bearer" scheme as defined in RFC 6750. + * If the header is null, empty, or does not start with "Bearer ", an empty {@link Optional} is returned. + * + * @param headerParamBearerToken the raw HTTP Authorization header value containing the Bearer token + * @return An {@link Optional} containing the raw Bearer token if present and valid; otherwise, an empty {@link Optional} + */ + public static Optional extractBearerTokenFromHeaderParam(String headerParamBearerToken) { + if (headerParamBearerToken != null && headerParamBearerToken.toLowerCase().startsWith(BEARER_AUTH_SCHEME.toLowerCase() + " ")) { + return Optional.of(headerParamBearerToken); + } + return Optional.empty(); + } +} diff --git a/src/main/java/edu/harvard/iq/dataverse/api/auth/BearerTokenAuthMechanism.java b/src/main/java/edu/harvard/iq/dataverse/api/auth/BearerTokenAuthMechanism.java index 31f524af3f0..3ee9bb909f2 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/auth/BearerTokenAuthMechanism.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/auth/BearerTokenAuthMechanism.java @@ -1,124 +1,65 @@ package edu.harvard.iq.dataverse.api.auth; -import com.nimbusds.oauth2.sdk.ParseException; -import com.nimbusds.oauth2.sdk.token.BearerAccessToken; import edu.harvard.iq.dataverse.UserServiceBean; import edu.harvard.iq.dataverse.authorization.AuthenticationServiceBean; -import edu.harvard.iq.dataverse.authorization.UserRecordIdentifier; -import edu.harvard.iq.dataverse.authorization.providers.oauth2.oidc.OIDCAuthProvider; +import edu.harvard.iq.dataverse.authorization.exceptions.AuthorizationException; import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser; import edu.harvard.iq.dataverse.authorization.users.User; import edu.harvard.iq.dataverse.settings.FeatureFlags; +import edu.harvard.iq.dataverse.util.BundleUtil; import jakarta.inject.Inject; import jakarta.ws.rs.container.ContainerRequestContext; import jakarta.ws.rs.core.HttpHeaders; -import java.io.IOException; -import java.util.List; + import java.util.Optional; import java.util.logging.Level; import java.util.logging.Logger; -import java.util.stream.Collectors; + +import static edu.harvard.iq.dataverse.api.auth.AuthUtil.extractBearerTokenFromHeaderParam; public class BearerTokenAuthMechanism implements AuthMechanism { - private static final String BEARER_AUTH_SCHEME = "Bearer"; private static final Logger logger = Logger.getLogger(BearerTokenAuthMechanism.class.getCanonicalName()); - - public static final String UNAUTHORIZED_BEARER_TOKEN = "Unauthorized bearer token"; - public static final String INVALID_BEARER_TOKEN = "Could not parse bearer token"; - public static final String BEARER_TOKEN_DETECTED_NO_OIDC_PROVIDER_CONFIGURED = "Bearer token detected, no OIDC provider configured"; @Inject protected AuthenticationServiceBean authSvc; @Inject protected UserServiceBean userSvc; - + @Override public User findUserFromRequest(ContainerRequestContext containerRequestContext) throws WrappedAuthErrorResponse { - if (FeatureFlags.API_BEARER_AUTH.enabled()) { - Optional bearerToken = getRequestApiKey(containerRequestContext); - // No Bearer Token present, hence no user can be authenticated - if (bearerToken.isEmpty()) { - return null; - } - - // Validate and verify provided Bearer Token, and retrieve UserRecordIdentifier - // TODO: Get the identifier from an invalidating cache to avoid lookup bursts of the same token. Tokens in the cache should be removed after some (configurable) time. - UserRecordIdentifier userInfo = verifyOidcBearerTokenAndGetUserIdentifier(bearerToken.get()); + if (!FeatureFlags.API_BEARER_AUTH.enabled()) { + return null; + } - // retrieve Authenticated User from AuthService - AuthenticatedUser authUser = authSvc.lookupUser(userInfo); - if (authUser != null) { - // track the API usage - authUser = userSvc.updateLastApiUseTime(authUser); - return authUser; - } else { - // a valid Token was presented, but we have no associated user account. - logger.log(Level.WARNING, "Bearer token detected, OIDC provider {0} validated Token but no linked UserAccount", userInfo.getUserRepoId()); - // TODO: Instead of returning null, we should throw a meaningful error to the client. - // Probably this will be a wrapped auth error response with an error code and a string describing the problem. - return null; - } + Optional bearerToken = getRequestBearerToken(containerRequestContext); + if (bearerToken.isEmpty()) { + return null; } - return null; - } - /** - * Verifies the given Bearer token and obtain information about the corresponding user within respective AuthProvider. - * - * @param token The string containing the encoded JWT - * @return - */ - private UserRecordIdentifier verifyOidcBearerTokenAndGetUserIdentifier(String token) throws WrappedAuthErrorResponse { + AuthenticatedUser authUser; try { - BearerAccessToken accessToken = BearerAccessToken.parse(token); - // Get list of all authentication providers using Open ID Connect - // @TASK: Limited to OIDCAuthProviders, could be widened to OAuth2Providers. - List providers = authSvc.getAuthenticationProviderIdsOfType(OIDCAuthProvider.class).stream() - .map(providerId -> (OIDCAuthProvider) authSvc.getAuthenticationProvider(providerId)) - .collect(Collectors.toUnmodifiableList()); - // If not OIDC Provider are configured we cannot validate a Token - if(providers.isEmpty()){ - logger.log(Level.WARNING, "Bearer token detected, no OIDC provider configured"); - throw new WrappedAuthErrorResponse(BEARER_TOKEN_DETECTED_NO_OIDC_PROVIDER_CONFIGURED); - } + authUser = authSvc.lookupUserByOIDCBearerToken(bearerToken.get()); + } catch (AuthorizationException e) { + logger.log(Level.WARNING, "Authorization failed: {0}", e.getMessage()); + throw new WrappedUnauthorizedAuthErrorResponse(e.getMessage()); + } - // Iterate over all OIDC providers if multiple. Sadly needed as do not know which provided the Token. - for (OIDCAuthProvider provider : providers) { - try { - // The OIDCAuthProvider need to verify a Bearer Token and equip the client means to identify the corresponding AuthenticatedUser. - Optional userInfo = provider.getUserIdentifier(accessToken); - if(userInfo.isPresent()) { - logger.log(Level.FINE, "Bearer token detected, provider {0} confirmed validity and provided identifier", provider.getId()); - return userInfo.get(); - } - } catch (IOException e) { - // TODO: Just logging this is not sufficient - if there is an IO error with the one provider - // which would have validated successfully, this is not the users fault. We need to - // take note and refer to that later when occurred. - logger.log(Level.FINE, "Bearer token detected, provider " + provider.getId() + " indicates an invalid Token, skipping", e); - } - } - } catch (ParseException e) { - logger.log(Level.FINE, "Bearer token detected, unable to parse bearer token (invalid Token)", e); - throw new WrappedAuthErrorResponse(INVALID_BEARER_TOKEN); + if (authUser == null) { + logger.log(Level.WARNING, "Bearer token detected, OIDC provider validated the token but no linked UserAccount"); + throw new WrappedForbiddenAuthErrorResponse(BundleUtil.getStringFromBundle("bearerTokenAuthMechanism.errors.tokenValidatedButNoRegisteredUser")); } - // No UserInfo returned means we have an invalid access token. - logger.log(Level.FINE, "Bearer token detected, yet no configured OIDC provider validated it."); - throw new WrappedAuthErrorResponse(UNAUTHORIZED_BEARER_TOKEN); + return userSvc.updateLastApiUseTime(authUser); } /** * Retrieve the raw, encoded token value from the Authorization Bearer HTTP header as defined in RFC 6750 + * * @return An {@link Optional} either empty if not present or the raw token from the header */ - private Optional getRequestApiKey(ContainerRequestContext containerRequestContext) { - String headerParamApiKey = containerRequestContext.getHeaderString(HttpHeaders.AUTHORIZATION); - if (headerParamApiKey != null && headerParamApiKey.toLowerCase().startsWith(BEARER_AUTH_SCHEME.toLowerCase() + " ")) { - return Optional.of(headerParamApiKey); - } else { - return Optional.empty(); - } + public static Optional getRequestBearerToken(ContainerRequestContext containerRequestContext) { + String headerParamBearerToken = containerRequestContext.getHeaderString(HttpHeaders.AUTHORIZATION); + return extractBearerTokenFromHeaderParam(headerParamBearerToken); } -} \ No newline at end of file +} diff --git a/src/main/java/edu/harvard/iq/dataverse/api/auth/CompoundAuthMechanism.java b/src/main/java/edu/harvard/iq/dataverse/api/auth/CompoundAuthMechanism.java index 801e2752b9e..e5be5144897 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/auth/CompoundAuthMechanism.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/auth/CompoundAuthMechanism.java @@ -5,6 +5,7 @@ import jakarta.inject.Inject; import jakarta.ws.rs.container.ContainerRequestContext; + import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -19,9 +20,9 @@ public class CompoundAuthMechanism implements AuthMechanism { private final List authMechanisms = new ArrayList<>(); @Inject - public CompoundAuthMechanism(ApiKeyAuthMechanism apiKeyAuthMechanism, WorkflowKeyAuthMechanism workflowKeyAuthMechanism, SignedUrlAuthMechanism signedUrlAuthMechanism, SessionCookieAuthMechanism sessionCookieAuthMechanism, BearerTokenAuthMechanism bearerTokenAuthMechanism) { + public CompoundAuthMechanism(ApiKeyAuthMechanism apiKeyAuthMechanism, WorkflowKeyAuthMechanism workflowKeyAuthMechanism, SignedUrlAuthMechanism signedUrlAuthMechanism, BearerTokenAuthMechanism bearerTokenAuthMechanism, SessionCookieAuthMechanism sessionCookieAuthMechanism) { // Auth mechanisms should be ordered by priority here - add(apiKeyAuthMechanism, workflowKeyAuthMechanism, signedUrlAuthMechanism, sessionCookieAuthMechanism,bearerTokenAuthMechanism); + add(apiKeyAuthMechanism, workflowKeyAuthMechanism, signedUrlAuthMechanism, bearerTokenAuthMechanism, sessionCookieAuthMechanism); } public CompoundAuthMechanism(AuthMechanism... authMechanisms) { diff --git a/src/main/java/edu/harvard/iq/dataverse/api/auth/SignedUrlAuthMechanism.java b/src/main/java/edu/harvard/iq/dataverse/api/auth/SignedUrlAuthMechanism.java index 258661f6495..30e8a3b9ca4 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/auth/SignedUrlAuthMechanism.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/auth/SignedUrlAuthMechanism.java @@ -43,7 +43,7 @@ public User findUserFromRequest(ContainerRequestContext containerRequestContext) if (user != null) { return user; } - throw new WrappedAuthErrorResponse(RESPONSE_MESSAGE_BAD_SIGNED_URL); + throw new WrappedUnauthorizedAuthErrorResponse(RESPONSE_MESSAGE_BAD_SIGNED_URL); } private String getSignedUrlRequestParameter(ContainerRequestContext containerRequestContext) { diff --git a/src/main/java/edu/harvard/iq/dataverse/api/auth/WorkflowKeyAuthMechanism.java b/src/main/java/edu/harvard/iq/dataverse/api/auth/WorkflowKeyAuthMechanism.java index bbd67713e85..df54b69af96 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/auth/WorkflowKeyAuthMechanism.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/auth/WorkflowKeyAuthMechanism.java @@ -30,7 +30,7 @@ public User findUserFromRequest(ContainerRequestContext containerRequestContext) if (authUser != null) { return authUser; } - throw new WrappedAuthErrorResponse(RESPONSE_MESSAGE_BAD_WORKFLOW_KEY); + throw new WrappedUnauthorizedAuthErrorResponse(RESPONSE_MESSAGE_BAD_WORKFLOW_KEY); } private String getRequestWorkflowKey(ContainerRequestContext containerRequestContext) { diff --git a/src/main/java/edu/harvard/iq/dataverse/api/auth/WrappedAuthErrorResponse.java b/src/main/java/edu/harvard/iq/dataverse/api/auth/WrappedAuthErrorResponse.java index 40431557261..da92d882197 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/auth/WrappedAuthErrorResponse.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/auth/WrappedAuthErrorResponse.java @@ -6,18 +6,24 @@ import jakarta.ws.rs.core.MediaType; import jakarta.ws.rs.core.Response; -public class WrappedAuthErrorResponse extends Exception { +public abstract class WrappedAuthErrorResponse extends Exception { private final String message; private final Response response; - public WrappedAuthErrorResponse(String message) { + public WrappedAuthErrorResponse(Response.Status status, String message) { this.message = message; - this.response = Response.status(Response.Status.UNAUTHORIZED) + this.response = createErrorResponse(status, message); + } + + protected Response createErrorResponse(Response.Status status, String message) { + return Response.status(status) .entity(NullSafeJsonBuilder.jsonObjectBuilder() .add("status", ApiConstants.STATUS_ERROR) .add("message", message).build() - ).type(MediaType.APPLICATION_JSON_TYPE).build(); + ) + .type(MediaType.APPLICATION_JSON_TYPE) + .build(); } public String getMessage() { diff --git a/src/main/java/edu/harvard/iq/dataverse/api/auth/WrappedForbiddenAuthErrorResponse.java b/src/main/java/edu/harvard/iq/dataverse/api/auth/WrappedForbiddenAuthErrorResponse.java new file mode 100644 index 00000000000..082ed3ca8d8 --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/api/auth/WrappedForbiddenAuthErrorResponse.java @@ -0,0 +1,10 @@ +package edu.harvard.iq.dataverse.api.auth; + +import jakarta.ws.rs.core.Response; + +public class WrappedForbiddenAuthErrorResponse extends WrappedAuthErrorResponse { + + public WrappedForbiddenAuthErrorResponse(String message) { + super(Response.Status.FORBIDDEN, message); + } +} diff --git a/src/main/java/edu/harvard/iq/dataverse/api/auth/WrappedUnauthorizedAuthErrorResponse.java b/src/main/java/edu/harvard/iq/dataverse/api/auth/WrappedUnauthorizedAuthErrorResponse.java new file mode 100644 index 00000000000..1d2eb8f8bd8 --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/api/auth/WrappedUnauthorizedAuthErrorResponse.java @@ -0,0 +1,10 @@ +package edu.harvard.iq.dataverse.api.auth; + +import jakarta.ws.rs.core.Response; + +public class WrappedUnauthorizedAuthErrorResponse extends WrappedAuthErrorResponse { + + public WrappedUnauthorizedAuthErrorResponse(String message) { + super(Response.Status.UNAUTHORIZED, message); + } +} diff --git a/src/main/java/edu/harvard/iq/dataverse/api/datadeposit/SWORDv2ServiceDocumentServlet.java b/src/main/java/edu/harvard/iq/dataverse/api/datadeposit/SWORDv2ServiceDocumentServlet.java index eab005d87fa..a32e97bcee2 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/datadeposit/SWORDv2ServiceDocumentServlet.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/datadeposit/SWORDv2ServiceDocumentServlet.java @@ -1,6 +1,8 @@ package edu.harvard.iq.dataverse.api.datadeposit; import java.io.IOException; + +import edu.harvard.iq.dataverse.engine.command.DataverseRequest; import jakarta.inject.Inject; import jakarta.servlet.ServletException; import jakarta.servlet.http.HttpServletRequest; @@ -29,6 +31,7 @@ public void init() throws ServletException { @Override protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException { + serviceDocumentManagerImpl.setIpAddress((new DataverseRequest(null, req)).getSourceAddress()); this.api.get(req, resp); } diff --git a/src/main/java/edu/harvard/iq/dataverse/api/datadeposit/ServiceDocumentManagerImpl.java b/src/main/java/edu/harvard/iq/dataverse/api/datadeposit/ServiceDocumentManagerImpl.java index 134d54aef88..62f23e97af9 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/datadeposit/ServiceDocumentManagerImpl.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/datadeposit/ServiceDocumentManagerImpl.java @@ -4,7 +4,9 @@ import edu.harvard.iq.dataverse.DataverseServiceBean; import edu.harvard.iq.dataverse.PermissionServiceBean; import edu.harvard.iq.dataverse.authorization.Permission; +import edu.harvard.iq.dataverse.authorization.groups.impl.ipaddress.ip.IpAddress; import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser; +import edu.harvard.iq.dataverse.engine.command.DataverseRequest; import edu.harvard.iq.dataverse.util.SystemConfig; import java.util.List; import java.util.logging.Logger; @@ -37,6 +39,8 @@ public class ServiceDocumentManagerImpl implements ServiceDocumentManager { @Inject UrlManager urlManager; + private IpAddress ipAddress = null; + @Override public ServiceDocument getServiceDocument(String sdUri, AuthCredentials authCredentials, SwordConfiguration config) throws SwordError, SwordServerException, SwordAuthException { @@ -65,7 +69,7 @@ public ServiceDocument getServiceDocument(String sdUri, AuthCredentials authCred * shibIdentityProvider String on AuthenticatedUser is only set when a * SAML assertion is made at runtime via the browser. */ - List dataverses = permissionService.getDataversesUserHasPermissionOn(user, Permission.AddDataset); + List dataverses = permissionService.findPermittedCollections(new DataverseRequest(user, ipAddress), user, Permission.AddDataset); for (Dataverse dataverse : dataverses) { String dvAlias = dataverse.getAlias(); if (dvAlias != null && !dvAlias.isEmpty()) { @@ -82,4 +86,7 @@ public ServiceDocument getServiceDocument(String sdUri, AuthCredentials authCred return service; } + public void setIpAddress(IpAddress ipAddress) { + this.ipAddress = ipAddress; + } } diff --git a/src/main/java/edu/harvard/iq/dataverse/api/dto/DatasetDTO.java b/src/main/java/edu/harvard/iq/dataverse/api/dto/DatasetDTO.java index 3fc31730ba2..ec8adfb4eef 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/dto/DatasetDTO.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/dto/DatasetDTO.java @@ -12,6 +12,7 @@ public class DatasetDTO implements java.io.Serializable { private String identifier; private String protocol; private String authority; + private String separator; private String globalIdCreateTime; private String publisher; private String publicationDate; @@ -51,6 +52,14 @@ public void setAuthority(String authority) { this.authority = authority; } + public String getSeparator() { + return separator; + } + + public void setSeparator(String separator) { + this.separator = separator; + } + public String getGlobalIdCreateTime() { return globalIdCreateTime; } @@ -94,7 +103,7 @@ public void setPublicationDate(String publicationDate) { @Override public String toString() { - return "DatasetDTO{" + "id=" + id + ", identifier=" + identifier + ", protocol=" + protocol + ", authority=" + authority + ", globalIdCreateTime=" + globalIdCreateTime + ", datasetVersion=" + datasetVersion + ", dataFiles=" + dataFiles + '}'; + return "DatasetDTO{" + "id=" + id + ", identifier=" + identifier + ", protocol=" + protocol + ", authority=" + authority + ", separator=" + separator + ", globalIdCreateTime=" + globalIdCreateTime + ", datasetVersion=" + datasetVersion + ", dataFiles=" + dataFiles + '}'; } public void setMetadataLanguage(String metadataLanguage) { diff --git a/src/main/java/edu/harvard/iq/dataverse/api/dto/DatasetVersionDTO.java b/src/main/java/edu/harvard/iq/dataverse/api/dto/DatasetVersionDTO.java index 37fe197280b..1bc080b8858 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/dto/DatasetVersionDTO.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/dto/DatasetVersionDTO.java @@ -10,7 +10,6 @@ * @author ellenk */ public class DatasetVersionDTO { - String archiveNote; String deacessionLink; // FIXME: Change to versionNumberMajor and versionNumberMinor? Some partial renaming of "minor" was done. Long versionNumber; @@ -47,6 +46,8 @@ public class DatasetVersionDTO { List fileMetadatas; List files; + String versionNote; + public boolean isInReview() { return inReview; } @@ -215,14 +216,6 @@ public void setFiles(List files) { this.files = files; } - public String getArchiveNote() { - return archiveNote; - } - - public void setArchiveNote(String archiveNote) { - this.archiveNote = archiveNote; - } - public String getDeacessionLink() { return deacessionLink; } @@ -328,17 +321,17 @@ public List getDatasetFields() { return null; } + public String getVersionNote() { + return versionNote; + } + + public void setVersionNote(String versionNote) { + this.versionNote = versionNote; + } + @Override public String toString() { - return "DatasetVersionDTO{" + "archiveNote=" + archiveNote + ", deacessionLink=" + deacessionLink + ", versionNumber=" + versionNumber + ", minorVersionNumber=" + versionMinorNumber + ", id=" + id + ", versionState=" + versionState + ", releaseDate=" + releaseDate + ", lastUpdateTime=" + lastUpdateTime + ", createTime=" + createTime + ", archiveTime=" + archiveTime + ", UNF=" + UNF + ", metadataBlocks=" + metadataBlocks + ", fileMetadatas=" + fileMetadatas + '}'; + return "DatasetVersionDTO{deacessionLink=" + deacessionLink + ", versionNumber=" + versionNumber + ", minorVersionNumber=" + versionMinorNumber + ", id=" + id + ", versionState=" + versionState + ", releaseDate=" + releaseDate + ", lastUpdateTime=" + lastUpdateTime + ", createTime=" + createTime + ", archiveTime=" + archiveTime + ", UNF=" + UNF + ", metadataBlocks=" + metadataBlocks + ", fileMetadatas=" + fileMetadatas + '}'; } - - - - - - - - - + } diff --git a/src/main/java/edu/harvard/iq/dataverse/api/dto/MetadataBlockDTO.java b/src/main/java/edu/harvard/iq/dataverse/api/dto/MetadataBlockDTO.java index 71eca470e75..168f48322bf 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/dto/MetadataBlockDTO.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/dto/MetadataBlockDTO.java @@ -56,7 +56,15 @@ public void addField(FieldDTO newField) { } else { // If this Field doesn't allow multiples, just replace the value // with the new field value. - current.value = newField.value; + // (or concatenate, if this is a primitive field) + if (newField.typeClass.equals("primitive")) { + String currentValue = current.getSinglePrimitive(); + String newValue = currentValue + " " + newField.getSinglePrimitive(); + current.setSinglePrimitive(newValue); + } else { + current.value = newField.value; + + } } } } diff --git a/src/main/java/edu/harvard/iq/dataverse/api/dto/NewDataverseFeaturedItemDTO.java b/src/main/java/edu/harvard/iq/dataverse/api/dto/NewDataverseFeaturedItemDTO.java new file mode 100644 index 00000000000..47003761abc --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/api/dto/NewDataverseFeaturedItemDTO.java @@ -0,0 +1,61 @@ +package edu.harvard.iq.dataverse.api.dto; + +import org.glassfish.jersey.media.multipart.FormDataContentDisposition; + +import java.io.InputStream; + +public class NewDataverseFeaturedItemDTO { + private String content; + private int displayOrder; + private InputStream imageFileInputStream; + private String imageFileName; + + public static NewDataverseFeaturedItemDTO fromFormData(String content, + int displayOrder, + InputStream imageFileInputStream, + FormDataContentDisposition contentDispositionHeader) { + NewDataverseFeaturedItemDTO newDataverseFeaturedItemDTO = new NewDataverseFeaturedItemDTO(); + + newDataverseFeaturedItemDTO.content = content; + newDataverseFeaturedItemDTO.displayOrder = displayOrder; + + if (imageFileInputStream != null) { + newDataverseFeaturedItemDTO.imageFileInputStream = imageFileInputStream; + newDataverseFeaturedItemDTO.imageFileName = contentDispositionHeader.getFileName(); + } + + return newDataverseFeaturedItemDTO; + } + + public void setContent(String content) { + this.content = content; + } + + public String getContent() { + return content; + } + + public void setDisplayOrder(int displayOrder) { + this.displayOrder = displayOrder; + } + + public int getDisplayOrder() { + return displayOrder; + } + + public void setImageFileInputStream(InputStream imageFileInputStream) { + this.imageFileInputStream = imageFileInputStream; + } + + public InputStream getImageFileInputStream() { + return imageFileInputStream; + } + + public void setImageFileName(String imageFileName) { + this.imageFileName = imageFileName; + } + + public String getImageFileName() { + return imageFileName; + } +} diff --git a/src/main/java/edu/harvard/iq/dataverse/api/dto/RoleDTO.java b/src/main/java/edu/harvard/iq/dataverse/api/dto/RoleDTO.java index 58e30ade584..5769ab430ad 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/dto/RoleDTO.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/dto/RoleDTO.java @@ -47,11 +47,11 @@ public void setPermissions(String[] permissions) { this.permissions = permissions; } - public DataverseRole asRole() { - DataverseRole r = new DataverseRole(); + public DataverseRole updateRoleFromDTO(DataverseRole r) { r.setAlias(alias); r.setDescription(description); r.setName(name); + r.clearPermissions(); if (permissions != null) { if (permissions.length > 0) { if (permissions[0].trim().toLowerCase().equals("all")) { @@ -65,5 +65,9 @@ public DataverseRole asRole() { } return r; } + + public DataverseRole asRole() { + return updateRoleFromDTO(new DataverseRole()); + } } diff --git a/src/main/java/edu/harvard/iq/dataverse/api/dto/UpdatedDataverseFeaturedItemDTO.java b/src/main/java/edu/harvard/iq/dataverse/api/dto/UpdatedDataverseFeaturedItemDTO.java new file mode 100644 index 00000000000..43d1afc31e2 --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/api/dto/UpdatedDataverseFeaturedItemDTO.java @@ -0,0 +1,72 @@ +package edu.harvard.iq.dataverse.api.dto; + +import org.glassfish.jersey.media.multipart.FormDataContentDisposition; + +import java.io.InputStream; + +public class UpdatedDataverseFeaturedItemDTO { + private String content; + private int displayOrder; + private boolean keepFile; + private InputStream imageFileInputStream; + private String imageFileName; + + public static UpdatedDataverseFeaturedItemDTO fromFormData(String content, + int displayOrder, + boolean keepFile, + InputStream imageFileInputStream, + FormDataContentDisposition contentDispositionHeader) { + UpdatedDataverseFeaturedItemDTO updatedDataverseFeaturedItemDTO = new UpdatedDataverseFeaturedItemDTO(); + + updatedDataverseFeaturedItemDTO.content = content; + updatedDataverseFeaturedItemDTO.displayOrder = displayOrder; + updatedDataverseFeaturedItemDTO.keepFile = keepFile; + + if (imageFileInputStream != null) { + updatedDataverseFeaturedItemDTO.imageFileInputStream = imageFileInputStream; + updatedDataverseFeaturedItemDTO.imageFileName = contentDispositionHeader.getFileName(); + } + + return updatedDataverseFeaturedItemDTO; + } + + public void setContent(String content) { + this.content = content; + } + + public String getContent() { + return content; + } + + public void setDisplayOrder(int displayOrder) { + this.displayOrder = displayOrder; + } + + public int getDisplayOrder() { + return displayOrder; + } + + public void setKeepFile(boolean keepFile) { + this.keepFile = keepFile; + } + + public boolean isKeepFile() { + return keepFile; + } + + public void setImageFileInputStream(InputStream imageFileInputStream) { + this.imageFileInputStream = imageFileInputStream; + } + + public InputStream getImageFileInputStream() { + return imageFileInputStream; + } + + public void setImageFileName(String imageFileName) { + this.imageFileName = imageFileName; + } + + public String getImageFileName() { + return imageFileName; + } +} diff --git a/src/main/java/edu/harvard/iq/dataverse/api/dto/UserDTO.java b/src/main/java/edu/harvard/iq/dataverse/api/dto/UserDTO.java new file mode 100644 index 00000000000..df1920c4d25 --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/api/dto/UserDTO.java @@ -0,0 +1,67 @@ +package edu.harvard.iq.dataverse.api.dto; + +public class UserDTO { + private String username; + private String firstName; + private String lastName; + private String emailAddress; + private String affiliation; + private String position; + private boolean termsAccepted; + + public String getUsername() { + return username; + } + + public void setUsername(String username) { + this.username = username; + } + + public String getFirstName() { + return firstName; + } + + public void setFirstName(String firstName) { + this.firstName = firstName; + } + + public String getLastName() { + return lastName; + } + + public void setLastName(String lastName) { + this.lastName = lastName; + } + + public String getEmailAddress() { + return emailAddress; + } + + public void setEmailAddress(String emailAddress) { + this.emailAddress = emailAddress; + } + + public String getAffiliation() { + return affiliation; + } + + public void setAffiliation(String affiliation) { + this.affiliation = affiliation; + } + + public String getPosition() { + return position; + } + + public void setPosition(String position) { + this.position = position; + } + + public boolean isTermsAccepted() { + return termsAccepted; + } + + public void setTermsAccepted(boolean termsAccepted) { + this.termsAccepted = termsAccepted; + } +} diff --git a/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportDDIServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportDDIServiceBean.java index 35d35316f73..31941d3c8c0 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportDDIServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportDDIServiceBean.java @@ -5,13 +5,21 @@ import edu.harvard.iq.dataverse.DatasetFieldType; import edu.harvard.iq.dataverse.DatasetVersion; import edu.harvard.iq.dataverse.DatasetVersion.VersionState; -import edu.harvard.iq.dataverse.api.dto.*; +import edu.harvard.iq.dataverse.api.dto.LicenseDTO; import edu.harvard.iq.dataverse.api.dto.FieldDTO; import edu.harvard.iq.dataverse.api.dto.MetadataBlockDTO; +import edu.harvard.iq.dataverse.api.dto.DatasetDTO; +import edu.harvard.iq.dataverse.api.dto.DatasetVersionDTO; +import edu.harvard.iq.dataverse.api.dto.FileMetadataDTO; +import edu.harvard.iq.dataverse.api.dto.DataFileDTO; +import edu.harvard.iq.dataverse.api.dto.DataTableDTO; + import edu.harvard.iq.dataverse.api.imports.ImportUtil.ImportType; import static edu.harvard.iq.dataverse.export.ddi.DdiExportUtil.NOTE_TYPE_CONTENTTYPE; import static edu.harvard.iq.dataverse.export.ddi.DdiExportUtil.NOTE_TYPE_TERMS_OF_ACCESS; +import edu.harvard.iq.dataverse.license.License; +import edu.harvard.iq.dataverse.license.LicenseServiceBean; import edu.harvard.iq.dataverse.util.StringUtil; import java.io.File; import java.io.FileInputStream; @@ -32,6 +40,9 @@ import org.apache.commons.lang3.StringUtils; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + /** * * @author ellenk @@ -103,6 +114,8 @@ public class ImportDDIServiceBean { @EJB DatasetFieldServiceBean datasetFieldService; @EJB ImportGenericServiceBean importGenericService; + + @EJB LicenseServiceBean licenseService; // TODO: stop passing the xml source as a string; (it could be huge!) -- L.A. 4.5 @@ -1180,7 +1193,24 @@ private void processDataAccs(XMLStreamReader xmlr, DatasetVersionDTO dvDTO) thro String noteType = xmlr.getAttributeValue(null, "type"); if (NOTE_TYPE_TERMS_OF_USE.equalsIgnoreCase(noteType) ) { if ( LEVEL_DV.equalsIgnoreCase(xmlr.getAttributeValue(null, "level"))) { - dvDTO.setTermsOfUse(parseText(xmlr, "notes")); + String termsOfUseStr = parseText(xmlr, "notes").trim(); + Pattern pattern = Pattern.compile("(.*)", Pattern.CASE_INSENSITIVE); + Matcher matcher = pattern.matcher(termsOfUseStr); + boolean matchFound = matcher.find(); + if (matchFound) { + String uri = matcher.group(1); + String license = matcher.group(2); + License lic = licenseService.getByNameOrUri(license); + if (lic != null) { + LicenseDTO licenseDTO = new LicenseDTO(); + licenseDTO.setName(license); + licenseDTO.setUri(uri); + dvDTO.setLicense(licenseDTO); + } + + } else { + dvDTO.setTermsOfUse(termsOfUseStr); + } } } else if (NOTE_TYPE_TERMS_OF_ACCESS.equalsIgnoreCase(noteType) ) { if (LEVEL_DV.equalsIgnoreCase(xmlr.getAttributeValue(null, "level"))) { diff --git a/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportGenericServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportGenericServiceBean.java index aa5b25e3967..d310a2ea6d4 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportGenericServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportGenericServiceBean.java @@ -299,9 +299,7 @@ private void processXMLElement(XMLStreamReader xmlr, String currentPath, String MetadataBlockDTO citationBlock = datasetDTO.getDatasetVersion().getMetadataBlocks().get(mappingDefinedFieldType.getMetadataBlock().getName()); citationBlock.addField(value); } - } else // Process the payload of this XML element: - //xxString dataverseFieldName = mappingDefined.getDatasetfieldName(); - if (dataverseFieldName != null && !dataverseFieldName.isEmpty()) { + } else if (dataverseFieldName != null && !dataverseFieldName.isEmpty()) { DatasetFieldType dataverseFieldType = datasetfieldService.findByNameOpt(dataverseFieldName); FieldDTO value; if (dataverseFieldType != null) { diff --git a/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportServiceBean.java index 7dc2aed799e..0b5fae8ee31 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportServiceBean.java @@ -215,6 +215,23 @@ public Dataset doImportHarvestedDataset(DataverseRequest dataverseRequest, File metadataFile, Date oaiDateStamp, PrintWriter cleanupLog) throws ImportException, IOException { + + logger.fine("importing " + metadataFormat + " saved in " + metadataFile.getAbsolutePath()); + + //@todo? check for an IOException here, throw ImportException instead, if caught + String metadataAsString = new String(Files.readAllBytes(metadataFile.toPath())); + return doImportHarvestedDataset(dataverseRequest, harvestingClient, harvestIdentifier, metadataFormat, metadataAsString, oaiDateStamp, cleanupLog); + } + + @TransactionAttribute(TransactionAttributeType.REQUIRES_NEW) + public Dataset doImportHarvestedDataset(DataverseRequest dataverseRequest, + HarvestingClient harvestingClient, + String harvestIdentifier, + String metadataFormat, + String metadataString, + Date oaiDateStamp, + PrintWriter cleanupLog) throws ImportException, IOException { + if (harvestingClient == null || harvestingClient.getDataverse() == null) { throw new ImportException("importHarvestedDataset called with a null harvestingClient, or an invalid harvestingClient."); } @@ -234,32 +251,28 @@ public Dataset doImportHarvestedDataset(DataverseRequest dataverseRequest, // Kraffmiller's export modules; replace the logic below with clean // programmatic lookup of the import plugin needed. + logger.fine("importing " + metadataFormat + " for " + harvestIdentifier); + if ("ddi".equalsIgnoreCase(metadataFormat) || "oai_ddi".equals(metadataFormat) || metadataFormat.toLowerCase().matches("^oai_ddi.*")) { try { - String xmlToParse = new String(Files.readAllBytes(metadataFile.toPath())); // TODO: // import type should be configurable - it should be possible to // select whether you want to harvest with or without files, // ImportType.HARVEST vs. ImportType.HARVEST_WITH_FILES - logger.fine("importing DDI "+metadataFile.getAbsolutePath()); - dsDTO = importDDIService.doImport(ImportType.HARVEST, xmlToParse); - } catch (IOException | XMLStreamException | ImportException e) { + dsDTO = importDDIService.doImport(ImportType.HARVEST, metadataString); + } catch (XMLStreamException | ImportException e) { throw new ImportException("Failed to process DDI XML record: "+ e.getClass() + " (" + e.getMessage() + ")"); } } else if ("dc".equalsIgnoreCase(metadataFormat) || "oai_dc".equals(metadataFormat)) { - logger.fine("importing DC "+metadataFile.getAbsolutePath()); try { - String xmlToParse = new String(Files.readAllBytes(metadataFile.toPath())); - dsDTO = importGenericService.processOAIDCxml(xmlToParse, harvestIdentifier, harvestingClient.isUseOaiIdentifiersAsPids()); - } catch (IOException | XMLStreamException e) { + dsDTO = importGenericService.processOAIDCxml(metadataString, harvestIdentifier, harvestingClient.isUseOaiIdentifiersAsPids()); + } catch (XMLStreamException e) { throw new ImportException("Failed to process Dublin Core XML record: "+ e.getClass() + " (" + e.getMessage() + ")"); } } else if ("dataverse_json".equals(metadataFormat)) { // This is Dataverse metadata already formatted in JSON. - // Simply read it into a string, and pass to the final import further down: - logger.fine("Attempting to import custom dataverse metadata from file "+metadataFile.getAbsolutePath()); - json = new String(Files.readAllBytes(metadataFile.toPath())); + json = metadataString; } else { throw new ImportException("Unsupported import metadata format: " + metadataFormat); } @@ -394,17 +407,23 @@ public Dataset doImportHarvestedDataset(DataverseRequest dataverseRequest, } catch (JsonParseException | ImportException | CommandException ex) { logger.fine("Failed to import harvested dataset: " + ex.getClass() + ": " + ex.getMessage()); - FileOutputStream savedJsonFileStream = new FileOutputStream(new File(metadataFile.getAbsolutePath() + ".json")); - byte[] jsonBytes = json.getBytes(); - int i = 0; - while (i < jsonBytes.length) { - int chunkSize = i + 8192 <= jsonBytes.length ? 8192 : jsonBytes.length - i; - savedJsonFileStream.write(jsonBytes, i, chunkSize); - i += chunkSize; - savedJsonFileStream.flush(); + + if (!"dataverse_json".equals(metadataFormat) && json != null) { + // If this was an xml format that were able to transform into + // our json, let's save it for debugging etc. purposes + File tempFile = File.createTempFile("meta", ".json"); + FileOutputStream savedJsonFileStream = new FileOutputStream(tempFile); + byte[] jsonBytes = json.getBytes(); + int i = 0; + while (i < jsonBytes.length) { + int chunkSize = i + 8192 <= jsonBytes.length ? 8192 : jsonBytes.length - i; + savedJsonFileStream.write(jsonBytes, i, chunkSize); + i += chunkSize; + savedJsonFileStream.flush(); + } + savedJsonFileStream.close(); + logger.info("JSON produced saved in " + tempFile.getAbsolutePath()); } - savedJsonFileStream.close(); - logger.info("JSON produced saved in " + metadataFile.getAbsolutePath() + ".json"); throw new ImportException("Failed to import harvested dataset: " + ex.getClass() + " (" + ex.getMessage() + ")", ex); } return importedDataset; diff --git a/src/main/java/edu/harvard/iq/dataverse/authorization/AuthUtil.java b/src/main/java/edu/harvard/iq/dataverse/authorization/AuthUtil.java index 27143199b5b..01f1cbdf0a3 100644 --- a/src/main/java/edu/harvard/iq/dataverse/authorization/AuthUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/authorization/AuthUtil.java @@ -3,6 +3,8 @@ import edu.harvard.iq.dataverse.authorization.providers.builtin.DataverseUserPage; import edu.harvard.iq.dataverse.authorization.providers.oauth2.AbstractOAuth2AuthenticationProvider; import edu.harvard.iq.dataverse.authorization.providers.shib.ShibAuthenticationProvider; +import edu.harvard.iq.dataverse.util.SystemConfig; + import java.util.Collection; import java.util.logging.Logger; @@ -10,12 +12,15 @@ public class AuthUtil { private static final Logger logger = Logger.getLogger(DataverseUserPage.class.getCanonicalName()); - public static boolean isNonLocalLoginEnabled(Collection providers) { + public static boolean isNonLocalSignupEnabled(Collection providers, SystemConfig systemConfig) { if (providers != null) { + for (AuthenticationProvider provider : providers) { if (provider instanceof AbstractOAuth2AuthenticationProvider || provider instanceof ShibAuthenticationProvider) { logger.fine("found an remote auth provider (returning true): " + provider.getId()); - return true; + if(!systemConfig.isSignupDisabledForRemoteAuthProvider(provider.getId())) { + return true; + } } else { logger.fine("not a remote auth provider: " + provider.getId()); } diff --git a/src/main/java/edu/harvard/iq/dataverse/authorization/AuthenticatedUserDisplayInfo.java b/src/main/java/edu/harvard/iq/dataverse/authorization/AuthenticatedUserDisplayInfo.java index 0bd1a81048a..94871c89083 100644 --- a/src/main/java/edu/harvard/iq/dataverse/authorization/AuthenticatedUserDisplayInfo.java +++ b/src/main/java/edu/harvard/iq/dataverse/authorization/AuthenticatedUserDisplayInfo.java @@ -14,16 +14,21 @@ public class AuthenticatedUserDisplayInfo extends RoleAssigneeDisplayInfo { @NotBlank(message = "{user.firstName}") private String firstName; private String position; + private String orcid; /* * @todo Shouldn't we persist the displayName too? It still exists on the * authenticateduser table. */ public AuthenticatedUserDisplayInfo(String firstName, String lastName, String emailAddress, String affiliation, String position) { + this(firstName, lastName, emailAddress, affiliation, position, null); + } + public AuthenticatedUserDisplayInfo(String firstName, String lastName, String emailAddress, String affiliation, String position, String orcid) { super(firstName + " " + lastName,emailAddress,affiliation); this.firstName = firstName; this.lastName = lastName; - this.position = position; + this.position = position; + this.orcid = orcid; } public AuthenticatedUserDisplayInfo() { @@ -31,6 +36,7 @@ public AuthenticatedUserDisplayInfo() { firstName=""; lastName=""; position=""; + orcid=null; } @@ -39,7 +45,7 @@ public AuthenticatedUserDisplayInfo() { * @param src the display info {@code this} will be a copy of. */ public AuthenticatedUserDisplayInfo( AuthenticatedUserDisplayInfo src ) { - this( src.getFirstName(), src.getLastName(), src.getEmailAddress(), src.getAffiliation(), src.getPosition()); + this( src.getFirstName(), src.getLastName(), src.getEmailAddress(), src.getAffiliation(), src.getPosition(), src.getOrcid()); } public String getLastName() { @@ -98,6 +104,27 @@ public boolean equals(Object obj) { } return Objects.equals(this.position, other.position) && super.equals(obj); } + + public void setOrcid(String orcidUrl) { + this.orcid=orcidUrl; + } + + public String getOrcid() { + return orcid; + } + + public String getOrcidForDisplay() { + String orcidUrl = getOrcid(); + if(orcidUrl == null) { + return null; + } + int index = orcidUrl.lastIndexOf('/'); + if (index > 0) { + return orcidUrl.substring(index + 1); + } else { + return orcidUrl; + } + } } diff --git a/src/main/java/edu/harvard/iq/dataverse/authorization/AuthenticationProvider.java b/src/main/java/edu/harvard/iq/dataverse/authorization/AuthenticationProvider.java index 2b137ca0dec..8dfe7ca86b3 100644 --- a/src/main/java/edu/harvard/iq/dataverse/authorization/AuthenticationProvider.java +++ b/src/main/java/edu/harvard/iq/dataverse/authorization/AuthenticationProvider.java @@ -33,16 +33,6 @@ public interface AuthenticationProvider { default boolean isUserInfoUpdateAllowed() { return false; }; default boolean isUserDeletionAllowed() { return false; }; default boolean isOAuthProvider() { return false; }; - /** @todo Consider moving some or all of these to AuthenticationProviderDisplayInfo.*/ - /** The identifier is only displayed in the UI if it's meaningful, such as an ORCID iD.*/ - default boolean isDisplayIdentifier() { return false; }; - /** ORCID calls their persistent id an "ORCID iD".*/ - default String getPersistentIdName() { return null; }; - /** ORCID has special language to describe their ID: http://members.orcid.org/logos-web-graphics */ - default String getPersistentIdDescription() { return null; }; - /** An ORCID example would be the "http://orcid.org/" part of http://orcid.org/0000-0002-7874-374X*/ - default String getPersistentIdUrlPrefix() { return null; }; - default String getLogo() { return null; }; diff --git a/src/main/java/edu/harvard/iq/dataverse/authorization/AuthenticationProvidersRegistrationServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/authorization/AuthenticationProvidersRegistrationServiceBean.java index fbad14645bc..1dfd7bdb713 100644 --- a/src/main/java/edu/harvard/iq/dataverse/authorization/AuthenticationProvidersRegistrationServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/authorization/AuthenticationProvidersRegistrationServiceBean.java @@ -15,6 +15,7 @@ import edu.harvard.iq.dataverse.authorization.providers.builtin.BuiltinUserServiceBean; import edu.harvard.iq.dataverse.authorization.providers.oauth2.AbstractOAuth2AuthenticationProvider; import edu.harvard.iq.dataverse.authorization.providers.oauth2.OAuth2AuthenticationProviderFactory; +import edu.harvard.iq.dataverse.authorization.providers.oauth2.impl.OrcidOAuth2AP; import edu.harvard.iq.dataverse.authorization.providers.oauth2.oidc.OIDCAuthenticationProviderFactory; import edu.harvard.iq.dataverse.authorization.providers.shib.ShibAuthenticationProviderFactory; import edu.harvard.iq.dataverse.settings.JvmSettings; @@ -87,6 +88,8 @@ public class AuthenticationProvidersRegistrationServiceBean { @PersistenceContext(unitName = "VDCNet-ejbPU") private EntityManager em; + + private AuthenticationProvider orcidProvider; // does this method also need an explicit @Lock(WRITE)? // - I'm assuming not; since it's guaranteed to only be called once, @@ -110,8 +113,9 @@ public void startup() { } // Now, load the providers. - em.createNamedQuery("AuthenticationProviderRow.findAllEnabled", AuthenticationProviderRow.class) + em.createNamedQuery("AuthenticationProviderRow.findAll", AuthenticationProviderRow.class) .getResultList().forEach((row) -> { + if(row.isEnabled()) { try { registerProvider( loadProvider(row) ); @@ -120,9 +124,28 @@ public void startup() { } catch (AuthorizationSetupException ex) { logger.log(Level.SEVERE, "Exception setting up the authentication provider '" + row.getId() + "': " + ex.getMessage(), ex); + } + } else { + // We still use an ORCID provider that is not enabled for login as a way to + // authenticate ORCIDs being added to account profiles + Map data = OAuth2AuthenticationProviderFactory + .parseFactoryData(row.getFactoryData()); + if ("orcid".equals(data.get("type"))) { + try { + setOrcidProvider(loadProvider(row)); + } catch (Exception e) { + logger.log(Level.SEVERE, "Cannot register ORCID provider '" + row.getId()); + } + } } - }); - + }); + // If there is an enabled ORCID provider, we'll still use that in preference to a disabled one (there should only be one but this would handle a case where, for example, someone has a disabled sandbox ORCID provider and a real enabled ORCID provider) + // Could be changed in the future if there's a need for two different clients for login and adding ORCIDs to profiles + for (AuthenticationProvider provider : authenticationProviders.values()) { + if (provider instanceof OrcidOAuth2AP) { + setOrcidProvider(provider); + } + } // Add providers registered via MPCONFIG if (JvmSettings.OIDC_ENABLED.lookupOptional(Boolean.class).orElse(false)) { try { @@ -133,6 +156,15 @@ public void startup() { } } + private void setOrcidProvider(AuthenticationProvider provider) { + orcidProvider = provider; + + } + + public AuthenticationProvider getOrcidProvider() { + return orcidProvider; + } + private void registerProviderFactory(AuthenticationProviderFactory aFactory) throws AuthorizationSetupException { diff --git a/src/main/java/edu/harvard/iq/dataverse/authorization/AuthenticationServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/authorization/AuthenticationServiceBean.java index 4a8fb123fd4..4b6fd5a1e69 100644 --- a/src/main/java/edu/harvard/iq/dataverse/authorization/AuthenticationServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/authorization/AuthenticationServiceBean.java @@ -1,11 +1,19 @@ package edu.harvard.iq.dataverse.authorization; +import com.nimbusds.oauth2.sdk.ParseException; +import com.nimbusds.oauth2.sdk.token.BearerAccessToken; +import com.nimbusds.openid.connect.sdk.claims.UserInfo; import edu.harvard.iq.dataverse.DatasetVersionServiceBean; import edu.harvard.iq.dataverse.DvObjectServiceBean; import edu.harvard.iq.dataverse.GuestbookResponseServiceBean; import edu.harvard.iq.dataverse.RoleAssigneeServiceBean; import edu.harvard.iq.dataverse.UserNotificationServiceBean; import edu.harvard.iq.dataverse.UserServiceBean; +import edu.harvard.iq.dataverse.authorization.exceptions.AuthorizationException; +import edu.harvard.iq.dataverse.authorization.providers.oauth2.OAuth2Exception; +import edu.harvard.iq.dataverse.authorization.providers.oauth2.OAuth2UserRecord; +import edu.harvard.iq.dataverse.authorization.providers.oauth2.impl.OrcidOAuth2AP; +import edu.harvard.iq.dataverse.authorization.providers.oauth2.oidc.OIDCAuthProvider; import edu.harvard.iq.dataverse.search.IndexServiceBean; import edu.harvard.iq.dataverse.actionlogging.ActionLogRecord; import edu.harvard.iq.dataverse.actionlogging.ActionLogServiceBean; @@ -34,21 +42,14 @@ import edu.harvard.iq.dataverse.validation.PasswordValidatorServiceBean; import edu.harvard.iq.dataverse.workflow.PendingWorkflowInvocation; import edu.harvard.iq.dataverse.workflows.WorkflowComment; + +import java.io.IOException; import java.sql.Timestamp; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Calendar; -import java.util.Collection; -import java.util.Date; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.TreeSet; +import java.util.*; import java.util.logging.Level; import java.util.logging.Logger; import java.util.stream.Collectors; -import jakarta.annotation.PostConstruct; + import jakarta.ejb.EJB; import jakarta.ejb.EJBException; import jakarta.ejb.Stateless; @@ -126,9 +127,8 @@ public class AuthenticationServiceBean { PrivateUrlServiceBean privateUrlService; @PersistenceContext(unitName = "VDCNet-ejbPU") - private EntityManager em; - - + EntityManager em; + public AbstractOAuth2AuthenticationProvider getOAuth2Provider( String id ) { return authProvidersRegistrationService.getOAuth2AuthProvidersMap().get(id); } @@ -978,4 +978,93 @@ public ApiToken getValidApiTokenForUser(User user) { } return apiToken; } -} + + /** + * Looks up an authenticated user based on the provided OIDC bearer token. + * + * @param bearerToken The OIDC bearer token. + * @return An instance of {@link AuthenticatedUser} representing the authenticated user. + * @throws AuthorizationException If the token is invalid or no OIDC provider is configured. + */ + public AuthenticatedUser lookupUserByOIDCBearerToken(String bearerToken) throws AuthorizationException { + // TODO: Get the identifier from an invalidating cache to avoid lookup bursts of the same token. + // Tokens in the cache should be removed after some (configurable) time. + OAuth2UserRecord oAuth2UserRecord = verifyOIDCBearerTokenAndGetOAuth2UserRecord(bearerToken); + return lookupUser(oAuth2UserRecord.getUserRecordIdentifier()); + } + + /** + * Verifies the given OIDC bearer token and retrieves the corresponding OAuth2UserRecord. + * + * @param bearerToken The OIDC bearer token. + * @return An {@link OAuth2UserRecord} containing the user's info. + * @throws AuthorizationException If the token is invalid or if no OIDC providers are available. + */ + public OAuth2UserRecord verifyOIDCBearerTokenAndGetOAuth2UserRecord(String bearerToken) throws AuthorizationException { + try { + BearerAccessToken accessToken = BearerAccessToken.parse(bearerToken); + List providers = getAvailableOidcProviders(); + + // Ensure at least one OIDC provider is configured to validate the token. + if (providers.isEmpty()) { + logger.log(Level.WARNING, "Bearer token detected, no OIDC provider configured"); + throw new AuthorizationException(BundleUtil.getStringFromBundle("authenticationServiceBean.errors.bearerTokenDetectedNoOIDCProviderConfigured")); + } + + // Attempt to validate the token with each configured OIDC provider. + for (OIDCAuthProvider provider : providers) { + try { + // Retrieve OAuth2UserRecord if UserInfo is present + Optional userInfo = provider.getUserInfo(accessToken); + if (userInfo.isPresent()) { + logger.log(Level.FINE, "Bearer token detected, provider {0} confirmed validity and provided user info", provider.getId()); + return provider.getUserRecord(userInfo.get()); + } + } catch (IOException | OAuth2Exception e) { + logger.log(Level.FINE, "Bearer token detected, provider " + provider.getId() + " indicates an invalid Token, skipping", e); + } + } + } catch (ParseException e) { + logger.log(Level.FINE, "Bearer token detected, unable to parse bearer token (invalid Token)", e); + throw new AuthorizationException(BundleUtil.getStringFromBundle("authenticationServiceBean.errors.invalidBearerToken")); + } + + // If no provider validated the token, throw an authorization exception. + logger.log(Level.FINE, "Bearer token detected, yet no configured OIDC provider validated it."); + throw new AuthorizationException(BundleUtil.getStringFromBundle("authenticationServiceBean.errors.unauthorizedBearerToken")); + } + + /** + * Retrieves a list of configured OIDC authentication providers. + * + * @return A list of available OIDCAuthProviders. + */ + private List getAvailableOidcProviders() { + return getAuthenticationProviderIdsOfType(OIDCAuthProvider.class).stream() + .map(providerId -> (OIDCAuthProvider) getAuthenticationProvider(providerId)) + .toList(); + } + + public OrcidOAuth2AP getOrcidAuthenticationProvider() { + return (OrcidOAuth2AP) authProvidersRegistrationService.getOrcidProvider(); + } + + public AuthenticatedUser lookupUserByOrcid(String orcid) { + if (orcid == null || orcid.isEmpty()) { + return null; + } + + try { + TypedQuery query = em.createQuery( + "SELECT au FROM AuthenticatedUser au WHERE au.authenticatedOrcid = :orcid", + AuthenticatedUser.class); + query.setParameter("orcid", orcid); + return query.getSingleResult(); + } catch (NoResultException e) { + return null; + } catch (NonUniqueResultException e) { + logger.log(Level.WARNING, "Multiple users found with ORCID: " + orcid, e); + return null; + } + } +} \ No newline at end of file diff --git a/src/main/java/edu/harvard/iq/dataverse/authorization/providers/AuthenticationProviderRow.java b/src/main/java/edu/harvard/iq/dataverse/authorization/providers/AuthenticationProviderRow.java index 2f37c777877..5b7fb09ebdf 100644 --- a/src/main/java/edu/harvard/iq/dataverse/authorization/providers/AuthenticationProviderRow.java +++ b/src/main/java/edu/harvard/iq/dataverse/authorization/providers/AuthenticationProviderRow.java @@ -43,6 +43,7 @@ public class AuthenticationProviderRow implements java.io.Serializable { private String factoryAlias; + //Enabled for login (and possibly for registration depending on the :AllowRemoteAuthSignUp setting) private boolean enabled; @Lob diff --git a/src/main/java/edu/harvard/iq/dataverse/authorization/providers/builtin/BuiltinAuthenticationProvider.java b/src/main/java/edu/harvard/iq/dataverse/authorization/providers/builtin/BuiltinAuthenticationProvider.java index c9edd04cc1e..94de21a88b9 100644 --- a/src/main/java/edu/harvard/iq/dataverse/authorization/providers/builtin/BuiltinAuthenticationProvider.java +++ b/src/main/java/edu/harvard/iq/dataverse/authorization/providers/builtin/BuiltinAuthenticationProvider.java @@ -156,9 +156,4 @@ public boolean isOAuthProvider() { return false; } - @Override - public boolean isDisplayIdentifier() { - return false; - } - } diff --git a/src/main/java/edu/harvard/iq/dataverse/authorization/providers/builtin/DataverseUserPage.java b/src/main/java/edu/harvard/iq/dataverse/authorization/providers/builtin/DataverseUserPage.java index 48afb2b830a..b43c66106b8 100644 --- a/src/main/java/edu/harvard/iq/dataverse/authorization/providers/builtin/DataverseUserPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/authorization/providers/builtin/DataverseUserPage.java @@ -36,6 +36,8 @@ import edu.harvard.iq.dataverse.util.BundleUtil; import edu.harvard.iq.dataverse.util.JsfHelper; import static edu.harvard.iq.dataverse.util.JsfHelper.JH; +import static edu.harvard.iq.dataverse.util.StringUtil.toOption; + import edu.harvard.iq.dataverse.util.SystemConfig; import edu.harvard.iq.dataverse.validation.PasswordValidatorServiceBean; import java.io.UnsupportedEncodingException; @@ -55,14 +57,18 @@ import jakarta.faces.application.FacesMessage; import jakarta.faces.component.UIComponent; import jakarta.faces.component.UIInput; +import jakarta.faces.context.ExternalContext; import jakarta.faces.context.FacesContext; import jakarta.faces.event.ActionEvent; import jakarta.faces.view.ViewScoped; import jakarta.inject.Inject; import jakarta.inject.Named; - +import jakarta.validation.constraints.NotBlank; +import edu.harvard.iq.dataverse.authorization.providers.oauth2.AbstractOAuth2AuthenticationProvider; +import edu.harvard.iq.dataverse.authorization.providers.oauth2.OAuth2LoginBackingBean; +import edu.harvard.iq.dataverse.authorization.providers.oauth2.impl.OrcidOAuth2AP; +import java.io.IOException; import org.apache.commons.lang3.StringUtils; -import org.hibernate.validator.constraints.NotBlank; import org.primefaces.event.TabChangeEvent; /** @@ -115,13 +121,17 @@ public enum EditMode { @EJB AuthenticationServiceBean authSvc; + + @Inject + private OAuth2LoginBackingBean oauth2LoginBackingBean; + private AuthenticatedUser currentUser; - private BuiltinUser builtinUser; private AuthenticatedUserDisplayInfo userDisplayInfo; private transient AuthenticationProvider userAuthProvider; private EditMode editMode; private String redirectPage = "dataverse.xhtml"; + private final String accountInfoTab = "dataverseuser.xhtml?selectTab=accountInfo"; @NotBlank(message = "{password.retype}") private String inputPassword; @@ -436,11 +446,13 @@ public void onTabChange(TabChangeEvent event) { if (event.getTab().getId().equals("notifications")) { displayNotification(); } + if (event.getTab().getId().equals("dataRelatedToMe")){ mydatapage.init(); } } + private String getRoleStringFromUser(AuthenticatedUser au, DvObject dvObj) { // Find user's role(s) for given dataverse/dataset Set roles = permissionService.assignmentsFor(au, dvObj); @@ -709,7 +721,7 @@ public void setUsername(String username) { } public boolean isNonLocalLoginEnabled() { - return AuthUtil.isNonLocalLoginEnabled(authenticationService.getAuthenticationProviders()); + return AuthUtil.isNonLocalSignupEnabled(authenticationService.getAuthenticationProviders(), systemConfig); } public String getReasonForReturn(DatasetVersion datasetVersion) { @@ -769,4 +781,47 @@ public boolean isDisabled(Type t) { return disabledNotifications.contains(t); } + public boolean isOrcidEnabled() { + return authenticationService.getOrcidAuthenticationProvider() != null; + } + + public void startOrcidAuthentication() { + OrcidOAuth2AP orcidProvider = authenticationService.getOrcidAuthenticationProvider(); + + if (orcidProvider == null) { + JsfHelper.addErrorMessage(BundleUtil.getStringFromBundle("auth.orcid.notConfigured")); + return; + } + + try { + // Use the appropriate method to get the authorization URL + String state = oauth2LoginBackingBean.createState(orcidProvider, toOption(accountInfoTab)); + String authorizationUrl = orcidProvider.buildAuthzUrl(state, + systemConfig.getDataverseSiteUrl() + "/oauth2/orcidConfirm.xhtml"); + ExternalContext externalContext = FacesContext.getCurrentInstance().getExternalContext(); + externalContext.redirect(authorizationUrl); + } catch (IOException ex) { + logger.log(Level.SEVERE, "Error starting ORCID authentication", ex); + JsfHelper.addErrorMessage(BundleUtil.getStringFromBundle("auth.orcid.error")); + } + } + + public void removeOrcid() { + currentUser.setAuthenticatedOrcid(null); + userService.save(currentUser); + } + + public String getOrcidForDisplay() { + if (currentUser == null || currentUser.getAuthenticatedOrcid() == null) { + return ""; + } + String orcidUrl = currentUser.getAuthenticatedOrcid(); + int index = orcidUrl.lastIndexOf('/'); + if (index > 0) { + return orcidUrl.substring(index + 1); + } else { + return orcidUrl; + } + } + } diff --git a/src/main/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/OAuth2FirstLoginPage.java b/src/main/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/OAuth2FirstLoginPage.java index 821e8a5ea6c..96ff5516c7b 100644 --- a/src/main/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/OAuth2FirstLoginPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/OAuth2FirstLoginPage.java @@ -181,7 +181,8 @@ public String createNewAccount() { newUser.getDisplayInfo().getLastName(), getSelectedEmail(), newUser.getDisplayInfo().getAffiliation(), - newUser.getDisplayInfo().getPosition()); + newUser.getDisplayInfo().getPosition(), + newUser.getDisplayInfo().getOrcid()); final AuthenticatedUser user = authenticationSvc.createAuthenticatedUser(newUser.getUserRecordIdentifier(), getUsername(), newAud, true); session.setUser(user); /** diff --git a/src/main/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/OAuth2LoginBackingBean.java b/src/main/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/OAuth2LoginBackingBean.java index 8f3dc07fdea..7e3b3fd3d90 100644 --- a/src/main/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/OAuth2LoginBackingBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/OAuth2LoginBackingBean.java @@ -5,7 +5,9 @@ import edu.harvard.iq.dataverse.authorization.AuthenticationProvider; import edu.harvard.iq.dataverse.authorization.AuthenticationServiceBean; import edu.harvard.iq.dataverse.authorization.UserRecordIdentifier; +import edu.harvard.iq.dataverse.authorization.providers.oauth2.impl.OrcidOAuth2AP; import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser; +import edu.harvard.iq.dataverse.authorization.users.User; import edu.harvard.iq.dataverse.util.BundleUtil; import edu.harvard.iq.dataverse.util.ClockUtil; import edu.harvard.iq.dataverse.util.StringUtil; @@ -125,6 +127,10 @@ public void exchangeCodeForToken() throws IOException { signUpDisabled = true; throw new OAuth2Exception(-1, "", MessageFormat.format(BundleUtil.getStringFromBundle("oauth2.callback.error.signupDisabledForProvider"), idp.getId())); } else { + if (idp instanceof OrcidOAuth2AP) { + oauthUser.getDisplayInfo() + .setOrcid(((OrcidOAuth2AP) idp).getOrcidUrl(oauthUser.getIdInService())); + } newAccountPage.setNewUser(oauthUser); Faces.redirect("/oauth2/firstLogin.xhtml"); } @@ -133,6 +139,13 @@ public void exchangeCodeForToken() throws IOException { // login the user and redirect to HOME of intended page (if any). // setUser checks for deactivated users. dvUser = userService.updateLastLogin(dvUser); + // On the first login (after this code was added) via the Orcid provider, set the user's ORCID + // Doing this here assures the user authenticated to ORCID before their profile's ORCID is set + // (and not, for example, when an account was created via API) + if((idp instanceof OrcidOAuth2AP) && dvUser.getAuthenticatedOrcid()==null) { + dvUser.setAuthenticatedOrcid(((OrcidOAuth2AP)idp).getOrcidUrl(oauthUser.getIdInService())); + userService.save(dvUser); + } session.setUser(dvUser); final OAuth2TokenData tokenData = oauthUser.getTokenData(); if (tokenData != null) { @@ -154,6 +167,53 @@ public void exchangeCodeForToken() throws IOException { } } + /** + * View action for orcidConfirm.xhtml, the browser redirect target for the OAuth2 provider. + * @throws IOException + */ + public void setOrcidInProfile() throws IOException { + HttpServletRequest req = Faces.getRequest(); + + try { + Optional oIdp = parseStateFromRequest(req.getParameter("state")); + Optional code = parseCodeFromRequest(req); + + if (oIdp.isPresent() && code.isPresent()) { + AbstractOAuth2AuthenticationProvider idp = oIdp.get(); + oauthUser = idp.getUserRecord(code.get(), req.getParameter("state"), systemConfig.getDataverseSiteUrl() + "/oauth2/orcidConfirm.xhtml"); + + UserRecordIdentifier idtf = oauthUser.getUserRecordIdentifier(); + User user = session.getUser(); + String orcid = ((OrcidOAuth2AP)idp).getOrcidUrl(oauthUser.getIdInService()); + if(authenticationSvc.lookupUserByOrcid(orcid)!= null) { + throw new OAuth2Exception(-1, "", MessageFormat.format(BundleUtil.getStringFromBundle("oauth2.callback.error.orcidInUse"), orcid)); + } + if(user != null && user.isAuthenticated()) { + AuthenticatedUser dvUser = (AuthenticatedUser) user; + if((idp instanceof OrcidOAuth2AP) && dvUser.getAuthenticatedOrcid()==null) { + dvUser.setAuthenticatedOrcid(orcid); + userService.save(dvUser); + } + session.setUser(dvUser); + + Faces.redirect(redirectPage.orElse("/")); + + } else { + throw new OAuth2Exception(-1, "", MessageFormat.format(BundleUtil.getStringFromBundle("oauth2.callback.error.accountNotFound"), idp.getId())); + + } + } + } catch (OAuth2Exception ex) { + error = ex; + logger.log(Level.INFO, "ORCID OAuth2Exception caught. HTTP return code: {0}. Message: {1}. Message body: {2}", new Object[]{error.getHttpReturnCode(), error.getLocalizedMessage(), error.getMessageBody()}); + Logger.getLogger(OAuth2LoginBackingBean.class.getName()).log(Level.SEVERE, null, ex); + } catch (InterruptedException | ExecutionException ex) { + error = new OAuth2Exception(-1, "Please see server logs for more details", "Could not login at ORCID due to threading exceptions."); + logger.log(Level.WARNING, "Threading exception caught. Message: {0}", ex.getLocalizedMessage()); + } + } + + /** * TODO: Refactor this to be included in calling method. * TODO: Use org.apache.commons.io.IOUtils.toString(req.getReader()) instead of overcomplicated code below. @@ -204,8 +264,14 @@ Optional parseStateFromRequest(@NotNull St } AbstractOAuth2AuthenticationProvider idp = authenticationSvc.getOAuth2Provider(topFields[0]); if (idp == null) { - logger.log(Level.INFO, "Can''t find IDP ''{0}''", topFields[0]); - return Optional.empty(); + //No login enabled provider matches, try the Orcid provider as this could be an attempt to add an ORCID to a profile + AbstractOAuth2AuthenticationProvider possibleIdp = authenticationSvc.getOrcidAuthenticationProvider(); + if(possibleIdp != null && possibleIdp.getId().equals(topFields[0])) { + idp = possibleIdp; + } else { + logger.log(Level.INFO, "Can''t find IDP ''{0}''", topFields[0]); + return Optional.empty(); + } } // Verify the response by decrypting values and check for state valid timeout @@ -235,7 +301,7 @@ Optional parseStateFromRequest(@NotNull St * @param redirectPage * @return Random state string, composed from system time, random numbers and redirectPage parameter */ - String createState(AbstractOAuth2AuthenticationProvider idp, Optional redirectPage) { + public String createState(AbstractOAuth2AuthenticationProvider idp, Optional redirectPage) { if (idp == null) { throw new IllegalArgumentException("idp cannot be null"); } diff --git a/src/main/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/impl/GitHubOAuth2AP.java b/src/main/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/impl/GitHubOAuth2AP.java index 8829a25336b..5b201ce77db 100644 --- a/src/main/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/impl/GitHubOAuth2AP.java +++ b/src/main/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/impl/GitHubOAuth2AP.java @@ -58,30 +58,4 @@ protected ParsedUserResponse parseUserResponse( String responseBody ) { } } - - @Override - public boolean isDisplayIdentifier() { - return false; - } - - @Override - public String getPersistentIdName() { - return BundleUtil.getStringFromBundle("auth.providers.persistentUserIdName.github"); - } - - @Override - public String getPersistentIdDescription() { - return BundleUtil.getStringFromBundle("auth.providers.persistentUserIdTooltip.github"); - } - - @Override - public String getPersistentIdUrlPrefix() { - return null; - } - - @Override - public String getLogo() { - return null; - } - } diff --git a/src/main/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/impl/GoogleOAuth2AP.java b/src/main/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/impl/GoogleOAuth2AP.java index a864ecb810a..913eb038d8e 100644 --- a/src/main/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/impl/GoogleOAuth2AP.java +++ b/src/main/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/impl/GoogleOAuth2AP.java @@ -63,10 +63,4 @@ protected ParsedUserResponse parseUserResponse(String responseBody) { return new ParsedUserResponse(displayInfo, persistentUserId, username); } } - - @Override - public boolean isDisplayIdentifier() { - return false; - } - } diff --git a/src/main/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/impl/OrcidOAuth2AP.java b/src/main/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/impl/OrcidOAuth2AP.java index 323c78ab47a..ddf527b95a7 100644 --- a/src/main/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/impl/OrcidOAuth2AP.java +++ b/src/main/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/impl/OrcidOAuth2AP.java @@ -12,6 +12,7 @@ import edu.harvard.iq.dataverse.authorization.providers.oauth2.OAuth2Exception; import edu.harvard.iq.dataverse.authorization.providers.oauth2.OAuth2TokenData; import edu.harvard.iq.dataverse.authorization.providers.oauth2.OAuth2UserRecord; +import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser; import edu.harvard.iq.dataverse.util.BundleUtil; import java.io.IOException; import java.io.StringReader; @@ -51,11 +52,10 @@ public class OrcidOAuth2AP extends AbstractOAuth2AuthenticationProvider { static final Logger logger = Logger.getLogger(OrcidOAuth2AP.class.getName()); - public static final String PROVIDER_ID_PRODUCTION = "orcid"; - public static final String PROVIDER_ID_SANDBOX = "orcid-sandbox"; + public static final String PROVIDER_ID = "orcid"; public OrcidOAuth2AP(String clientId, String clientSecret, String userEndpoint) { - + this.id=PROVIDER_ID; if(userEndpoint != null && userEndpoint.startsWith("https://pub")) { this.scope = Arrays.asList("/authenticate"); } else { @@ -78,7 +78,7 @@ public String getUserEndpoint( OAuth2AccessToken token ) { @Override public DefaultApi20 getApiInstance() { - return OrcidApi.instance( ! baseUserEndpoint.contains("sandbox") ); + return OrcidApi.instance( isProduction() ); } @Override @@ -233,34 +233,20 @@ private NodeList xpathMatches(Document doc, String pattern) { @Override public AuthenticationProviderDisplayInfo getInfo() { - if (PROVIDER_ID_PRODUCTION.equals(getId())) { + if (isProduction()) { return new AuthenticationProviderDisplayInfo(getId(), BundleUtil.getStringFromBundle("auth.providers.title.orcid"), "ORCID user repository"); } return new AuthenticationProviderDisplayInfo(getId(), "ORCID Sandbox", "ORCID dev sandbox "); } - @Override - public boolean isDisplayIdentifier() { - return true; - } - - @Override - public String getPersistentIdName() { - return BundleUtil.getStringFromBundle("auth.providers.persistentUserIdName.orcid"); - } - - @Override - public String getPersistentIdDescription() { - return BundleUtil.getStringFromBundle("auth.providers.persistentUserIdTooltip.orcid"); - } - - @Override public String getPersistentIdUrlPrefix() { - return "https://orcid.org/"; + if(isProduction()) { + return "https://orcid.org/"; + } + return "https://sandbox.orcid.org/"; } - @Override - public String getLogo() { + public final static String getLogo() { return "/resources/images/orcid_16x16.png"; } @@ -329,4 +315,12 @@ protected AuthenticatedUserDisplayInfo parseActivitiesResponse( String responseB return null; } + + public String getOrcidUrl(String id) { + return (id == null) ? null : getPersistentIdUrlPrefix() + id; + } + + private boolean isProduction() { + return !baseUserEndpoint.contains("sandbox"); + } } diff --git a/src/main/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/oidc/OIDCAuthProvider.java b/src/main/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/oidc/OIDCAuthProvider.java index 5eb2b391eb7..5cf8ca2ea55 100644 --- a/src/main/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/oidc/OIDCAuthProvider.java +++ b/src/main/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/oidc/OIDCAuthProvider.java @@ -94,18 +94,6 @@ public OIDCAuthProvider(String aClientId, String aClientSecret, String issuerEnd this.pkceMethod = CodeChallengeMethod.parse(pkceMethod); } - /** - * Although this is defined in {@link edu.harvard.iq.dataverse.authorization.AuthenticationProvider}, - * this needs to be present due to bugs in ELResolver (has been modified for Spring). - * TODO: for the future it might be interesting to make this configurable via the provider JSON (it's used for ORCID!) - * @see JBoss Issue 159 - * @see Jakarta EE Bug 43 - * @return false - */ - @Override - public boolean isDisplayIdentifier() { - return false; - } /** * Setup metadata from OIDC provider during creation of the provider representation @@ -242,7 +230,7 @@ public OAuth2UserRecord getUserRecord(String code, String state, String redirect * @param userInfo * @return the usable user record for processing ing {@link edu.harvard.iq.dataverse.authorization.providers.oauth2.OAuth2LoginBackingBean} */ - OAuth2UserRecord getUserRecord(UserInfo userInfo) { + public OAuth2UserRecord getUserRecord(UserInfo userInfo) { return new OAuth2UserRecord( this.getId(), userInfo.getSubject().getValue(), @@ -291,7 +279,7 @@ Optional getAccessToken(AuthorizationGrant grant) throws IOEx * Retrieve User Info from provider. Encapsulate for testing. * @param accessToken The access token to enable reading data from userinfo endpoint */ - Optional getUserInfo(BearerAccessToken accessToken) throws IOException, OAuth2Exception { + public Optional getUserInfo(BearerAccessToken accessToken) throws IOException, OAuth2Exception { // Retrieve data HTTPResponse response = new UserInfoRequest(this.idpMetadata.getUserInfoEndpointURI(), accessToken) .toHTTPRequest() @@ -316,44 +304,4 @@ Optional getUserInfo(BearerAccessToken accessToken) throws IOException throw new OAuth2Exception(-1, ex.getMessage(), BundleUtil.getStringFromBundle("auth.providers.exception.userinfo", Arrays.asList(this.getTitle()))); } } - - /** - * Trades an access token for an {@link UserRecordIdentifier} (if valid). - * - * @apiNote The resulting {@link UserRecordIdentifier} may be used with - * {@link edu.harvard.iq.dataverse.authorization.AuthenticationServiceBean#lookupUser(UserRecordIdentifier)} - * to look up an {@link edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser} from the database. - * @see edu.harvard.iq.dataverse.api.auth.BearerTokenAuthMechanism - * - * @param accessToken The token to use when requesting user information from the provider - * @return Returns an {@link UserRecordIdentifier} for a valid access token or an empty {@link Optional}. - * @throws IOException In case communication with the endpoint fails to succeed for an I/O reason - */ - public Optional getUserIdentifier(BearerAccessToken accessToken) throws IOException { - OAuth2UserRecord userRecord; - try { - // Try to retrieve with given token (throws if invalid token) - Optional userInfo = getUserInfo(accessToken); - - if (userInfo.isPresent()) { - // Take this detour to avoid code duplication and potentially hard to track conversion errors. - userRecord = getUserRecord(userInfo.get()); - } else { - // This should not happen - an error at the provider side will lead to an exception. - logger.log(Level.WARNING, - "User info retrieval from {0} returned empty optional but expected exception for token {1}.", - List.of(getId(), accessToken).toArray() - ); - return Optional.empty(); - } - } catch (OAuth2Exception e) { - logger.log(Level.FINE, - "Could not retrieve user info with token {0} at provider {1}: {2}", - List.of(accessToken, getId(), e.getMessage()).toArray()); - logger.log(Level.FINER, "Retrieval failed, details as follows: ", e); - return Optional.empty(); - } - - return Optional.of(userRecord.getUserRecordIdentifier()); - } } diff --git a/src/main/java/edu/harvard/iq/dataverse/authorization/providers/shib/ShibAuthenticationProvider.java b/src/main/java/edu/harvard/iq/dataverse/authorization/providers/shib/ShibAuthenticationProvider.java index e7dccc34300..0f6be352ed9 100644 --- a/src/main/java/edu/harvard/iq/dataverse/authorization/providers/shib/ShibAuthenticationProvider.java +++ b/src/main/java/edu/harvard/iq/dataverse/authorization/providers/shib/ShibAuthenticationProvider.java @@ -28,11 +28,6 @@ public boolean isOAuthProvider() { return false; } - @Override - public boolean isDisplayIdentifier() { - return false; - } - // We don't override "isEmailVerified" because we're using timestamps // ("emailconfirmed" on the "authenticateduser" table) to know if // Shib users have confirmed/verified their email or not. diff --git a/src/main/java/edu/harvard/iq/dataverse/authorization/users/AuthenticatedUser.java b/src/main/java/edu/harvard/iq/dataverse/authorization/users/AuthenticatedUser.java index d6d3e0317ed..123155f06e0 100644 --- a/src/main/java/edu/harvard/iq/dataverse/authorization/users/AuthenticatedUser.java +++ b/src/main/java/edu/harvard/iq/dataverse/authorization/users/AuthenticatedUser.java @@ -152,6 +152,10 @@ public class AuthenticatedUser implements User, Serializable { @Min(value = 1, message = "Rate Limit Tier must be greater than 0.") private int rateLimitTier = 1; + //The user's ORCID - only populated if user has authenticated to ORCID to assure they own it + @Column(nullable=true, length=45) + private String authenticatedOrcid; + @PrePersist void prePersist() { mutedNotifications = Type.toStringValue(mutedNotificationsSet); @@ -238,7 +242,7 @@ public List getRequestedDataFiles(){ @Override public AuthenticatedUserDisplayInfo getDisplayInfo() { - return new AuthenticatedUserDisplayInfo(firstName, lastName, email, affiliation, position); + return new AuthenticatedUserDisplayInfo(firstName, lastName, email, affiliation, position, authenticatedOrcid); } /** @@ -257,6 +261,9 @@ public void applyDisplayInfo( AuthenticatedUserDisplayInfo inf ) { if ( nonEmpty(inf.getPosition()) ) { setPosition( inf.getPosition()); } + if ( nonEmpty(inf.getOrcid()) ) { + setAuthenticatedOrcid(inf.getOrcid()); + } } // For Shib users, set "email confirmed" timestamp on login. @@ -554,14 +561,6 @@ public Timestamp getLastApiUseTime(){ return this.lastApiUseTime; } - - public String getOrcidId() { - String authProviderId = getAuthenticatedUserLookup().getAuthenticationProviderId(); - if (OrcidOAuth2AP.PROVIDER_ID_PRODUCTION.equals(authProviderId)) { - return getAuthenticatedUserLookup().getPersistentUserId(); - } - return null; - } public Cart getCart() { if (cart == null){ @@ -605,4 +604,12 @@ public boolean hasNotificationMuted(Type type) { } return this.mutedNotificationsSet.contains(type); } + + public String getAuthenticatedOrcid() { + return authenticatedOrcid; + } + + public void setAuthenticatedOrcid(String orcid) { + this.authenticatedOrcid = orcid; + } } diff --git a/src/main/java/edu/harvard/iq/dataverse/dashboard/DashboardDatamovePage.java b/src/main/java/edu/harvard/iq/dataverse/dashboard/DashboardMoveDatasetPage.java similarity index 79% rename from src/main/java/edu/harvard/iq/dataverse/dashboard/DashboardDatamovePage.java rename to src/main/java/edu/harvard/iq/dataverse/dashboard/DashboardMoveDatasetPage.java index 6fc80312bf5..b1333b02a46 100644 --- a/src/main/java/edu/harvard/iq/dataverse/dashboard/DashboardDatamovePage.java +++ b/src/main/java/edu/harvard/iq/dataverse/dashboard/DashboardMoveDatasetPage.java @@ -28,13 +28,11 @@ import jakarta.faces.view.ViewScoped; import jakarta.inject.Inject; import jakarta.inject.Named; -import jakarta.persistence.EntityManager; -import jakarta.persistence.PersistenceContext; import jakarta.servlet.http.HttpServletRequest; @ViewScoped -@Named("DashboardDatamovePage") -public class DashboardDatamovePage implements java.io.Serializable { +@Named("DashboardMoveDatasetPage") +public class DashboardMoveDatasetPage implements java.io.Serializable { @Inject DataverseSession session; @@ -49,11 +47,8 @@ public class DashboardDatamovePage implements java.io.Serializable { DataverseServiceBean dataverseService; @Inject SettingsWrapper settingsWrapper; - - @PersistenceContext(unitName = "VDCNet-ejbPU") - private EntityManager em; - private static final Logger logger = Logger.getLogger(DashboardDatamovePage.class.getCanonicalName()); + private static final Logger logger = Logger.getLogger(DashboardMoveDatasetPage.class.getCanonicalName()); private AuthenticatedUser authUser = null; @@ -122,18 +117,18 @@ public String init() { FacesContext.getCurrentInstance().addMessage(null, new FacesMessage(FacesMessage.SEVERITY_INFO, - BundleUtil.getStringFromBundle("dashboard.card.datamove.manage"), - BundleUtil.getStringFromBundle("dashboard.card.datamove.message", Arrays.asList(settingsWrapper.getGuidesBaseUrl(), settingsWrapper.getGuidesVersion())))); + BundleUtil.getStringFromBundle("dashboard.card.move.dataset.manage"), + BundleUtil.getStringFromBundle("dashboard.move.dataset.message", Arrays.asList(settingsWrapper.getGuidesBaseUrl(), settingsWrapper.getGuidesVersion())))); return null; } public void move(){ Dataset ds = selectedSourceDataset; - String dsPersistentId = ds!=null?ds.getGlobalId().asString():null; - String srcAlias = ds!=null?ds.getOwner().getAlias():null; + String dsPersistentId = ds != null ? ds.getGlobalId().asString() : null; + String srcAlias = ds != null ? ds.getOwner().getAlias() : null; Dataverse target = selectedDestinationDataverse; - String dstAlias = target!=null?target.getAlias():null; + String dstAlias = target != null ? target.getAlias() : null; if (ds == null || target == null) { // Move only works if both inputs are correct @@ -148,9 +143,9 @@ public void move(){ // construct arguments for message List arguments = new ArrayList<>(); - arguments.add(ds!=null?ds.getDisplayName():"-"); - arguments.add(dsPersistentId!=null?dsPersistentId:"-"); - arguments.add(target!=null?target.getName():"-"); + arguments.add(ds != null ? ds.getDisplayName() : "-"); + arguments.add(dsPersistentId != null ? dsPersistentId : "-"); + arguments.add(target != null ? target.getName() : "-"); // copied logic from Datasets API move //Command requires Super user - it will be tested by the command @@ -163,7 +158,7 @@ public void move(){ logger.info("Moved " + dsPersistentId + " from " + srcAlias + " to " + dstAlias); - JsfHelper.addSuccessMessage(BundleUtil.getStringFromBundle("dashboard.card.datamove.message.success", arguments)); + JsfHelper.addSuccessMessage(BundleUtil.getStringFromBundle("dashboard.move.dataset.message.success", arguments)); } catch (CommandException e) { logger.log(Level.SEVERE,"Unable to move "+ dsPersistentId + " from " + srcAlias + " to " + dstAlias, e); @@ -172,25 +167,20 @@ public void move(){ String guidesBaseUrl = settingsWrapper.getGuidesBaseUrl(); String version = settingsWrapper.getGuidesVersion(); // Suggest using the API to force the move. - arguments.add(BundleUtil.getStringFromBundle("dashboard.card.datamove.dataset.command.error.unforced.suggestForce", Arrays.asList(guidesBaseUrl, version))); + arguments.add(BundleUtil.getStringFromBundle("dashboard.move.dataset.command.error.unforced.suggestForce", Arrays.asList(guidesBaseUrl, version))); } else { String emptyStringNoDetails = ""; arguments.add(emptyStringNoDetails); } FacesContext.getCurrentInstance().addMessage(null, new FacesMessage(FacesMessage.SEVERITY_ERROR, - BundleUtil.getStringFromBundle("dashboard.card.datamove.message.failure.summary"), - BundleUtil.getStringFromBundle("dashboard.card.datamove.message.failure.details", arguments))); + BundleUtil.getStringFromBundle("dashboard.move.dataset.message.failure.summary"), + BundleUtil.getStringFromBundle("dashboard.move.dataset.message.failure.details", arguments))); } } - public String getDataverseCount() { - long count = em.createQuery("SELECT count(dv) FROM Dataverse dv", Long.class).getSingleResult(); - return NumberFormat.getInstance().format(count); - } - public String getDatasetCount() { - long count = em.createQuery("SELECT count(ds) FROM Dataset ds", Long.class).getSingleResult(); + long count = datasetService.getDatasetCount(); return NumberFormat.getInstance().format(count); } diff --git a/src/main/java/edu/harvard/iq/dataverse/dashboard/DashboardMoveDataversePage.java b/src/main/java/edu/harvard/iq/dataverse/dashboard/DashboardMoveDataversePage.java new file mode 100644 index 00000000000..be3d05a823e --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/dashboard/DashboardMoveDataversePage.java @@ -0,0 +1,163 @@ +package edu.harvard.iq.dataverse.dashboard; + +import edu.harvard.iq.dataverse.Dataverse; +import edu.harvard.iq.dataverse.DataverseServiceBean; +import edu.harvard.iq.dataverse.DataverseSession; +import edu.harvard.iq.dataverse.EjbDataverseEngine; +import edu.harvard.iq.dataverse.PermissionsWrapper; +import edu.harvard.iq.dataverse.SettingsWrapper; +import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser; +import edu.harvard.iq.dataverse.engine.command.DataverseRequest; +import edu.harvard.iq.dataverse.engine.command.exception.CommandException; +import edu.harvard.iq.dataverse.engine.command.impl.MoveDataverseCommand; +import edu.harvard.iq.dataverse.util.BundleUtil; +import edu.harvard.iq.dataverse.util.JsfHelper; +import java.text.NumberFormat; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.logging.Level; +import java.util.logging.Logger; +import jakarta.ejb.EJB; +import jakarta.faces.application.FacesMessage; +import jakarta.faces.component.UIInput; +import jakarta.faces.context.FacesContext; +import jakarta.faces.view.ViewScoped; +import jakarta.inject.Inject; +import jakarta.inject.Named; +import jakarta.servlet.http.HttpServletRequest; + +@ViewScoped +@Named("DashboardMoveDataversePage") +public class DashboardMoveDataversePage implements java.io.Serializable { + + @Inject + DataverseSession session; + @Inject + PermissionsWrapper permissionsWrapper; + @EJB + EjbDataverseEngine commandEngine; + @EJB + DataverseServiceBean dataverseService; + @Inject + SettingsWrapper settingsWrapper; + + private static final Logger logger = Logger.getLogger(DashboardMoveDataversePage.class.getCanonicalName()); + + private AuthenticatedUser authUser = null; + + // source dataverse + + public UIInput getSelectedSourceDataverseMenu() { + return selectedSourceDataverseMenu; + } + + public void setSelectedSourceDataverseMenu(UIInput selectedSourceDataverseMenu) { + this.selectedSourceDataverseMenu = selectedSourceDataverseMenu; + } + + UIInput selectedSourceDataverseMenu; + + public Dataverse getSelectedSourceDataverse() { + return selectedSourceDataverse; + } + + public void setSelectedSourceDataverse(Dataverse selectedSourceDataverse) { + this.selectedSourceDataverse = selectedSourceDataverse; + } + + private Dataverse selectedSourceDataverse; + + // destination dataverse + + public UIInput getSelectedDataverseMenu() { + return selectedDataverseMenu; + } + + public void setSelectedDataverseMenu(UIInput selectedDataverseMenu) { + this.selectedDataverseMenu = selectedDataverseMenu; + } + + UIInput selectedDataverseMenu; + + public Dataverse getSelectedDestinationDataverse() { + return selectedDestinationDataverse; + } + + public void setSelectedDestinationDataverse(Dataverse selectedDestinationDataverse) { + this.selectedDestinationDataverse = selectedDestinationDataverse; + } + + private Dataverse selectedDestinationDataverse; + + public List completeSelectedDataverse(String query) { + return dataverseService.filterByAliasQuery(query); + } + + public String init() { + + if ((session.getUser() != null) && (session.getUser().isAuthenticated()) && (session.getUser().isSuperuser())) { + authUser = (AuthenticatedUser) session.getUser(); + // initialize components, if any need it + } else { + return permissionsWrapper.notAuthorized(); + // redirect to login OR give some type of โ€˜you must be logged in' message + } + + FacesContext.getCurrentInstance().addMessage(null, + new FacesMessage(FacesMessage.SEVERITY_INFO, + BundleUtil.getStringFromBundle("dashboard.move.dataverse.message.summary"), + BundleUtil.getStringFromBundle("dashboard.move.dataverse.message.detail", Arrays.asList(settingsWrapper.getGuidesBaseUrl(), settingsWrapper.getGuidesVersion())))); + return null; + } + + public void move(){ + Dataverse dvSource = selectedSourceDataverse; + String srcAlias = dvSource != null ? dvSource.getAlias() : null; + + Dataverse target = selectedDestinationDataverse; + String dstAlias = target != null ? target.getAlias() : null; + + if (dvSource == null || target == null) { + // Move only works if both inputs are correct + // But if these inputs are required, we should never get here + // Since we never get here, we aren't bothering to move this English to the bundle. + FacesContext.getCurrentInstance().addMessage(null, + new FacesMessage("Please specify all fields")); + return; + } + + // construct arguments for message + List arguments = new ArrayList<>(); + arguments.add(dvSource != null ? dvSource.getName() : "-"); + arguments.add(target != null ? target.getName() : "-"); + + // copied logic from Dataverse API move + //Command requires Super user - it will be tested by the command + try { + HttpServletRequest httpServletRequest = (HttpServletRequest) FacesContext.getCurrentInstance().getExternalContext().getRequest(); + DataverseRequest dataverseRequest = new DataverseRequest(authUser, httpServletRequest); + commandEngine.submit(new MoveDataverseCommand( + dataverseRequest, dvSource, target, false + )); + + logger.info("Moved " + srcAlias + " to " + dstAlias); + + JsfHelper.addSuccessMessage(BundleUtil.getStringFromBundle("dashboard.move.dataverse.message.success", arguments)); + } + catch (CommandException e) { + logger.log(Level.SEVERE,"Unable to move "+ srcAlias + " to " + dstAlias, e); + arguments.add(e.getLocalizedMessage()); + FacesContext.getCurrentInstance().addMessage(null, + new FacesMessage(FacesMessage.SEVERITY_ERROR, + BundleUtil.getStringFromBundle("dashboard.move.dataverse.message.failure.summary"), + BundleUtil.getStringFromBundle("dashboard.move.dataverse.message.failure.details", arguments))); + } + } + + public String getDataverseCount() { + long count = dataverseService.getDataverseCount(); + return NumberFormat.getInstance().format(count); + } + +} diff --git a/src/main/java/edu/harvard/iq/dataverse/dataset/DatasetType.java b/src/main/java/edu/harvard/iq/dataverse/dataset/DatasetType.java index 78bf232e1a6..727703852eb 100644 --- a/src/main/java/edu/harvard/iq/dataverse/dataset/DatasetType.java +++ b/src/main/java/edu/harvard/iq/dataverse/dataset/DatasetType.java @@ -1,17 +1,23 @@ package edu.harvard.iq.dataverse.dataset; +import edu.harvard.iq.dataverse.MetadataBlock; import jakarta.json.Json; +import jakarta.json.JsonArrayBuilder; import jakarta.json.JsonObjectBuilder; +import jakarta.persistence.CascadeType; import jakarta.persistence.Column; import jakarta.persistence.Entity; import jakarta.persistence.GeneratedValue; import jakarta.persistence.GenerationType; import jakarta.persistence.Id; +import jakarta.persistence.ManyToMany; import jakarta.persistence.NamedQueries; import jakarta.persistence.NamedQuery; import jakarta.persistence.Table; import jakarta.persistence.UniqueConstraint; import java.io.Serializable; +import java.util.ArrayList; +import java.util.List; @NamedQueries({ @NamedQuery(name = "DatasetType.findAll", @@ -42,6 +48,12 @@ public class DatasetType implements Serializable { @Column(nullable = false) private String name; + /** + * The metadata blocks this dataset type is linked to. + */ + @ManyToMany(cascade = {CascadeType.MERGE}) + private List metadataBlocks = new ArrayList<>(); + public DatasetType() { } @@ -61,10 +73,23 @@ public void setName(String name) { this.name = name; } + public List getMetadataBlocks() { + return metadataBlocks; + } + + public void setMetadataBlocks(List metadataBlocks) { + this.metadataBlocks = metadataBlocks; + } + public JsonObjectBuilder toJson() { + JsonArrayBuilder linkedMetadataBlocks = Json.createArrayBuilder(); + for (MetadataBlock metadataBlock : this.getMetadataBlocks()) { + linkedMetadataBlocks.add(metadataBlock.getName()); + } return Json.createObjectBuilder() .add("id", getId()) - .add("name", getName()); + .add("name", getName()) + .add("linkedMetadataBlocks", linkedMetadataBlocks); } } diff --git a/src/main/java/edu/harvard/iq/dataverse/dataset/DatasetUtil.java b/src/main/java/edu/harvard/iq/dataverse/dataset/DatasetUtil.java index cacd409b365..012ba464ecc 100644 --- a/src/main/java/edu/harvard/iq/dataverse/dataset/DatasetUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/dataset/DatasetUtil.java @@ -530,7 +530,7 @@ public static String[] getDatasetSummaryFieldNames(String customFieldNames) { } else { summaryFieldNames = customFieldNames; } - return summaryFieldNames.split(","); + return summaryFieldNames.split("\\s*,\\s*"); } public static boolean isRsyncAppropriateStorageDriver(Dataset dataset){ diff --git a/src/main/java/edu/harvard/iq/dataverse/dataverse/featured/DataverseFeaturedItem.java b/src/main/java/edu/harvard/iq/dataverse/dataverse/featured/DataverseFeaturedItem.java new file mode 100644 index 00000000000..53d09516789 --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/dataverse/featured/DataverseFeaturedItem.java @@ -0,0 +1,88 @@ +package edu.harvard.iq.dataverse.dataverse.featured; + +import edu.harvard.iq.dataverse.Dataverse; +import edu.harvard.iq.dataverse.util.SystemConfig; +import jakarta.persistence.*; +import jakarta.validation.constraints.NotBlank; +import jakarta.validation.constraints.Min; +import jakarta.validation.constraints.Size; + +@NamedQueries({ + @NamedQuery(name = "DataverseFeaturedItem.deleteById", + query = "DELETE FROM DataverseFeaturedItem item WHERE item.id=:id"), + @NamedQuery(name = "DataverseFeaturedItem.findByDataverseOrderedByDisplayOrder", + query = "SELECT item FROM DataverseFeaturedItem item WHERE item.dataverse = :dataverse ORDER BY item.displayOrder ASC") +}) +@Entity +@Table(indexes = @Index(columnList = "displayOrder")) +public class DataverseFeaturedItem { + + public static final int MAX_FEATURED_ITEM_CONTENT_SIZE = 15000; + + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + private Long id; + + @ManyToOne + @JoinColumn(nullable = false) + private Dataverse dataverse; + + @NotBlank + @Size(max = MAX_FEATURED_ITEM_CONTENT_SIZE) + @Lob + @Column(columnDefinition = "TEXT", nullable = false) + private String content; + + @Min(0) + @Column(nullable = false) + private int displayOrder; + + private String imageFileName; + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public Dataverse getDataverse() { + return dataverse; + } + + public void setDataverse(Dataverse dataverse) { + this.dataverse = dataverse; + } + + public String getContent() { + return content; + } + + public void setContent(String content) { + this.content = content; + } + + public int getDisplayOrder() { + return displayOrder; + } + + public void setDisplayOrder(int displayOrder) { + this.displayOrder = displayOrder; + } + + public String getImageFileName() { + return imageFileName; + } + + public void setImageFileName(String imageFileName) { + this.imageFileName = imageFileName; + } + + public String getImageFileUrl() { + if (id != null && imageFileName != null) { + return SystemConfig.getDataverseSiteUrlStatic() + "/api/access/dataverseFeaturedItemImage/" + id; + } + return null; + } +} diff --git a/src/main/java/edu/harvard/iq/dataverse/dataverse/featured/DataverseFeaturedItemServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/dataverse/featured/DataverseFeaturedItemServiceBean.java new file mode 100644 index 00000000000..56cdaf5692e --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/dataverse/featured/DataverseFeaturedItemServiceBean.java @@ -0,0 +1,100 @@ +package edu.harvard.iq.dataverse.dataverse.featured; + +import edu.harvard.iq.dataverse.Dataverse; +import edu.harvard.iq.dataverse.settings.JvmSettings; +import edu.harvard.iq.dataverse.util.BundleUtil; +import edu.harvard.iq.dataverse.util.FileUtil; +import jakarta.ejb.Stateless; +import jakarta.inject.Named; +import jakarta.persistence.EntityManager; +import jakarta.persistence.PersistenceContext; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.Serializable; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.util.List; + +@Stateless +@Named +public class DataverseFeaturedItemServiceBean implements Serializable { + + public static class InvalidImageFileException extends Exception { + public InvalidImageFileException(String message) { + super(message); + } + } + + @PersistenceContext(unitName = "VDCNet-ejbPU") + private EntityManager em; + + public DataverseFeaturedItem findById(Long id) { + return em.find(DataverseFeaturedItem.class, id); + } + + public DataverseFeaturedItem save(DataverseFeaturedItem dataverseFeaturedItem) { + if (dataverseFeaturedItem.getId() == null) { + em.persist(dataverseFeaturedItem); + em.flush(); + } else { + dataverseFeaturedItem = em.merge(dataverseFeaturedItem); + } + return dataverseFeaturedItem; + } + + public void delete(Long id) { + em.createNamedQuery("DataverseFeaturedItem.deleteById", DataverseFeaturedItem.class) + .setParameter("id", id) + .executeUpdate(); + } + + public List findAllByDataverseOrdered(Dataverse dataverse) { + return em + .createNamedQuery("DataverseFeaturedItem.findByDataverseOrderedByDisplayOrder", DataverseFeaturedItem.class) + .setParameter("dataverse", dataverse) + .getResultList(); + } + + public InputStream getImageFileAsInputStream(DataverseFeaturedItem dataverseFeaturedItem) throws IOException { + Path imagePath = Path.of(JvmSettings.DOCROOT_DIRECTORY.lookup(), + JvmSettings.FEATURED_ITEMS_IMAGE_UPLOADS_DIRECTORY.lookup(), + dataverseFeaturedItem.getDataverse().getId().toString(), + dataverseFeaturedItem.getImageFileName()); + return Files.newInputStream(imagePath); + } + + public void saveDataverseFeaturedItemImageFile(InputStream inputStream, String imageFileName, Long dataverseId) throws IOException, InvalidImageFileException { + File tempFile = FileUtil.inputStreamToFile(inputStream); + validateImageFile(tempFile); + + Path imageDir = FileUtil.createDirStructure( + JvmSettings.DOCROOT_DIRECTORY.lookup(), + JvmSettings.FEATURED_ITEMS_IMAGE_UPLOADS_DIRECTORY.lookup(), + dataverseId.toString() + ); + File uploadedFile = new File(imageDir.toFile(), imageFileName); + + if (!uploadedFile.exists()) { + uploadedFile.createNewFile(); + } + + Files.copy(tempFile.toPath(), uploadedFile.toPath(), StandardCopyOption.REPLACE_EXISTING); + } + + private void validateImageFile(File file) throws IOException, InvalidImageFileException { + if (!FileUtil.isFileOfImageType(file)) { + throw new InvalidImageFileException( + BundleUtil.getStringFromBundle("dataverse.create.featuredItem.error.invalidFileType") + ); + } + Integer maxAllowedSize = JvmSettings.FEATURED_ITEMS_IMAGE_MAXSIZE.lookup(Integer.class); + if (file.length() > maxAllowedSize) { + throw new InvalidImageFileException( + BundleUtil.getStringFromBundle("dataverse.create.featuredItem.error.fileSizeExceedsLimit", List.of(maxAllowedSize.toString())) + ); + } + } +} diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/CommandContext.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/CommandContext.java index 282cbb88988..42f2616cd80 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/CommandContext.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/CommandContext.java @@ -1,28 +1,10 @@ package edu.harvard.iq.dataverse.engine.command; -import edu.harvard.iq.dataverse.DataFileServiceBean; -import edu.harvard.iq.dataverse.DatasetFieldServiceBean; -import edu.harvard.iq.dataverse.DatasetLinkingServiceBean; -import edu.harvard.iq.dataverse.DatasetServiceBean; -import edu.harvard.iq.dataverse.DatasetVersionServiceBean; -import edu.harvard.iq.dataverse.DataverseFacetServiceBean; -import edu.harvard.iq.dataverse.DataverseFieldTypeInputLevelServiceBean; -import edu.harvard.iq.dataverse.DataverseLinkingServiceBean; -import edu.harvard.iq.dataverse.DataverseRoleServiceBean; -import edu.harvard.iq.dataverse.DataverseServiceBean; +import edu.harvard.iq.dataverse.*; import edu.harvard.iq.dataverse.authorization.providers.builtin.BuiltinUserServiceBean; -import edu.harvard.iq.dataverse.DvObjectServiceBean; -import edu.harvard.iq.dataverse.FeaturedDataverseServiceBean; -import edu.harvard.iq.dataverse.FileDownloadServiceBean; -import edu.harvard.iq.dataverse.GuestbookResponseServiceBean; -import edu.harvard.iq.dataverse.GuestbookServiceBean; -import edu.harvard.iq.dataverse.MetadataBlockServiceBean; +import edu.harvard.iq.dataverse.dataverse.featured.DataverseFeaturedItemServiceBean; import edu.harvard.iq.dataverse.search.IndexServiceBean; -import edu.harvard.iq.dataverse.PermissionServiceBean; -import edu.harvard.iq.dataverse.RoleAssigneeServiceBean; import edu.harvard.iq.dataverse.search.SearchServiceBean; -import edu.harvard.iq.dataverse.TemplateServiceBean; -import edu.harvard.iq.dataverse.UserNotificationServiceBean; import edu.harvard.iq.dataverse.actionlogging.ActionLogServiceBean; import edu.harvard.iq.dataverse.authorization.AuthenticationServiceBean; import edu.harvard.iq.dataverse.authorization.groups.GroupServiceBean; @@ -152,4 +134,6 @@ public interface CommandContext { public void addCommand(Command command); public DatasetFieldServiceBean dsField(); + + public DataverseFeaturedItemServiceBean dataverseFeaturedItems(); } diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/exception/InvalidCommandArgumentsException.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/exception/InvalidCommandArgumentsException.java new file mode 100644 index 00000000000..95c6f52b880 --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/exception/InvalidCommandArgumentsException.java @@ -0,0 +1,25 @@ +package edu.harvard.iq.dataverse.engine.command.exception; + +import edu.harvard.iq.dataverse.engine.command.Command; + +/** + * Exception thrown when a {@link Command} is executed with invalid or malformed arguments. + *

+ * This exception typically indicates that the input parameters provided to the command + * do not meet the required criteria (e.g., missing fields, invalid formats, or other + * constraints). + *

+ *

+ * Example scenarios: + *

    + *
  • A required argument is null or missing.
    • + *
  • An argument is in an invalid format (e.g., a malformed email address).
    • + *
  • Arguments violate business rules or constraints.
    • + *
+ */ +public class InvalidCommandArgumentsException extends CommandException { + + public InvalidCommandArgumentsException(String message, Command aCommand) { + super(message, aCommand); + } +} diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/exception/InvalidFieldsCommandException.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/exception/InvalidFieldsCommandException.java new file mode 100644 index 00000000000..9bd1869f8a9 --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/exception/InvalidFieldsCommandException.java @@ -0,0 +1,42 @@ +package edu.harvard.iq.dataverse.engine.command.exception; + +import edu.harvard.iq.dataverse.engine.command.Command; +import java.util.Map; + +public class InvalidFieldsCommandException extends CommandException { + + private final Map fieldErrors; + + /** + * Constructs a new InvalidFieldsCommandException with the specified detail message, + * command, and a map of field errors. + * + * @param message The detail message. + * @param aCommand The command where the exception was encountered. + * @param fieldErrors A map containing the fields as keys and the reasons for their errors as values. + */ + public InvalidFieldsCommandException(String message, Command aCommand, Map fieldErrors) { + super(message, aCommand); + this.fieldErrors = fieldErrors; + } + + /** + * Gets the map of fields and their corresponding error messages. + * + * @return The map of field errors. + */ + public Map getFieldErrors() { + return fieldErrors; + } + + /** + * Returns a string representation of this exception, including the + * message and details of the invalid fields and their errors. + * + * @return A string representation of this exception. + */ + @Override + public String toString() { + return super.toString() + ", fieldErrors=" + fieldErrors; + } +} diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/exception/PermissionException.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/exception/PermissionException.java index a7881fc7b6e..2ca63c9c4aa 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/exception/PermissionException.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/exception/PermissionException.java @@ -3,6 +3,7 @@ import edu.harvard.iq.dataverse.DvObject; import edu.harvard.iq.dataverse.authorization.Permission; import edu.harvard.iq.dataverse.engine.command.Command; + import java.util.Set; /** @@ -12,22 +13,31 @@ * @author michael */ public class PermissionException extends CommandException { - - private final Set required; - private final DvObject dvObject; - - public PermissionException(String message, Command failedCommand, Set required, DvObject aDvObject ) { - super(message, failedCommand); - this.required = required; - dvObject = aDvObject; - } - - public Set getRequiredPermissions() { - return required; - } - - public DvObject getDvObject() { - return dvObject; - } - + + private final Set required; + private final DvObject dvObject; + private final boolean isDetailedMessageRequired; + + public PermissionException(String message, Command failedCommand, Set required, DvObject dvObject, boolean isDetailedMessageRequired) { + super(message, failedCommand); + this.required = required; + this.dvObject = dvObject; + this.isDetailedMessageRequired = isDetailedMessageRequired; + } + + public PermissionException(String message, Command failedCommand, Set required, DvObject dvObject) { + this(message, failedCommand, required, dvObject, false); + } + + public Set getRequiredPermissions() { + return required; + } + + public DvObject getDvObject() { + return dvObject; + } + + public boolean isDetailedMessageRequired() { + return isDetailedMessageRequired; + } } diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/AbstractWriteDataverseCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/AbstractWriteDataverseCommand.java index 91f3a5b823c..8227572da3b 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/AbstractWriteDataverseCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/AbstractWriteDataverseCommand.java @@ -19,15 +19,13 @@ abstract class AbstractWriteDataverseCommand extends AbstractCommand private final List inputLevels; private final List facets; protected final List metadataBlocks; - private final boolean resetRelationsOnNullValues; public AbstractWriteDataverseCommand(Dataverse dataverse, Dataverse affectedDataverse, DataverseRequest request, List facets, List inputLevels, - List metadataBlocks, - boolean resetRelationsOnNullValues) { + List metadataBlocks) { super(request, affectedDataverse); this.dataverse = dataverse; if (facets != null) { @@ -45,7 +43,6 @@ public AbstractWriteDataverseCommand(Dataverse dataverse, } else { this.metadataBlocks = null; } - this.resetRelationsOnNullValues = resetRelationsOnNullValues; } @Override @@ -59,46 +56,61 @@ public Dataverse execute(CommandContext ctxt) throws CommandException { return ctxt.dataverses().save(dataverse); } + /* + metadataBlocks = null - ignore + metadataBlocks is empty - delete and inherit from parent + metadataBlocks is not empty - set with new updated values + */ private void processMetadataBlocks() { - if (metadataBlocks != null && !metadataBlocks.isEmpty()) { - dataverse.setMetadataBlockRoot(true); - dataverse.setMetadataBlocks(metadataBlocks); - } else if (resetRelationsOnNullValues) { - dataverse.setMetadataBlockRoot(false); - dataverse.clearMetadataBlocks(); + if (metadataBlocks != null) { + if (metadataBlocks.isEmpty()) { + dataverse.setMetadataBlockRoot(false); + dataverse.clearMetadataBlocks(); + } else { + dataverse.setMetadataBlockRoot(true); + dataverse.setMetadataBlocks(metadataBlocks); + } } } + /* + facets = null - ignore + facets is empty - delete and inherit from parent + facets is not empty - set with new updated values + */ private void processFacets(CommandContext ctxt) { if (facets != null) { - ctxt.facets().deleteFacetsFor(dataverse); - dataverse.setDataverseFacets(new ArrayList<>()); - - if (!facets.isEmpty()) { + if (facets.isEmpty()) { + ctxt.facets().deleteFacetsFor(dataverse); + dataverse.setFacetRoot(false); + } else { + ctxt.facets().deleteFacetsFor(dataverse); + dataverse.setDataverseFacets(new ArrayList<>()); dataverse.setFacetRoot(true); + for (int i = 0; i < facets.size(); i++) { + ctxt.facets().create(i, facets.get(i), dataverse); + } } - - for (int i = 0; i < facets.size(); i++) { - ctxt.facets().create(i, facets.get(i), dataverse); - } - } else if (resetRelationsOnNullValues) { - ctxt.facets().deleteFacetsFor(dataverse); - dataverse.setFacetRoot(false); } } + /* + inputLevels = null - ignore + inputLevels is empty - delete + inputLevels is not empty - set with new updated values + */ private void processInputLevels(CommandContext ctxt) { if (inputLevels != null) { - if (!inputLevels.isEmpty()) { + if (inputLevels.isEmpty()) { + ctxt.fieldTypeInputLevels().deleteDataverseFieldTypeInputLevelFor(dataverse); + } else { dataverse.addInputLevelsMetadataBlocksIfNotPresent(inputLevels); + ctxt.fieldTypeInputLevels().deleteDataverseFieldTypeInputLevelFor(dataverse); + inputLevels.forEach(inputLevel -> { + inputLevel.setDataverse(dataverse); + ctxt.fieldTypeInputLevels().create(inputLevel); + }); } - ctxt.fieldTypeInputLevels().deleteFacetsFor(dataverse); - inputLevels.forEach(inputLevel -> { - inputLevel.setDataverse(dataverse); - ctxt.fieldTypeInputLevels().create(inputLevel); - }); - } else if (resetRelationsOnNullValues) { - ctxt.fieldTypeInputLevels().deleteFacetsFor(dataverse); } } diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/AbstractWriteDataverseFeaturedItemCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/AbstractWriteDataverseFeaturedItemCommand.java new file mode 100644 index 00000000000..8c4a8281345 --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/AbstractWriteDataverseFeaturedItemCommand.java @@ -0,0 +1,77 @@ +package edu.harvard.iq.dataverse.engine.command.impl; + +import edu.harvard.iq.dataverse.*; +import edu.harvard.iq.dataverse.authorization.Permission; +import edu.harvard.iq.dataverse.dataverse.featured.DataverseFeaturedItem; +import edu.harvard.iq.dataverse.dataverse.featured.DataverseFeaturedItemServiceBean; +import edu.harvard.iq.dataverse.engine.command.AbstractCommand; +import edu.harvard.iq.dataverse.engine.command.CommandContext; +import edu.harvard.iq.dataverse.engine.command.DataverseRequest; +import edu.harvard.iq.dataverse.engine.command.RequiredPermissions; +import edu.harvard.iq.dataverse.engine.command.exception.CommandException; +import edu.harvard.iq.dataverse.engine.command.exception.InvalidCommandArgumentsException; +import edu.harvard.iq.dataverse.util.BundleUtil; +import edu.harvard.iq.dataverse.util.MarkupChecker; + +import java.io.IOException; +import java.io.InputStream; +import java.text.MessageFormat; +import java.util.List; + +/** + * An abstract base class for commands that perform write operations on {@link DataverseFeaturedItem}s. + */ +@RequiredPermissions({Permission.EditDataverse}) +abstract class AbstractWriteDataverseFeaturedItemCommand extends AbstractCommand { + + protected final Dataverse dataverse; + + public AbstractWriteDataverseFeaturedItemCommand(DataverseRequest request, Dataverse affectedDataverse) { + super(request, affectedDataverse); + this.dataverse = affectedDataverse; + } + + protected void validateAndSetContent(DataverseFeaturedItem featuredItem, String content) throws InvalidCommandArgumentsException { + if (content == null || content.trim().isEmpty()) { + throw new InvalidCommandArgumentsException( + BundleUtil.getStringFromBundle("dataverse.create.featuredItem.error.contentShouldBeProvided"), + this + ); + } + content = MarkupChecker.sanitizeAdvancedHTML(content); + if (content.length() > DataverseFeaturedItem.MAX_FEATURED_ITEM_CONTENT_SIZE) { + throw new InvalidCommandArgumentsException( + MessageFormat.format( + BundleUtil.getStringFromBundle("dataverse.create.featuredItem.error.contentExceedsLengthLimit"), + List.of(DataverseFeaturedItem.MAX_FEATURED_ITEM_CONTENT_SIZE) + ), + this + ); + } + featuredItem.setContent(content); + } + + protected void setFileImageIfAvailableOrNull(DataverseFeaturedItem featuredItem, String imageFileName, InputStream imageFileInputStream, CommandContext ctxt) throws CommandException { + if (imageFileName != null && imageFileInputStream != null) { + try { + ctxt.dataverseFeaturedItems().saveDataverseFeaturedItemImageFile(imageFileInputStream, imageFileName, dataverse.getId()); + } catch (DataverseFeaturedItemServiceBean.InvalidImageFileException e) { + throw new InvalidCommandArgumentsException( + e.getMessage(), + this + ); + } catch (IOException e) { + throw new CommandException( + BundleUtil.getStringFromBundle( + "dataverse.create.featuredItem.error.imageFileProcessing", + List.of(e.getMessage()) + ), + this + ); + } + featuredItem.setImageFileName(imageFileName); + } else { + featuredItem.setImageFileName(null); + } + } +} diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CheckRateLimitForDatasetFeedbackCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CheckRateLimitForDatasetFeedbackCommand.java new file mode 100644 index 00000000000..d25dbd974c2 --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CheckRateLimitForDatasetFeedbackCommand.java @@ -0,0 +1,18 @@ +package edu.harvard.iq.dataverse.engine.command.impl; + +import edu.harvard.iq.dataverse.DvObject; +import edu.harvard.iq.dataverse.engine.command.AbstractVoidCommand; +import edu.harvard.iq.dataverse.engine.command.CommandContext; +import edu.harvard.iq.dataverse.engine.command.DataverseRequest; +import edu.harvard.iq.dataverse.engine.command.exception.CommandException; + +public class CheckRateLimitForDatasetFeedbackCommand extends AbstractVoidCommand { + + public CheckRateLimitForDatasetFeedbackCommand(DataverseRequest aRequest, DvObject dvObject) { + super(aRequest, dvObject); + } + + @Override + protected void executeImpl(CommandContext ctxt) throws CommandException { } +} + diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateDataverseCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateDataverseCommand.java index 3728f3ee6ce..145cfb6199c 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateDataverseCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateDataverseCommand.java @@ -39,7 +39,7 @@ public CreateDataverseCommand(Dataverse created, List facets, List inputLevels, List metadataBlocks) { - super(created, created.getOwner(), request, facets, inputLevels, metadataBlocks, false); + super(created, created.getOwner(), request, facets, inputLevels, metadataBlocks); } @Override diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateDataverseFeaturedItemCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateDataverseFeaturedItemCommand.java new file mode 100644 index 00000000000..24732d05c8b --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateDataverseFeaturedItemCommand.java @@ -0,0 +1,42 @@ +package edu.harvard.iq.dataverse.engine.command.impl; + +import edu.harvard.iq.dataverse.Dataverse; +import edu.harvard.iq.dataverse.dataverse.featured.DataverseFeaturedItem; +import edu.harvard.iq.dataverse.api.dto.NewDataverseFeaturedItemDTO; +import edu.harvard.iq.dataverse.engine.command.CommandContext; +import edu.harvard.iq.dataverse.engine.command.DataverseRequest; +import edu.harvard.iq.dataverse.engine.command.exception.CommandException; + +/** + * Creates a featured item {@link DataverseFeaturedItem} for a {@link Dataverse}. + */ +public class CreateDataverseFeaturedItemCommand extends AbstractWriteDataverseFeaturedItemCommand { + + private final NewDataverseFeaturedItemDTO newDataverseFeaturedItemDTO; + + public CreateDataverseFeaturedItemCommand(DataverseRequest request, + Dataverse dataverse, + NewDataverseFeaturedItemDTO newDataverseFeaturedItemDTO) { + super(request, dataverse); + this.newDataverseFeaturedItemDTO = newDataverseFeaturedItemDTO; + } + + @Override + public DataverseFeaturedItem execute(CommandContext ctxt) throws CommandException { + DataverseFeaturedItem dataverseFeaturedItem = new DataverseFeaturedItem(); + + validateAndSetContent(dataverseFeaturedItem, newDataverseFeaturedItemDTO.getContent()); + dataverseFeaturedItem.setDisplayOrder(newDataverseFeaturedItemDTO.getDisplayOrder()); + + setFileImageIfAvailableOrNull( + dataverseFeaturedItem, + newDataverseFeaturedItemDTO.getImageFileName(), + newDataverseFeaturedItemDTO.getImageFileInputStream(), + ctxt + ); + + dataverseFeaturedItem.setDataverse(dataverse); + + return ctxt.dataverseFeaturedItems().save(dataverseFeaturedItem); + } +} diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateNewDataFilesCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateNewDataFilesCommand.java index e9a2025b112..e4130b534b3 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateNewDataFilesCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateNewDataFilesCommand.java @@ -53,7 +53,7 @@ import static edu.harvard.iq.dataverse.util.FileUtil.MIME_TYPE_UNDETERMINED_DEFAULT; import static edu.harvard.iq.dataverse.util.FileUtil.createIngestFailureReport; import static edu.harvard.iq.dataverse.util.FileUtil.determineFileType; -import static edu.harvard.iq.dataverse.util.FileUtil.determineFileTypeByNameAndExtension; +import static edu.harvard.iq.dataverse.util.FileUtil.determineRemoteFileType; import static edu.harvard.iq.dataverse.util.FileUtil.getFilesTempDirectory; import static edu.harvard.iq.dataverse.util.FileUtil.saveInputStreamInTempFile; import static edu.harvard.iq.dataverse.util.FileUtil.useRecognizedType; @@ -574,6 +574,8 @@ public CreateDataFileResult execute(CommandContext ctxt) throws CommandException } else { // Direct upload. + finalType = StringUtils.isBlank(suppliedContentType) ? FileUtil.MIME_TYPE_UNDETERMINED_DEFAULT : suppliedContentType; + // Since this is a direct upload, and therefore no temp file associated // with it, we may, OR MAY NOT know the size of the file. If this is // a direct upload via the UI, the page must have already looked up @@ -593,18 +595,6 @@ public CreateDataFileResult execute(CommandContext ctxt) throws CommandException } } - // Default to suppliedContentType if set or the overall undetermined default if a contenttype isn't supplied - finalType = StringUtils.isBlank(suppliedContentType) ? FileUtil.MIME_TYPE_UNDETERMINED_DEFAULT : suppliedContentType; - String type = determineFileTypeByNameAndExtension(fileName); - if (!StringUtils.isBlank(type)) { - //Use rules for deciding when to trust browser supplied type - if (useRecognizedType(finalType, type)) { - finalType = type; - } - logger.fine("Supplied type: " + suppliedContentType + ", finalType: " + finalType); - } - - } // Finally, if none of the special cases above were applicable (or @@ -635,6 +625,30 @@ public CreateDataFileResult execute(CommandContext ctxt) throws CommandException DataFile datafile = FileUtil.createSingleDataFile(version, newFile, newStorageIdentifier, fileName, finalType, newCheckSumType, newCheckSum); if (datafile != null) { + if (newStorageIdentifier != null) { + // Direct upload case + // Improve the MIMEType + // Need the owner for the StorageIO class to get the file/S3 path from the + // storageIdentifier + // Currently owner is null, but using this flag will avoid making changes here + // if that isn't true in the future + boolean ownerSet = datafile.getOwner() != null; + if (!ownerSet) { + datafile.setOwner(version.getDataset()); + } + String type = determineRemoteFileType(datafile, fileName); + if (!StringUtils.isBlank(type)) { + // Use rules for deciding when to trust browser supplied type + if (useRecognizedType(finalType, type)) { + datafile.setContentType(type); + } + logger.fine("Supplied type: " + suppliedContentType + ", finalType: " + finalType); + } + // Avoid changing + if (!ownerSet) { + datafile.setOwner(null); + } + } if (warningMessage != null) { createIngestFailureReport(datafile, warningMessage); diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateRoleCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateRoleCommand.java index 8cffcd3d821..4a897adefa2 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateRoleCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateRoleCommand.java @@ -22,12 +22,12 @@ @RequiredPermissions(Permission.ManageDataversePermissions) public class CreateRoleCommand extends AbstractCommand { - private final DataverseRole created; + private final DataverseRole role; private final Dataverse dv; public CreateRoleCommand(DataverseRole aRole, DataverseRequest aRequest, Dataverse anAffectedDataverse) { super(aRequest, anAffectedDataverse); - created = aRole; + role = aRole; dv = anAffectedDataverse; } @@ -41,16 +41,16 @@ public DataverseRole execute(CommandContext ctxt) throws CommandException { //Test to see if the role already exists in DB try { DataverseRole testRole = ctxt.em().createNamedQuery("DataverseRole.findDataverseRoleByAlias", DataverseRole.class) - .setParameter("alias", created.getAlias()) + .setParameter("alias", role.getAlias()) .getSingleResult(); - if (!(testRole == null)) { + if (testRole != null && !testRole.getId().equals(role.getId())) { throw new IllegalCommandException(BundleUtil.getStringFromBundle("permission.role.not.created.alias.already.exists"), this); } } catch (NoResultException nre) { - // we want no results because that meand we can create a role + // we want no results because that meant we can create a role } - dv.addRole(created); - return ctxt.roles().save(created); + dv.addRole(role); + return ctxt.roles().save(role); } } diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CuratePublishedDatasetVersionCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CuratePublishedDatasetVersionCommand.java index e378e2e2ef7..3629432b7e4 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CuratePublishedDatasetVersionCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CuratePublishedDatasetVersionCommand.java @@ -72,6 +72,10 @@ public Dataset execute(CommandContext ctxt) throws CommandException { TermsOfUseAndAccess newTerms = newVersion.getTermsOfUseAndAccess(); newTerms.setDatasetVersion(updateVersion); updateVersion.setTermsOfUseAndAccess(newTerms); + + //Creation Note + updateVersion.setVersionNote(newVersion.getVersionNote()); + // Clear unnecessary terms relationships .... newVersion.setTermsOfUseAndAccess(null); oldTerms.setDatasetVersion(null); diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/DeleteDataverseCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/DeleteDataverseCommand.java index c7c592f9458..84a0ab0f3f2 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/DeleteDataverseCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/DeleteDataverseCommand.java @@ -1,6 +1,7 @@ package edu.harvard.iq.dataverse.engine.command.impl; import edu.harvard.iq.dataverse.Dataverse; +import edu.harvard.iq.dataverse.dataverse.featured.DataverseFeaturedItem; import edu.harvard.iq.dataverse.DataverseFieldTypeInputLevel; import edu.harvard.iq.dataverse.authorization.DataverseRole; import edu.harvard.iq.dataverse.RoleAssignment; @@ -78,6 +79,14 @@ protected void executeImpl(CommandContext ctxt) throws CommandException { ctxt.em().remove(merged); } doomed.setDataverseFieldTypeInputLevels(new ArrayList<>()); + + // Featured Items + for (DataverseFeaturedItem featuredItem : doomed.getDataverseFeaturedItems()) { + DataverseFeaturedItem merged = ctxt.em().merge(featuredItem); + ctxt.em().remove(merged); + } + doomed.setDataverseFeaturedItems(new ArrayList<>()); + // DATAVERSE Dataverse doomedAndMerged = ctxt.em().merge(doomed); ctxt.em().remove(doomedAndMerged); diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/DeleteDataverseFeaturedItemCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/DeleteDataverseFeaturedItemCommand.java new file mode 100644 index 00000000000..215863a44da --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/DeleteDataverseFeaturedItemCommand.java @@ -0,0 +1,26 @@ +package edu.harvard.iq.dataverse.engine.command.impl; + +import edu.harvard.iq.dataverse.Dataverse; +import edu.harvard.iq.dataverse.dataverse.featured.DataverseFeaturedItem; +import edu.harvard.iq.dataverse.authorization.Permission; +import edu.harvard.iq.dataverse.engine.command.*; +import edu.harvard.iq.dataverse.engine.command.exception.CommandException; + +/** + * Deletes a particular featured item {@link DataverseFeaturedItem} of a {@link Dataverse}. + */ +@RequiredPermissions({Permission.EditDataverse}) +public class DeleteDataverseFeaturedItemCommand extends AbstractVoidCommand { + + private final DataverseFeaturedItem doomed; + + public DeleteDataverseFeaturedItemCommand(DataverseRequest request, DataverseFeaturedItem doomed) { + super(request, doomed.getDataverse()); + this.doomed = doomed; + } + + @Override + protected void executeImpl(CommandContext ctxt) throws CommandException { + ctxt.dataverseFeaturedItems().delete(doomed.getId()); + } +} diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetDataverseFeaturedItemCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetDataverseFeaturedItemCommand.java new file mode 100644 index 00000000000..c594887b6ed --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetDataverseFeaturedItemCommand.java @@ -0,0 +1,37 @@ +package edu.harvard.iq.dataverse.engine.command.impl; + +import edu.harvard.iq.dataverse.dataverse.featured.DataverseFeaturedItem; +import edu.harvard.iq.dataverse.authorization.Permission; +import edu.harvard.iq.dataverse.engine.command.AbstractCommand; +import edu.harvard.iq.dataverse.engine.command.CommandContext; +import edu.harvard.iq.dataverse.engine.command.DataverseRequest; +import edu.harvard.iq.dataverse.engine.command.exception.CommandException; + +import java.util.Collections; +import java.util.Map; +import java.util.Set; + +/** + * Retrieves a particular featured item {@link DataverseFeaturedItem}. + */ +public class GetDataverseFeaturedItemCommand extends AbstractCommand { + + private final DataverseFeaturedItem dataverseFeaturedItem; + + public GetDataverseFeaturedItemCommand(DataverseRequest request, DataverseFeaturedItem dataverseFeaturedItem) { + super(request, dataverseFeaturedItem.getDataverse()); + this.dataverseFeaturedItem = dataverseFeaturedItem; + } + + @Override + public DataverseFeaturedItem execute(CommandContext ctxt) throws CommandException { + return dataverseFeaturedItem; + } + + @Override + public Map> getRequiredPermissions() { + return Collections.singletonMap("", + dataverseFeaturedItem.getDataverse().isReleased() ? Collections.emptySet() + : Collections.singleton(Permission.ViewUnpublishedDataverse)); + } +} diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetUserPermittedCollectionsCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetUserPermittedCollectionsCommand.java new file mode 100644 index 00000000000..c4888c8c99c --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetUserPermittedCollectionsCommand.java @@ -0,0 +1,61 @@ +package edu.harvard.iq.dataverse.engine.command.impl; + +import edu.harvard.iq.dataverse.Dataverse; +import edu.harvard.iq.dataverse.DvObject; +import edu.harvard.iq.dataverse.authorization.Permission; +import edu.harvard.iq.dataverse.authorization.groups.impl.ipaddress.ip.IpAddress; +import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser; +import edu.harvard.iq.dataverse.engine.command.AbstractCommand; +import edu.harvard.iq.dataverse.engine.command.CommandContext; +import edu.harvard.iq.dataverse.engine.command.DataverseRequest; +import edu.harvard.iq.dataverse.engine.command.RequiredPermissions; +import edu.harvard.iq.dataverse.engine.command.exception.CommandException; +import jakarta.json.Json; +import jakarta.json.JsonArrayBuilder; +import jakarta.json.JsonObjectBuilder; + +import java.util.List; +import java.util.logging.Logger; + +import static edu.harvard.iq.dataverse.util.json.JsonPrinter.json; + +@RequiredPermissions({}) +public class GetUserPermittedCollectionsCommand extends AbstractCommand { + private static final Logger logger = Logger.getLogger(GetUserPermittedCollectionsCommand.class.getCanonicalName()); + + private DataverseRequest request; + private AuthenticatedUser user; + private String permission; + public GetUserPermittedCollectionsCommand(DataverseRequest request, AuthenticatedUser user, String permission) { + super(request, (DvObject) null); + this.request = request; + this.user = user; + this.permission = permission; + } + + @Override + public JsonObjectBuilder execute(CommandContext ctxt) throws CommandException { + if (user == null) { + throw new CommandException("User not found.", this); + } + int permissionBit; + try { + permissionBit = permission.equalsIgnoreCase("any") ? + Integer.MAX_VALUE : (1 << Permission.valueOf(permission).ordinal()); + } catch (IllegalArgumentException e) { + throw new CommandException("Permission not valid.", this); + } + List collections = ctxt.permissions().findPermittedCollections(request, user, permissionBit); + if (collections != null) { + JsonObjectBuilder job = Json.createObjectBuilder(); + JsonArrayBuilder jab = Json.createArrayBuilder(); + for (Dataverse dv : collections) { + jab.add(json(dv)); + } + job.add("count", collections.size()); + job.add("items", jab); + return job; + } + return null; + } +} diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/ListDataverseFeaturedItemsCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/ListDataverseFeaturedItemsCommand.java new file mode 100644 index 00000000000..0d4051fc7d5 --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/ListDataverseFeaturedItemsCommand.java @@ -0,0 +1,36 @@ +package edu.harvard.iq.dataverse.engine.command.impl; + +import edu.harvard.iq.dataverse.Dataverse; +import edu.harvard.iq.dataverse.dataverse.featured.DataverseFeaturedItem; +import edu.harvard.iq.dataverse.authorization.Permission; +import edu.harvard.iq.dataverse.engine.command.AbstractCommand; +import edu.harvard.iq.dataverse.engine.command.CommandContext; +import edu.harvard.iq.dataverse.engine.command.DataverseRequest; +import edu.harvard.iq.dataverse.engine.command.exception.CommandException; + +import java.util.*; + +/** + * Lists the featured items {@link DataverseFeaturedItem} of a {@link Dataverse}. + */ +public class ListDataverseFeaturedItemsCommand extends AbstractCommand> { + + private final Dataverse dataverse; + + public ListDataverseFeaturedItemsCommand(DataverseRequest request, Dataverse dataverse) { + super(request, dataverse); + this.dataverse = dataverse; + } + + @Override + public List execute(CommandContext ctxt) throws CommandException { + return ctxt.dataverseFeaturedItems().findAllByDataverseOrdered(dataverse); + } + + @Override + public Map> getRequiredPermissions() { + return Collections.singletonMap("", + dataverse.isReleased() ? Collections.emptySet() + : Collections.singleton(Permission.ViewUnpublishedDataverse)); + } +} diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/ListMetadataBlocksCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/ListMetadataBlocksCommand.java index 8275533ced2..e79d36de07d 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/ListMetadataBlocksCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/ListMetadataBlocksCommand.java @@ -3,15 +3,18 @@ import edu.harvard.iq.dataverse.Dataverse; import edu.harvard.iq.dataverse.MetadataBlock; import edu.harvard.iq.dataverse.authorization.Permission; +import edu.harvard.iq.dataverse.dataset.DatasetType; import edu.harvard.iq.dataverse.engine.command.AbstractCommand; import edu.harvard.iq.dataverse.engine.command.CommandContext; import edu.harvard.iq.dataverse.engine.command.DataverseRequest; import edu.harvard.iq.dataverse.engine.command.exception.CommandException; +import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.stream.Stream; /** * Lists the metadata blocks of a {@link Dataverse}. @@ -23,11 +26,13 @@ public class ListMetadataBlocksCommand extends AbstractCommand execute(CommandContext ctxt) throws CommandException if (onlyDisplayedOnCreate) { return listMetadataBlocksDisplayedOnCreate(ctxt, dataverse); } - return dataverse.getMetadataBlocks(); + List orig = dataverse.getMetadataBlocks(); + List extraFromDatasetTypes = new ArrayList<>(); + if (datasetType != null) { + extraFromDatasetTypes = datasetType.getMetadataBlocks(); + } + return Stream.concat(orig.stream(), extraFromDatasetTypes.stream()).toList(); } private List listMetadataBlocksDisplayedOnCreate(CommandContext ctxt, Dataverse dataverse) { if (dataverse.isMetadataBlockRoot() || dataverse.getOwner() == null) { - return ctxt.metadataBlocks().listMetadataBlocksDisplayedOnCreate(dataverse); + List orig = ctxt.metadataBlocks().listMetadataBlocksDisplayedOnCreate(dataverse); + List extraFromDatasetTypes = new ArrayList<>(); + if (datasetType != null) { + extraFromDatasetTypes = datasetType.getMetadataBlocks(); + } + return Stream.concat(orig.stream(), extraFromDatasetTypes.stream()).toList(); } return listMetadataBlocksDisplayedOnCreate(ctxt, dataverse.getOwner()); } diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/MoveDatasetCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/MoveDatasetCommand.java index bee5dc648b9..1c3a62ec6de 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/MoveDatasetCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/MoveDatasetCommand.java @@ -10,7 +10,6 @@ import edu.harvard.iq.dataverse.DatasetLock; import edu.harvard.iq.dataverse.Dataverse; import edu.harvard.iq.dataverse.Guestbook; -import edu.harvard.iq.dataverse.authorization.DataverseRole; import edu.harvard.iq.dataverse.authorization.Permission; import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser; import edu.harvard.iq.dataverse.engine.command.AbstractVoidCommand; @@ -27,7 +26,6 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; -import java.util.logging.Level; import java.util.logging.Logger; /** @@ -72,13 +70,13 @@ public void executeImpl(CommandContext ctxt) throws CommandException { // validate the move makes sense if (moved.getOwner().equals(destination)) { - throw new IllegalCommandException(BundleUtil.getStringFromBundle("dashboard.card.datamove.dataset.command.error.targetDataverseSameAsOriginalDataverse"), this); + throw new IllegalCommandException(BundleUtil.getStringFromBundle("dashboard.move.dataset.command.error.targetDataverseSameAsOriginalDataverse"), this); } // if dataset is published make sure that its target is published if (moved.isReleased() && !destination.isReleased()){ - throw new IllegalCommandException(BundleUtil.getStringFromBundle("dashboard.card.datamove.dataset.command.error.targetDataverseUnpublishedDatasetPublished", Arrays.asList(destination.getDisplayName())), this); + throw new IllegalCommandException(BundleUtil.getStringFromBundle("dashboard.move.dataset.command.error.targetDataverseUnpublishedDatasetPublished", Arrays.asList(destination.getDisplayName())), this); } //if the datasets guestbook is not contained in the new dataverse then remove it @@ -130,10 +128,10 @@ public void executeImpl(CommandContext ctxt) throws CommandException { if (removeGuestbook || removeLinkDs) { StringBuilder errorString = new StringBuilder(); if (removeGuestbook) { - errorString.append(BundleUtil.getStringFromBundle("dashboard.card.datamove.dataset.command.error.unforced.datasetGuestbookNotInTargetDataverse")); + errorString.append(BundleUtil.getStringFromBundle("dashboard.move.dataset.command.error.unforced.datasetGuestbookNotInTargetDataverse")); } if (removeLinkDs) { - errorString.append(BundleUtil.getStringFromBundle("dashboard.card.datamove.dataset.command.error.unforced.linkedToTargetDataverseOrOneOfItsParents")); + errorString.append(BundleUtil.getStringFromBundle("dashboard.move.dataset.command.error.unforced.linkedToTargetDataverseOrOneOfItsParents")); } throw new UnforcedCommandException(errorString.toString(), this); } diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/MoveDataverseCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/MoveDataverseCommand.java index ea38f5a7af7..c8b59b1818a 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/MoveDataverseCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/MoveDataverseCommand.java @@ -10,7 +10,6 @@ import edu.harvard.iq.dataverse.Template; import edu.harvard.iq.dataverse.authorization.Permission; import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser; -import edu.harvard.iq.dataverse.batch.util.LoggingUtil; import edu.harvard.iq.dataverse.engine.command.AbstractVoidCommand; import edu.harvard.iq.dataverse.engine.command.CommandContext; import edu.harvard.iq.dataverse.engine.command.DataverseRequest; @@ -20,14 +19,12 @@ import edu.harvard.iq.dataverse.engine.command.exception.IllegalCommandException; import edu.harvard.iq.dataverse.engine.command.exception.PermissionException; import edu.harvard.iq.dataverse.util.BundleUtil; -import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.Iterator; import java.util.List; import java.util.logging.Logger; -import org.apache.solr.client.solrj.SolrServerException; /** * A command to move a {@link Dataverse} between two {@link Dataverse}s. diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/PublishDatasetCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/PublishDatasetCommand.java index 902bea7f833..915ef6ea2a1 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/PublishDatasetCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/PublishDatasetCommand.java @@ -2,6 +2,7 @@ import edu.harvard.iq.dataverse.Dataset; import edu.harvard.iq.dataverse.DatasetLock; +import edu.harvard.iq.dataverse.Dataverse; import edu.harvard.iq.dataverse.authorization.Permission; import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser; import edu.harvard.iq.dataverse.engine.command.CommandContext; @@ -9,6 +10,7 @@ import edu.harvard.iq.dataverse.engine.command.RequiredPermissions; import edu.harvard.iq.dataverse.engine.command.exception.CommandException; import edu.harvard.iq.dataverse.engine.command.exception.IllegalCommandException; +import edu.harvard.iq.dataverse.util.BundleUtil; import edu.harvard.iq.dataverse.workflow.Workflow; import edu.harvard.iq.dataverse.workflow.WorkflowContext.TriggerType; @@ -231,9 +233,20 @@ private void verifyCommandArguments(CommandContext ctxt) throws IllegalCommandEx if (minorRelease && !getDataset().getLatestVersion().isMinorUpdate()) { throw new IllegalCommandException("Cannot release as minor version. Re-try as major release.", this); } + + if (getDataset().getFiles().isEmpty() && getEffectiveRequiresFilesToPublishDataset()) { + throw new IllegalCommandException(BundleUtil.getStringFromBundle("dataset.mayNotPublish.FilesRequired"), this); + } + } + } + private boolean getEffectiveRequiresFilesToPublishDataset() { + if (getUser().isSuperuser()) { + return false; + } else { + Dataverse dv = getDataset().getOwner(); + return dv != null && dv.getEffectiveRequiresFilesToPublishDataset(); } } - @Override public boolean onSuccess(CommandContext ctxt, Object r) { diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/RegisterOIDCUserCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/RegisterOIDCUserCommand.java new file mode 100644 index 00000000000..c7745c75aa9 --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/RegisterOIDCUserCommand.java @@ -0,0 +1,204 @@ +package edu.harvard.iq.dataverse.engine.command.impl; + +import edu.harvard.iq.dataverse.DvObject; +import edu.harvard.iq.dataverse.api.dto.UserDTO; +import edu.harvard.iq.dataverse.authorization.AuthenticatedUserDisplayInfo; +import edu.harvard.iq.dataverse.authorization.UserRecordIdentifier; +import edu.harvard.iq.dataverse.authorization.exceptions.AuthorizationException; +import edu.harvard.iq.dataverse.authorization.providers.oauth2.OAuth2UserRecord; +import edu.harvard.iq.dataverse.engine.command.*; +import edu.harvard.iq.dataverse.engine.command.exception.CommandException; +import edu.harvard.iq.dataverse.engine.command.exception.IllegalCommandException; +import edu.harvard.iq.dataverse.engine.command.exception.PermissionException; +import edu.harvard.iq.dataverse.engine.command.exception.InvalidFieldsCommandException; +import edu.harvard.iq.dataverse.settings.FeatureFlags; +import edu.harvard.iq.dataverse.util.BundleUtil; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +@RequiredPermissions({}) +public class RegisterOIDCUserCommand extends AbstractVoidCommand { + + private static final String FIELD_USERNAME = "username"; + private static final String FIELD_FIRST_NAME = "firstName"; + private static final String FIELD_LAST_NAME = "lastName"; + private static final String FIELD_EMAIL_ADDRESS = "emailAddress"; + private static final String FIELD_TERMS_ACCEPTED = "termsAccepted"; + + private final String bearerToken; + private final UserDTO userDTO; + + public RegisterOIDCUserCommand(DataverseRequest aRequest, String bearerToken, UserDTO userDTO) { + super(aRequest, (DvObject) null); + this.bearerToken = bearerToken; + this.userDTO = userDTO; + } + + @Override + protected void executeImpl(CommandContext ctxt) throws CommandException { + try { + OAuth2UserRecord oAuth2UserRecord = ctxt.authentication().verifyOIDCBearerTokenAndGetOAuth2UserRecord(bearerToken); + UserRecordIdentifier userRecordIdentifier = oAuth2UserRecord.getUserRecordIdentifier(); + + if (ctxt.authentication().lookupUser(userRecordIdentifier) != null) { + throw new IllegalCommandException(BundleUtil.getStringFromBundle("registerOidcUserCommand.errors.userAlreadyRegisteredWithToken"), this); + } + + boolean provideMissingClaimsEnabled = FeatureFlags.API_BEARER_AUTH_PROVIDE_MISSING_CLAIMS.enabled(); + + updateUserDTO(oAuth2UserRecord, provideMissingClaimsEnabled); + + AuthenticatedUserDisplayInfo userDisplayInfo = new AuthenticatedUserDisplayInfo( + userDTO.getFirstName(), + userDTO.getLastName(), + userDTO.getEmailAddress(), + userDTO.getAffiliation() != null ? userDTO.getAffiliation() : "", + userDTO.getPosition() != null ? userDTO.getPosition() : "" + ); + + validateUserFields(ctxt, provideMissingClaimsEnabled); + + ctxt.authentication().createAuthenticatedUser(userRecordIdentifier, userDTO.getUsername(), userDisplayInfo, true); + + } catch (AuthorizationException ex) { + throw new PermissionException(ex.getMessage(), this, null, null, true); + } + } + + private void updateUserDTO(OAuth2UserRecord oAuth2UserRecord, boolean provideMissingClaimsEnabled) throws InvalidFieldsCommandException { + if (provideMissingClaimsEnabled) { + Map fieldErrors = validateConflictingClaims(oAuth2UserRecord); + throwInvalidFieldsCommandExceptionIfErrorsExist(fieldErrors); + updateUserDTOWithClaims(oAuth2UserRecord); + } else { + Map fieldErrors = validateUserDTOHasNoClaims(); + throwInvalidFieldsCommandExceptionIfErrorsExist(fieldErrors); + overwriteUserDTOWithClaims(oAuth2UserRecord); + } + } + + private Map validateConflictingClaims(OAuth2UserRecord oAuth2UserRecord) { + Map fieldErrors = new HashMap<>(); + + addFieldErrorIfConflict(FIELD_USERNAME, oAuth2UserRecord.getUsername(), userDTO.getUsername(), fieldErrors); + addFieldErrorIfConflict(FIELD_FIRST_NAME, oAuth2UserRecord.getDisplayInfo().getFirstName(), userDTO.getFirstName(), fieldErrors); + addFieldErrorIfConflict(FIELD_LAST_NAME, oAuth2UserRecord.getDisplayInfo().getLastName(), userDTO.getLastName(), fieldErrors); + addFieldErrorIfConflict(FIELD_EMAIL_ADDRESS, oAuth2UserRecord.getDisplayInfo().getEmailAddress(), userDTO.getEmailAddress(), fieldErrors); + + return fieldErrors; + } + + private void addFieldErrorIfConflict(String fieldName, String claimValue, String existingValue, Map fieldErrors) { + if (claimValue != null && !claimValue.trim().isEmpty() && existingValue != null && !claimValue.equals(existingValue)) { + String errorMessage = BundleUtil.getStringFromBundle( + "registerOidcUserCommand.errors.provideMissingClaimsEnabled.fieldAlreadyPresentInProvider", + List.of(fieldName) + ); + fieldErrors.put(fieldName, errorMessage); + } + } + + private Map validateUserDTOHasNoClaims() { + Map fieldErrors = new HashMap<>(); + if (userDTO.getUsername() != null) { + String errorMessage = BundleUtil.getStringFromBundle( + "registerOidcUserCommand.errors.provideMissingClaimsDisabled.unableToSetFieldViaJSON", + List.of(FIELD_USERNAME) + ); + fieldErrors.put(FIELD_USERNAME, errorMessage); + } + if (userDTO.getEmailAddress() != null) { + String errorMessage = BundleUtil.getStringFromBundle( + "registerOidcUserCommand.errors.provideMissingClaimsDisabled.unableToSetFieldViaJSON", + List.of(FIELD_EMAIL_ADDRESS) + ); + fieldErrors.put(FIELD_EMAIL_ADDRESS, errorMessage); + } + if (userDTO.getFirstName() != null) { + String errorMessage = BundleUtil.getStringFromBundle( + "registerOidcUserCommand.errors.provideMissingClaimsDisabled.unableToSetFieldViaJSON", + List.of(FIELD_FIRST_NAME) + ); + fieldErrors.put(FIELD_FIRST_NAME, errorMessage); + } + if (userDTO.getLastName() != null) { + String errorMessage = BundleUtil.getStringFromBundle( + "registerOidcUserCommand.errors.provideMissingClaimsDisabled.unableToSetFieldViaJSON", + List.of(FIELD_LAST_NAME) + ); + fieldErrors.put(FIELD_LAST_NAME, errorMessage); + } + return fieldErrors; + } + + private void updateUserDTOWithClaims(OAuth2UserRecord oAuth2UserRecord) { + userDTO.setUsername(getValueOrDefault(oAuth2UserRecord.getUsername(), userDTO.getUsername())); + userDTO.setFirstName(getValueOrDefault(oAuth2UserRecord.getDisplayInfo().getFirstName(), userDTO.getFirstName())); + userDTO.setLastName(getValueOrDefault(oAuth2UserRecord.getDisplayInfo().getLastName(), userDTO.getLastName())); + userDTO.setEmailAddress(getValueOrDefault(oAuth2UserRecord.getDisplayInfo().getEmailAddress(), userDTO.getEmailAddress())); + } + + private void overwriteUserDTOWithClaims(OAuth2UserRecord oAuth2UserRecord) { + userDTO.setUsername(oAuth2UserRecord.getUsername()); + userDTO.setFirstName(oAuth2UserRecord.getDisplayInfo().getFirstName()); + userDTO.setLastName(oAuth2UserRecord.getDisplayInfo().getLastName()); + userDTO.setEmailAddress(oAuth2UserRecord.getDisplayInfo().getEmailAddress()); + } + + private void throwInvalidFieldsCommandExceptionIfErrorsExist(Map fieldErrors) throws InvalidFieldsCommandException { + if (!fieldErrors.isEmpty()) { + throw new InvalidFieldsCommandException( + BundleUtil.getStringFromBundle("registerOidcUserCommand.errors.invalidFields"), + this, + fieldErrors + ); + } + } + + private String getValueOrDefault(String oidcValue, String dtoValue) { + return (oidcValue == null || oidcValue.trim().isEmpty()) ? dtoValue : oidcValue; + } + + private void validateUserFields(CommandContext ctxt, boolean provideMissingClaimsEnabled) throws InvalidFieldsCommandException { + Map fieldErrors = new HashMap<>(); + + if (!FeatureFlags.API_BEARER_AUTH_HANDLE_TOS_ACCEPTANCE_IN_IDP.enabled()) { + validateTermsAccepted(fieldErrors); + } + + validateField(fieldErrors, FIELD_EMAIL_ADDRESS, userDTO.getEmailAddress(), ctxt, provideMissingClaimsEnabled); + validateField(fieldErrors, FIELD_USERNAME, userDTO.getUsername(), ctxt, provideMissingClaimsEnabled); + validateField(fieldErrors, FIELD_FIRST_NAME, userDTO.getFirstName(), ctxt, provideMissingClaimsEnabled); + validateField(fieldErrors, FIELD_LAST_NAME, userDTO.getLastName(), ctxt, provideMissingClaimsEnabled); + + throwInvalidFieldsCommandExceptionIfErrorsExist(fieldErrors); + } + + private void validateTermsAccepted(Map fieldErrors) { + if (!userDTO.isTermsAccepted()) { + fieldErrors.put(FIELD_TERMS_ACCEPTED, BundleUtil.getStringFromBundle("registerOidcUserCommand.errors.userShouldAcceptTerms")); + } + } + + private void validateField(Map fieldErrors, String fieldName, String fieldValue, CommandContext ctxt, boolean provideMissingClaimsEnabled) { + if (fieldValue == null || fieldValue.isEmpty()) { + String errorKey = provideMissingClaimsEnabled ? + "registerOidcUserCommand.errors.provideMissingClaimsEnabled.fieldRequired" : + "registerOidcUserCommand.errors.provideMissingClaimsDisabled.fieldRequired"; + fieldErrors.put(fieldName, BundleUtil.getStringFromBundle(errorKey, List.of(fieldName))); + } else if (isFieldInUse(ctxt, fieldName, fieldValue)) { + fieldErrors.put(fieldName, BundleUtil.getStringFromBundle("registerOidcUserCommand.errors." + fieldName + "InUse")); + } + } + + private boolean isFieldInUse(CommandContext ctxt, String fieldName, String value) { + if (FIELD_EMAIL_ADDRESS.equals(fieldName)) { + return ctxt.authentication().getAuthenticatedUserByEmail(value) != null; + } else if (FIELD_USERNAME.equals(fieldName)) { + return ctxt.authentication().getAuthenticatedUser(value) != null; + } + return false; + } +} diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDatasetTypeLinksToMetadataBlocksCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDatasetTypeLinksToMetadataBlocksCommand.java new file mode 100644 index 00000000000..57b6da3f90c --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDatasetTypeLinksToMetadataBlocksCommand.java @@ -0,0 +1,37 @@ +package edu.harvard.iq.dataverse.engine.command.impl; + +import edu.harvard.iq.dataverse.DvObject; +import edu.harvard.iq.dataverse.MetadataBlock; +import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser; +import edu.harvard.iq.dataverse.dataset.DatasetType; +import edu.harvard.iq.dataverse.engine.command.AbstractVoidCommand; +import edu.harvard.iq.dataverse.engine.command.CommandContext; +import edu.harvard.iq.dataverse.engine.command.DataverseRequest; +import edu.harvard.iq.dataverse.engine.command.RequiredPermissions; +import edu.harvard.iq.dataverse.engine.command.exception.CommandException; +import edu.harvard.iq.dataverse.engine.command.exception.PermissionException; +import java.util.List; + +@RequiredPermissions({}) +public class UpdateDatasetTypeLinksToMetadataBlocksCommand extends AbstractVoidCommand { + + final DatasetType datasetType; + List metadataBlocks; + + public UpdateDatasetTypeLinksToMetadataBlocksCommand(DataverseRequest dataverseRequest, DatasetType datasetType, List metadataBlocks) { + super(dataverseRequest, (DvObject) null); + this.datasetType = datasetType; + this.metadataBlocks = metadataBlocks; + } + + @Override + protected void executeImpl(CommandContext ctxt) throws CommandException { + if (!(getUser() instanceof AuthenticatedUser) || !getUser().isSuperuser()) { + throw new PermissionException("Update dataset type links to metadata block command can only be called by superusers.", + this, null, null); + } + datasetType.setMetadataBlocks(metadataBlocks); + ctxt.em().merge(datasetType); + } + +} diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDatasetVersionCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDatasetVersionCommand.java index dc8884405ef..209791faafb 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDatasetVersionCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDatasetVersionCommand.java @@ -230,7 +230,8 @@ public Dataset execute(CommandContext ctxt) throws CommandException { if (!theDataset.getOrCreateEditVersion().equals(fmd.getDatasetVersion())) { fmd = FileMetadataUtil.getFmdForFileInEditVersion(fmd, theDataset.getOrCreateEditVersion()); } - } + } + fmd.setDataFile(ctxt.em().merge(fmd.getDataFile())); fmd = ctxt.em().merge(fmd); // There are two datafile cases as well - the file has been released, so we're @@ -241,13 +242,15 @@ public Dataset execute(CommandContext ctxt) throws CommandException { ctxt.engine().submit(new DeleteDataFileCommand(fmd.getDataFile(), getRequest())); // and remove the file from the dataset's list theDataset.getFiles().remove(fmd.getDataFile()); + ctxt.em().remove(fmd.getDataFile()); + ctxt.em().remove(fmd); } else { - // if we aren't removing the file, we need to explicitly remove the fmd from the - // context and then remove it from the datafile's list ctxt.em().remove(fmd); + // if we aren't removing the file, we need to remove it from the datafile's list FileMetadataUtil.removeFileMetadataFromList(fmd.getDataFile().getFileMetadatas(), fmd); } - // In either case, to fully remove the fmd, we have to remove any other possible + // In either case, we've removed from the context + // And, to fully remove the fmd, we have to remove any other possible // references // From the datasetversion FileMetadataUtil.removeFileMetadataFromList(theDataset.getOrCreateEditVersion().getFileMetadatas(), fmd); @@ -255,6 +258,7 @@ public Dataset execute(CommandContext ctxt) throws CommandException { for (DataFileCategory cat : theDataset.getCategories()) { FileMetadataUtil.removeFileMetadataFromList(cat.getFileMetadatas(), fmd); } + } for(FileMetadata fmd: theDataset.getOrCreateEditVersion().getFileMetadatas()) { logger.fine("FMD: " + fmd.getId() + " for file: " + fmd.getDataFile().getId() + "is in final draft version"); diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDataverseAttributeCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDataverseAttributeCommand.java index 57ac20fcee6..ab12d8eea26 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDataverseAttributeCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDataverseAttributeCommand.java @@ -24,7 +24,7 @@ public class UpdateDataverseAttributeCommand extends AbstractCommand private static final String ATTRIBUTE_DESCRIPTION = "description"; private static final String ATTRIBUTE_AFFILIATION = "affiliation"; private static final String ATTRIBUTE_FILE_PIDS_ENABLED = "filePIDsEnabled"; - + private static final String ATTRIBUTE_REQUIRE_FILES_TO_PUBLISH_DATASET = "requireFilesToPublishDataset"; private final Dataverse dataverse; private final String attributeName; private final Object attributeValue; @@ -45,8 +45,9 @@ public Dataverse execute(CommandContext ctxt) throws CommandException { case ATTRIBUTE_AFFILIATION: setStringAttribute(attributeName, attributeValue); break; + case ATTRIBUTE_REQUIRE_FILES_TO_PUBLISH_DATASET: case ATTRIBUTE_FILE_PIDS_ENABLED: - setBooleanAttributeForFilePIDs(ctxt); + setBooleanAttribute(ctxt, true); break; default: throw new IllegalCommandException("'" + attributeName + "' is not a supported attribute", this); @@ -86,25 +87,33 @@ private void setStringAttribute(String attributeName, Object attributeValue) thr } /** - * Helper method to handle the "filePIDsEnabled" boolean attribute. + * Helper method to handle boolean attributes. * * @param ctxt The command context. + * @param adminOnly True if this attribute can only be modified by an Administrator * @throws PermissionException if the user doesn't have permission to modify this attribute. */ - private void setBooleanAttributeForFilePIDs(CommandContext ctxt) throws CommandException { - if (!getRequest().getUser().isSuperuser()) { + private void setBooleanAttribute(CommandContext ctxt, boolean adminOnly) throws CommandException { + if (adminOnly && !getRequest().getUser().isSuperuser()) { throw new PermissionException("You must be a superuser to change this setting", this, Collections.singleton(Permission.EditDataset), dataverse); } - if (!ctxt.settings().isTrueForKey(SettingsServiceBean.Key.AllowEnablingFilePIDsPerCollection, false)) { - throw new PermissionException("Changing File PID policy per collection is not enabled on this server", - this, Collections.singleton(Permission.EditDataset), dataverse); - } if (!(attributeValue instanceof Boolean)) { - throw new IllegalCommandException("'" + ATTRIBUTE_FILE_PIDS_ENABLED + "' requires a boolean value", this); + throw new IllegalCommandException("'" + attributeName + "' requires a boolean value", this); + } + switch (attributeName) { + case ATTRIBUTE_FILE_PIDS_ENABLED: + if (!ctxt.settings().isTrueForKey(SettingsServiceBean.Key.AllowEnablingFilePIDsPerCollection, false)) { + throw new PermissionException("Changing File PID policy per collection is not enabled on this server", + this, Collections.singleton(Permission.EditDataset), dataverse); + } + dataverse.setFilePIDsEnabled((Boolean) attributeValue); + case ATTRIBUTE_REQUIRE_FILES_TO_PUBLISH_DATASET: + dataverse.setRequireFilesToPublishDataset((Boolean) attributeValue); + break; + default: + throw new IllegalCommandException("Unsupported boolean attribute: " + attributeName, this); } - - dataverse.setFilePIDsEnabled((Boolean) attributeValue); } } diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDataverseCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDataverseCommand.java index 6dc4ab4d00d..55cc3708097 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDataverseCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDataverseCommand.java @@ -32,7 +32,7 @@ public UpdateDataverseCommand(Dataverse dataverse, List featuredDataverses, DataverseRequest request, List inputLevels) { - this(dataverse, facets, featuredDataverses, request, inputLevels, null, null, false); + this(dataverse, facets, featuredDataverses, request, inputLevels, null, null); } public UpdateDataverseCommand(Dataverse dataverse, @@ -41,9 +41,8 @@ public UpdateDataverseCommand(Dataverse dataverse, DataverseRequest request, List inputLevels, List metadataBlocks, - DataverseDTO updatedDataverseDTO, - boolean resetRelationsOnNullValues) { - super(dataverse, dataverse, request, facets, inputLevels, metadataBlocks, resetRelationsOnNullValues); + DataverseDTO updatedDataverseDTO) { + super(dataverse, dataverse, request, facets, inputLevels, metadataBlocks); if (featuredDataverses != null) { this.featuredDataverseList = new ArrayList<>(featuredDataverses); } else { diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDataverseFeaturedItemCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDataverseFeaturedItemCommand.java new file mode 100644 index 00000000000..ed6fe825b03 --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDataverseFeaturedItemCommand.java @@ -0,0 +1,41 @@ +package edu.harvard.iq.dataverse.engine.command.impl; + +import edu.harvard.iq.dataverse.Dataverse; +import edu.harvard.iq.dataverse.dataverse.featured.DataverseFeaturedItem; +import edu.harvard.iq.dataverse.api.dto.UpdatedDataverseFeaturedItemDTO; +import edu.harvard.iq.dataverse.engine.command.*; +import edu.harvard.iq.dataverse.engine.command.exception.CommandException; + +/** + * Updates a particular featured item {@link DataverseFeaturedItem} of a {@link Dataverse}. + */ +public class UpdateDataverseFeaturedItemCommand extends AbstractWriteDataverseFeaturedItemCommand { + + private final DataverseFeaturedItem dataverseFeaturedItem; + private final UpdatedDataverseFeaturedItemDTO updatedDataverseFeaturedItemDTO; + + public UpdateDataverseFeaturedItemCommand(DataverseRequest request, + DataverseFeaturedItem dataverseFeaturedItem, + UpdatedDataverseFeaturedItemDTO updatedDataverseFeaturedItemDTO) { + super(request, dataverseFeaturedItem.getDataverse()); + this.dataverseFeaturedItem = dataverseFeaturedItem; + this.updatedDataverseFeaturedItemDTO = updatedDataverseFeaturedItemDTO; + } + + @Override + public DataverseFeaturedItem execute(CommandContext ctxt) throws CommandException { + validateAndSetContent(dataverseFeaturedItem, updatedDataverseFeaturedItemDTO.getContent()); + dataverseFeaturedItem.setDisplayOrder(updatedDataverseFeaturedItemDTO.getDisplayOrder()); + + if (!updatedDataverseFeaturedItemDTO.isKeepFile()) { + setFileImageIfAvailableOrNull( + dataverseFeaturedItem, + updatedDataverseFeaturedItemDTO.getImageFileName(), + updatedDataverseFeaturedItemDTO.getImageFileInputStream(), + ctxt + ); + } + + return ctxt.dataverseFeaturedItems().save(dataverseFeaturedItem); + } +} diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDataverseFeaturedItemsCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDataverseFeaturedItemsCommand.java new file mode 100644 index 00000000000..0368efef6b0 --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDataverseFeaturedItemsCommand.java @@ -0,0 +1,78 @@ +package edu.harvard.iq.dataverse.engine.command.impl; + +import edu.harvard.iq.dataverse.Dataverse; +import edu.harvard.iq.dataverse.dataverse.featured.DataverseFeaturedItem; +import edu.harvard.iq.dataverse.api.dto.NewDataverseFeaturedItemDTO; +import edu.harvard.iq.dataverse.api.dto.UpdatedDataverseFeaturedItemDTO; +import edu.harvard.iq.dataverse.authorization.Permission; +import edu.harvard.iq.dataverse.engine.command.*; +import edu.harvard.iq.dataverse.engine.command.exception.CommandException; + +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; +import java.util.Map; + +/** + * Updates all featured items ({@link DataverseFeaturedItem}) for a specified {@link Dataverse}. + *

+ * This command allows for the creation of multiple new featured items, updates to existing items with new parameters, + * or the deletion of existing items, all in a single command. + *

+ **/ +@RequiredPermissions({Permission.EditDataverse}) +public class UpdateDataverseFeaturedItemsCommand extends AbstractCommand> { + + private final Dataverse dataverse; + private final List newDataverseFeaturedItemDTOs; + private final Map dataverseFeaturedItemsToUpdate; + + public UpdateDataverseFeaturedItemsCommand(DataverseRequest request, Dataverse dataverse, List newDataverseFeaturedItemDTOs, Map dataverseFeaturedItemsToUpdate) { + super(request, dataverse); + this.dataverse = dataverse; + this.newDataverseFeaturedItemDTOs = newDataverseFeaturedItemDTOs; + this.dataverseFeaturedItemsToUpdate = dataverseFeaturedItemsToUpdate; + } + + @Override + public List execute(CommandContext ctxt) throws CommandException { + List dataverseFeaturedItems = updateOrDeleteExistingFeaturedItems(ctxt); + dataverseFeaturedItems.addAll(createNewFeaturedItems(ctxt)); + dataverseFeaturedItems.sort(Comparator.comparingInt(DataverseFeaturedItem::getDisplayOrder)); + return dataverseFeaturedItems; + } + + private List updateOrDeleteExistingFeaturedItems(CommandContext ctxt) throws CommandException { + List updatedFeaturedItems = new ArrayList<>(); + List featuredItemsToDelete = dataverse.getDataverseFeaturedItems(); + + for (Map.Entry entry : dataverseFeaturedItemsToUpdate.entrySet()) { + DataverseFeaturedItem featuredItem = entry.getKey(); + UpdatedDataverseFeaturedItemDTO updatedDTO = entry.getValue(); + + featuredItemsToDelete.stream() + .filter(item -> item.getId().equals(featuredItem.getId())) + .findFirst().ifPresent(featuredItemsToDelete::remove); + + DataverseFeaturedItem updatedFeatureItem = ctxt.engine().submit(new UpdateDataverseFeaturedItemCommand(getRequest(), featuredItem, updatedDTO)); + updatedFeaturedItems.add(updatedFeatureItem); + } + + for (DataverseFeaturedItem featuredItem : featuredItemsToDelete) { + ctxt.engine().submit(new DeleteDataverseFeaturedItemCommand(getRequest(), featuredItem)); + } + + return updatedFeaturedItems; + } + + private List createNewFeaturedItems(CommandContext ctxt) throws CommandException { + List createdFeaturedItems = new ArrayList<>(); + + for (NewDataverseFeaturedItemDTO dto : newDataverseFeaturedItemDTOs) { + DataverseFeaturedItem createdFeatureItem = ctxt.engine().submit(new CreateDataverseFeaturedItemCommand(getRequest(), dataverse, dto)); + createdFeaturedItems.add(createdFeatureItem); + } + + return createdFeaturedItems; + } +} diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDataverseInputLevelsCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDataverseInputLevelsCommand.java index b9b08992919..cc2c525e8f6 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDataverseInputLevelsCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDataverseInputLevelsCommand.java @@ -2,6 +2,7 @@ import edu.harvard.iq.dataverse.Dataverse; import edu.harvard.iq.dataverse.DataverseFieldTypeInputLevel; +import edu.harvard.iq.dataverse.MetadataBlock; import edu.harvard.iq.dataverse.authorization.Permission; import edu.harvard.iq.dataverse.engine.command.AbstractCommand; import edu.harvard.iq.dataverse.engine.command.CommandContext; @@ -28,8 +29,18 @@ public Dataverse execute(CommandContext ctxt) throws CommandException { if (inputLevelList == null || inputLevelList.isEmpty()) { throw new CommandException("Error while updating dataverse input levels: Input level list cannot be null or empty", this); } + + if (!dataverse.isMetadataBlockRoot()) { + Dataverse root = ctxt.dataverses().findRootDataverse(); + if (root != null) { + List inheritedBlocks = new ArrayList<>(root.getMetadataBlocks()); + dataverse.setMetadataBlocks(inheritedBlocks); + dataverse.setMetadataBlockRoot(true); + } + } + dataverse.addInputLevelsMetadataBlocksIfNotPresent(inputLevelList); - dataverse.setMetadataBlockRoot(true); + return ctxt.engine().submit(new UpdateDataverseCommand(dataverse, null, null, getRequest(), inputLevelList)); } } diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdatePublishedDatasetVersionCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdatePublishedDatasetVersionCommand.java new file mode 100644 index 00000000000..f8f5d05d972 --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdatePublishedDatasetVersionCommand.java @@ -0,0 +1,63 @@ +package edu.harvard.iq.dataverse.engine.command.impl; + +import edu.harvard.iq.dataverse.DatasetVersion; +import edu.harvard.iq.dataverse.authorization.Permission; +import edu.harvard.iq.dataverse.engine.command.AbstractCommand; +import edu.harvard.iq.dataverse.engine.command.CommandContext; +import edu.harvard.iq.dataverse.engine.command.DataverseRequest; +import edu.harvard.iq.dataverse.engine.command.RequiredPermissions; +import edu.harvard.iq.dataverse.engine.command.exception.CommandException; +import edu.harvard.iq.dataverse.engine.command.exception.IllegalCommandException; + +/** + * Updates the specified dataset version, which must already be published/not + * draft. The command is only usable by superusers. The initial use is to allow + * adding version notes to existing versions but it is otherwise generic + * assuming there may be other cases where updating some aspect of an existing + * version is needed. Note this is similar to the + * CuratePublishedDatasetVersionCommand, but in that case changes in an existing + * draft version are pushed into the latest published version as a form of + * republishing (and the draft version ceases to exist). This command assumes + * changes have been made to the existing dataset version of interest, which may + * not be the latest published one, and it does not make any changes to a + * dataset's draft version if that exists. + */ +@RequiredPermissions(Permission.EditDataset) +public class UpdatePublishedDatasetVersionCommand extends AbstractCommand { + + private final DatasetVersion datasetVersion; + + public UpdatePublishedDatasetVersionCommand(DataverseRequest aRequest, DatasetVersion datasetVersion) { + super(aRequest, datasetVersion.getDataset()); + this.datasetVersion = datasetVersion; + } + + @Override + public DatasetVersion execute(CommandContext ctxt) throws CommandException { + // Check if the user is a superuser + if (!getUser().isSuperuser()) { + throw new IllegalCommandException("Only superusers can update published dataset versions", this); + } + + // Ensure the version is published + if (!datasetVersion.isReleased()) { + throw new IllegalCommandException("This command can only be used on published dataset versions", this); + } + + // Save the changes + DatasetVersion savedVersion = ctxt.em().merge(datasetVersion); + + return savedVersion; + } + + @Override + public boolean onSuccess(CommandContext ctxt, Object r) { + DatasetVersion version = (DatasetVersion) r; + // Only need to reindex if this version is the latest published version for the + // dataset + if (version.equals(version.getDataset().getLatestVersionForCopy())) { + ctxt.index().asyncIndexDataset(version.getDataset(), true); + } + return true; + } +} diff --git a/src/main/java/edu/harvard/iq/dataverse/export/SchemaDotOrgExporter.java b/src/main/java/edu/harvard/iq/dataverse/export/SchemaDotOrgExporter.java index 0c4b39fd641..d4f2f95389f 100644 --- a/src/main/java/edu/harvard/iq/dataverse/export/SchemaDotOrgExporter.java +++ b/src/main/java/edu/harvard/iq/dataverse/export/SchemaDotOrgExporter.java @@ -111,7 +111,11 @@ public Boolean isAvailableToUsers() { @Override public String getMediaType() { - return MediaType.APPLICATION_JSON; + /** + * Changed from "application/json" to "application/ld+json" because + * that's what Signposting expects. + */ + return "application/ld+json"; } } diff --git a/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java b/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java index 05ddbe83e78..1a02089aef9 100644 --- a/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java @@ -5,11 +5,14 @@ import edu.harvard.iq.dataverse.ControlledVocabularyValue; import edu.harvard.iq.dataverse.DatasetFieldConstant; import edu.harvard.iq.dataverse.DvObjectContainer; +import edu.harvard.iq.dataverse.GlobalId; +import edu.harvard.iq.dataverse.api.dto.MetadataBlockDTO; import edu.harvard.iq.dataverse.api.dto.DatasetDTO; import edu.harvard.iq.dataverse.api.dto.DatasetVersionDTO; -import edu.harvard.iq.dataverse.api.dto.FieldDTO; import edu.harvard.iq.dataverse.api.dto.FileDTO; -import edu.harvard.iq.dataverse.api.dto.MetadataBlockDTO; +import edu.harvard.iq.dataverse.api.dto.FieldDTO; +import edu.harvard.iq.dataverse.api.dto.LicenseDTO; + import static edu.harvard.iq.dataverse.export.DDIExportServiceBean.LEVEL_FILE; import static edu.harvard.iq.dataverse.export.DDIExportServiceBean.NOTE_SUBJECT_TAG; @@ -83,6 +86,10 @@ public class DdiExportUtil { public static final String NOTE_SUBJECT_CONTENTTYPE = "Content/MIME Type"; public static final String CITATION_BLOCK_NAME = "citation"; + //Some tests don't send real PIDs that can be parsed + //Use constant empty PID in these cases + private static final String EMPTY_PID = "null:nullnullnull"; + public static String datasetDtoAsJson2ddi(String datasetDtoAsJson) { Gson gson = new Gson(); DatasetDTO datasetDto = gson.fromJson(datasetDtoAsJson, DatasetDTO.class); @@ -167,11 +174,14 @@ private static void createStdyDscr(XMLStreamWriter xmlw, DatasetDTO datasetDto) String persistentAuthority = datasetDto.getAuthority(); String persistentId = datasetDto.getIdentifier(); - String pid = persistentProtocol + ":" + persistentAuthority + "/" + persistentId; - String pidUri = pid; - //Some tests don't send real PIDs - don't try to get their URL form - if(!pidUri.equals("null:null/null")) { - pidUri= PidUtil.parseAsGlobalID(persistentProtocol, persistentAuthority, persistentId).asURL(); + GlobalId pid = PidUtil.parseAsGlobalID(persistentProtocol, persistentAuthority, persistentId); + String pidUri, pidString; + if(pid != null) { + pidUri = pid.asURL(); + pidString = pid.asString(); + } else { + pidUri = EMPTY_PID; + pidString = EMPTY_PID; } // The "persistentAgency" tag is used for the "agency" attribute of the // ddi section; back in the DVN3 days we used "handle" and "DOI" @@ -201,7 +211,7 @@ private static void createStdyDscr(XMLStreamWriter xmlw, DatasetDTO datasetDto) XmlWriterUtil.writeAttribute(xmlw, "agency", persistentAgency); - xmlw.writeCharacters(pid); + xmlw.writeCharacters(pidString); xmlw.writeEndElement(); // IDNo writeOtherIdElement(xmlw, version); xmlw.writeEndElement(); // titlStmt @@ -313,8 +323,16 @@ private static void writeDataAccess(XMLStreamWriter xmlw , DatasetVersionDTO ver XmlWriterUtil.writeFullElement(xmlw, "conditions", version.getConditions()); XmlWriterUtil.writeFullElement(xmlw, "disclaimer", version.getDisclaimer()); xmlw.writeEndElement(); //useStmt - + /* any s: */ + if (version.getTermsOfUse() != null && !version.getTermsOfUse().trim().equals("")) { + xmlw.writeStartElement("notes"); + xmlw.writeAttribute("type", NOTE_TYPE_TERMS_OF_USE); + xmlw.writeAttribute("level", LEVEL_DV); + xmlw.writeCharacters(version.getTermsOfUse()); + xmlw.writeEndElement(); //notes + } + if (version.getTermsOfAccess() != null && !version.getTermsOfAccess().trim().equals("")) { xmlw.writeStartElement("notes"); xmlw.writeAttribute("type", NOTE_TYPE_TERMS_OF_ACCESS); @@ -322,6 +340,19 @@ private static void writeDataAccess(XMLStreamWriter xmlw , DatasetVersionDTO ver xmlw.writeCharacters(version.getTermsOfAccess()); xmlw.writeEndElement(); //notes } + + LicenseDTO license = version.getLicense(); + if (license != null) { + String name = license.getName(); + String uri = license.getUri(); + if ((name != null && !name.trim().equals("")) && (uri != null && !uri.trim().equals(""))) { + xmlw.writeStartElement("notes"); + xmlw.writeAttribute("type", NOTE_TYPE_TERMS_OF_USE); + xmlw.writeAttribute("level", LEVEL_DV); + xmlw.writeCharacters("" + name + ""); + xmlw.writeEndElement(); //notes + } + } xmlw.writeEndElement(); //dataAccs } @@ -341,14 +372,21 @@ private static void writeDocDescElement (XMLStreamWriter xmlw, DatasetDTO datase String persistentAuthority = datasetDto.getAuthority(); String persistentId = datasetDto.getIdentifier(); - + GlobalId pid = PidUtil.parseAsGlobalID(persistentProtocol, persistentAuthority, persistentId); + String pidString; + if(pid != null) { + pidString = pid.asString(); + } else { + pidString = EMPTY_PID; + } + xmlw.writeStartElement("docDscr"); xmlw.writeStartElement("citation"); xmlw.writeStartElement("titlStmt"); XmlWriterUtil.writeFullElement(xmlw, "titl", XmlWriterUtil.dto2Primitive(version, DatasetFieldConstant.title), datasetDto.getMetadataLanguage()); xmlw.writeStartElement("IDNo"); XmlWriterUtil.writeAttribute(xmlw, "agency", persistentAgency); - xmlw.writeCharacters(persistentProtocol + ":" + persistentAuthority + "/" + persistentId); + xmlw.writeCharacters(pidString); xmlw.writeEndElement(); // IDNo xmlw.writeEndElement(); // titlStmt xmlw.writeStartElement("distStmt"); @@ -373,12 +411,18 @@ private static void writeDocDescElement (XMLStreamWriter xmlw, DatasetDTO datase private static void writeVersionStatement(XMLStreamWriter xmlw, DatasetVersionDTO datasetVersionDTO) throws XMLStreamException{ xmlw.writeStartElement("verStmt"); - xmlw.writeAttribute("source","archive"); + xmlw.writeAttribute("source","archive"); xmlw.writeStartElement("version"); XmlWriterUtil.writeAttribute(xmlw,"date", datasetVersionDTO.getReleaseTime().substring(0, 10)); - XmlWriterUtil.writeAttribute(xmlw,"type", datasetVersionDTO.getVersionState().toString()); + XmlWriterUtil.writeAttribute(xmlw,"type", datasetVersionDTO.getVersionState().toString()); xmlw.writeCharacters(datasetVersionDTO.getVersionNumber().toString()); xmlw.writeEndElement(); // version + if (!StringUtils.isBlank(datasetVersionDTO.getVersionNote())) { + xmlw.writeStartElement("notes"); + xmlw.writeCharacters(datasetVersionDTO.getVersionNote()); + xmlw.writeEndElement(); // notes + } + xmlw.writeEndElement(); // verStmt } @@ -647,7 +691,7 @@ private static void writeMethodElement(XMLStreamWriter xmlw , DatasetVersionDTO xmlw.writeStartElement("dataColl"); XmlWriterUtil.writeI18NElement(xmlw, "timeMeth", version, DatasetFieldConstant.timeMethod,lang); XmlWriterUtil.writeI18NElement(xmlw, "dataCollector", version, DatasetFieldConstant.dataCollector, lang); - XmlWriterUtil.writeI18NElement(xmlw, "collectorTraining", version, DatasetFieldConstant.collectorTraining, lang); + XmlWriterUtil.writeI18NElement(xmlw, "collectorTraining", version, DatasetFieldConstant.collectorTraining, lang); XmlWriterUtil.writeI18NElement(xmlw, "frequenc", version, DatasetFieldConstant.frequencyOfDataCollection, lang); XmlWriterUtil.writeI18NElement(xmlw, "sampProc", version, DatasetFieldConstant.samplingProcedure, lang); @@ -668,7 +712,7 @@ private static void writeMethodElement(XMLStreamWriter xmlw , DatasetVersionDTO } } /* and so does : */ - XmlWriterUtil.writeI18NElement(xmlw, "resInstru", version, DatasetFieldConstant.researchInstrument, lang); + XmlWriterUtil.writeI18NElement(xmlw, "resInstru", version, DatasetFieldConstant.researchInstrument, lang); xmlw.writeStartElement("sources"); XmlWriterUtil.writeFullElementList(xmlw, "dataSrc", dto2PrimitiveList(version, DatasetFieldConstant.dataSources)); XmlWriterUtil.writeI18NElement(xmlw, "srcOrig", version, DatasetFieldConstant.originOfSources, lang); @@ -681,7 +725,7 @@ private static void writeMethodElement(XMLStreamWriter xmlw , DatasetVersionDTO XmlWriterUtil.writeI18NElement(xmlw, "actMin", version, DatasetFieldConstant.actionsToMinimizeLoss, lang); /* "" has the uppercase C: */ XmlWriterUtil.writeI18NElement(xmlw, "ConOps", version, DatasetFieldConstant.controlOperations, lang); - XmlWriterUtil.writeI18NElement(xmlw, "weight", version, DatasetFieldConstant.weighting, lang); + XmlWriterUtil.writeI18NElement(xmlw, "weight", version, DatasetFieldConstant.weighting, lang); XmlWriterUtil.writeI18NElement(xmlw, "cleanOps", version, DatasetFieldConstant.cleaningOperations, lang); xmlw.writeEndElement(); //dataColl @@ -692,7 +736,7 @@ private static void writeMethodElement(XMLStreamWriter xmlw , DatasetVersionDTO //XmlWriterUtil.writeFullElement(xmlw, "anylInfo", dto2Primitive(version, DatasetFieldConstant.datasetLevelErrorNotes)); XmlWriterUtil.writeI18NElement(xmlw, "respRate", version, DatasetFieldConstant.responseRate, lang); XmlWriterUtil.writeI18NElement(xmlw, "EstSmpErr", version, DatasetFieldConstant.samplingErrorEstimates, lang); - XmlWriterUtil.writeI18NElement(xmlw, "dataAppr", version, DatasetFieldConstant.otherDataAppraisal, lang); + XmlWriterUtil.writeI18NElement(xmlw, "dataAppr", version, DatasetFieldConstant.otherDataAppraisal, lang); xmlw.writeEndElement(); //anlyInfo xmlw.writeEndElement();//method @@ -844,7 +888,7 @@ private static void writeAuthorsElement(XMLStreamWriter xmlw, DatasetVersionDTO } if (!authorName.isEmpty()){ xmlw.writeStartElement("AuthEnty"); - XmlWriterUtil.writeAttribute(xmlw,"affiliation",authorAffiliation); + XmlWriterUtil.writeAttribute(xmlw,"affiliation",authorAffiliation); xmlw.writeCharacters(authorName); xmlw.writeEndElement(); //AuthEnty } @@ -905,8 +949,8 @@ private static void writeContactsElement(XMLStreamWriter xmlw, DatasetVersionDTO // TODO: Since datasetContactEmail is a required field but datasetContactName is not consider not checking if datasetContactName is empty so we can write out datasetContactEmail. if (!datasetContactName.isEmpty()){ xmlw.writeStartElement("contact"); - XmlWriterUtil.writeAttribute(xmlw,"affiliation",datasetContactAffiliation); - XmlWriterUtil.writeAttribute(xmlw,"email",datasetContactEmail); + XmlWriterUtil.writeAttribute(xmlw,"affiliation",datasetContactAffiliation); + XmlWriterUtil.writeAttribute(xmlw,"email",datasetContactEmail); xmlw.writeCharacters(datasetContactName); xmlw.writeEndElement(); //AuthEnty } @@ -1131,7 +1175,7 @@ private static void writeAbstractElement(XMLStreamWriter xmlw, DatasetVersionDTO } if (!descriptionText.isEmpty()){ xmlw.writeStartElement("abstract"); - XmlWriterUtil.writeAttribute(xmlw,"date",descriptionDate); + XmlWriterUtil.writeAttribute(xmlw,"date",descriptionDate); if(DvObjectContainer.isMetadataLanguageSet(lang)) { xmlw.writeAttribute("xml:lang", lang); } @@ -1166,7 +1210,7 @@ private static void writeGrantElement(XMLStreamWriter xmlw, DatasetVersionDTO da } if (!grantNumber.isEmpty()){ xmlw.writeStartElement("grantNo"); - XmlWriterUtil.writeAttribute(xmlw,"agency",grantAgency); + XmlWriterUtil.writeAttribute(xmlw,"agency",grantAgency); xmlw.writeCharacters(grantNumber); xmlw.writeEndElement(); //grantno } @@ -1198,7 +1242,7 @@ private static void writeOtherIdElement(XMLStreamWriter xmlw, DatasetVersionDTO } if (!otherId.isEmpty()){ xmlw.writeStartElement("IDNo"); - XmlWriterUtil.writeAttribute(xmlw,"agency",otherIdAgency); + XmlWriterUtil.writeAttribute(xmlw,"agency",otherIdAgency); xmlw.writeCharacters(otherId); xmlw.writeEndElement(); //IDNo } @@ -1230,7 +1274,7 @@ private static void writeSoftwareElement(XMLStreamWriter xmlw, DatasetVersionDTO } if (!softwareName.isEmpty()){ xmlw.writeStartElement("software"); - XmlWriterUtil.writeAttribute(xmlw,"version",softwareVersion); + XmlWriterUtil.writeAttribute(xmlw,"version",softwareVersion); xmlw.writeCharacters(softwareName); xmlw.writeEndElement(); //software } @@ -1343,8 +1387,8 @@ private static void writeNotesElement(XMLStreamWriter xmlw, DatasetVersionDTO da } if (!notesText.isEmpty()) { xmlw.writeStartElement("notes"); - XmlWriterUtil.writeAttribute(xmlw,"type",notesType); - XmlWriterUtil.writeAttribute(xmlw,"subject",notesSubject); + XmlWriterUtil.writeAttribute(xmlw,"type",notesType); + XmlWriterUtil.writeAttribute(xmlw,"subject",notesSubject); xmlw.writeCharacters(notesText); xmlw.writeEndElement(); } @@ -1418,9 +1462,9 @@ private static void createOtherMatsFromFileMetadatas(XMLStreamWriter xmlw, JsonA xmlw.writeStartElement("otherMat"); xmlw.writeAttribute("ID", "f" + fileJson.getJsonNumber(("id").toString())); if (fileJson.containsKey("pidUrl")){ - XmlWriterUtil.writeAttribute(xmlw, "URI", fileJson.getString("pidUrl")); + XmlWriterUtil.writeAttribute(xmlw, "URI", fileJson.getString("pidUrl")); } else { - xmlw.writeAttribute("URI", dataverseUrl + "/api/access/datafile/" + fileJson.getJsonNumber("id").toString()); + xmlw.writeAttribute("URI", dataverseUrl + "/api/access/datafile/" + fileJson.getJsonNumber("id").toString()); } xmlw.writeAttribute("level", "datafile"); @@ -1491,7 +1535,7 @@ private static FieldDTO dto2FieldDTO(DatasetVersionDTO datasetVersionDTO, String } return null; } - + private static boolean StringUtilisEmpty(String str) { if (str == null || str.trim().equals("")) { diff --git a/src/main/java/edu/harvard/iq/dataverse/export/openaire/OpenAireExportUtil.java b/src/main/java/edu/harvard/iq/dataverse/export/openaire/OpenAireExportUtil.java index dd01750942d..a2ff980ca28 100644 --- a/src/main/java/edu/harvard/iq/dataverse/export/openaire/OpenAireExportUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/export/openaire/OpenAireExportUtil.java @@ -1271,12 +1271,16 @@ private static void writeDescriptionElement(XMLStreamWriter xmlw, String descrip */ public static void writeGeoLocationsElement(XMLStreamWriter xmlw, DatasetVersionDTO datasetVersionDTO, String language) throws XMLStreamException { // geoLocation -> geoLocationPlace - String geoLocationPlace = dto2Primitive(datasetVersionDTO, DatasetFieldConstant.productionPlace); + List geoLocationPlaces = dto2MultiplePrimitive(datasetVersionDTO, DatasetFieldConstant.productionPlace); boolean geoLocations_check = false; // write geoLocations geoLocations_check = writeOpenTag(xmlw, "geoLocations", geoLocations_check); - writeGeolocationPlace(xmlw, geoLocationPlace, language); + if (geoLocationPlaces != null) { + for (String geoLocationPlace : geoLocationPlaces) { + writeGeolocationPlace(xmlw, geoLocationPlace, language); + } + } // get DatasetFieldConstant.geographicBoundingBox for (Map.Entry entry : datasetVersionDTO.getMetadataBlocks().entrySet()) { @@ -1457,6 +1461,26 @@ private static String dto2Primitive(DatasetVersionDTO datasetVersionDTO, String } return null; } + + /** + * + * @param datasetVersionDTO + * @param datasetFieldTypeName + * @return List Multiple Primitive + * + */ + private static List dto2MultiplePrimitive(DatasetVersionDTO datasetVersionDTO, String datasetFieldTypeName) { + // give the single value of the given metadata + for (Map.Entry entry : datasetVersionDTO.getMetadataBlocks().entrySet()) { + MetadataBlockDTO value = entry.getValue(); + for (FieldDTO fieldDTO : value.getFields()) { + if (datasetFieldTypeName.equals(fieldDTO.getTypeName())) { + return fieldDTO.getMultiplePrimitive(); + } + } + } + return null; + } /** * Write a full tag. diff --git a/src/main/java/edu/harvard/iq/dataverse/feedback/Feedback.java b/src/main/java/edu/harvard/iq/dataverse/feedback/Feedback.java index c1162eb8db6..60742ca8a91 100644 --- a/src/main/java/edu/harvard/iq/dataverse/feedback/Feedback.java +++ b/src/main/java/edu/harvard/iq/dataverse/feedback/Feedback.java @@ -55,4 +55,10 @@ public JsonObjectBuilder toJsonObjectBuilder() { .add("body", body); } + public JsonObjectBuilder toLimitedJsonObjectBuilder() { + return new NullSafeJsonBuilder() + .add("fromEmail", fromEmail) + .add("subject", subject) + .add("body", body); + } } diff --git a/src/main/java/edu/harvard/iq/dataverse/globus/ExpiredTokenException.java b/src/main/java/edu/harvard/iq/dataverse/globus/ExpiredTokenException.java new file mode 100644 index 00000000000..7902bb780fd --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/globus/ExpiredTokenException.java @@ -0,0 +1,16 @@ +package edu.harvard.iq.dataverse.globus; + +/** + * + * @author landreev + */ +public class ExpiredTokenException extends Exception { + public ExpiredTokenException(String message) { + super(message); + } + + public ExpiredTokenException(String message, Throwable cause) { + super(message, cause); + } + +} diff --git a/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java index 58992805dc8..789e0883a7c 100644 --- a/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java @@ -118,12 +118,18 @@ public class GlobusServiceBean implements java.io.Serializable { private static final Logger logger = Logger.getLogger(GlobusServiceBean.class.getCanonicalName()); private static final SimpleDateFormat logFormatter = new SimpleDateFormat("yyyy-MM-dd'T'HH-mm-ss"); - private String getRuleId(GlobusEndpoint endpoint, String principal, String permissions) - throws MalformedURLException { + private String getRuleId(GlobusEndpoint endpoint, String principal, String permissions) { String principalType = "identity"; - - URL url = new URL("https://transfer.api.globusonline.org/v0.10/endpoint/" + endpoint.getId() + "/access_list"); + String apiUrlString = "https://transfer.api.globusonline.org/v0.10/endpoint/" + endpoint.getId() + "/access_list"; + URL url = null; + + try { + url = new URL(apiUrlString); + } catch (MalformedURLException mue) { + logger.severe("Malformed URL exception when attempting to look up ACL rule via Globus API: " + apiUrlString); + return null; + } MakeRequestResponse result = makeRequest(url, "Bearer", endpoint.getClientToken(), "GET", null); if (result.status == 200) { AccessList al = parseJson(result.jsonResponse, AccessList.class, false); @@ -154,27 +160,45 @@ private String getRuleId(GlobusEndpoint endpoint, String principal, String permi * @param dataset - the dataset associated with the rule * @param globusLogger - a separate logger instance, may be null */ - public void deletePermission(String ruleId, Dataset dataset, Logger globusLogger) { + private void deletePermission(String ruleId, Dataset dataset, Logger globusLogger) { globusLogger.fine("Start deleting rule " + ruleId + " for dataset " + dataset.getId()); if (ruleId.length() > 0) { if (dataset != null) { GlobusEndpoint endpoint = getGlobusEndpoint(dataset); if (endpoint != null) { - String accessToken = endpoint.getClientToken(); - globusLogger.info("Start deleting permissions."); - try { - URL url = new URL("https://transfer.api.globusonline.org/v0.10/endpoint/" + endpoint.getId() - + "/access/" + ruleId); - MakeRequestResponse result = makeRequest(url, "Bearer", accessToken, "DELETE", null); - if (result.status != 200) { - globusLogger.warning("Cannot delete access rule " + ruleId); - } else { - globusLogger.info("Access rule " + ruleId + " was deleted successfully"); - } - } catch (MalformedURLException ex) { - logger.log(Level.WARNING, - "Failed to delete access rule " + ruleId + " on endpoint " + endpoint.getId(), ex); + deletePermission(ruleId, endpoint, globusLogger); + } + } + } + } + + /** + * Call to delete a globus rule, via the ruleId and supplied endpoint + * + * @param ruleId - Globus rule id - assumed to be associated with the + * dataset's file path (should not be called with a user + * specified rule id w/o further checking) + * @param endpoint - the Globus endpoint associated with the rule + * @param globusLogger - a separate logger instance, may be null + */ + private void deletePermission(String ruleId, GlobusEndpoint endpoint, Logger globusLogger) { + globusLogger.fine("Start deleting rule " + ruleId + " for endpoint " + endpoint.getBasePath()); + if (ruleId.length() > 0) { + if (endpoint != null) { + String accessToken = endpoint.getClientToken(); + globusLogger.info("Start deleting permissions."); + try { + URL url = new URL("https://transfer.api.globusonline.org/v0.10/endpoint/" + endpoint.getId() + + "/access/" + ruleId); + MakeRequestResponse result = makeRequest(url, "Bearer", accessToken, "DELETE", null); + if (result.status != 200) { + globusLogger.warning("Cannot delete access rule " + ruleId); + } else { + globusLogger.info("Access rule " + ruleId + " was deleted successfully"); } + } catch (MalformedURLException ex) { + globusLogger.log(Level.WARNING, + "Failed to delete access rule " + ruleId + " on endpoint " + endpoint.getId(), ex); } } } @@ -209,6 +233,29 @@ public JsonObject requestAccessiblePaths(String principal, Dataset dataset, int } //The dir for the dataset's data exists, so try to request permission for the principal int requestPermStatus = requestPermission(endpoint, dataset, permissions); + + if (requestPermStatus == 409) { + // This is a special case - a 409 *may* mean that the rule already + // exists for this endnote and for this user (if, for example, + // Dataverse failed to remove it after the last upload has completed). + // That should be ok with us (but let's confirm that is indeed the + // case; alternatively it may mean that permissions cannot be issued + // for some other reason): + String ruleId = getRuleId(endpoint, principal, "rw"); + if (ruleId != null) { + logger.warning("Attention: potentially stale write access rule found for Globus path " + + endpoint.getBasePath() + + " for the principal " + + principal + + "; check the Globus endpoints for stale rules that are not properly deleted."); + requestPermStatus = 201; + } + // Unlike with DOWNloads, it should be somewhat safe not to worry + // about multiple transfers happening in parallel, all using the + // same access rule - since the dataset gets locked for the duration + // of an upload. + } + response.add("status", requestPermStatus); if (requestPermStatus == 201) { String driverId = dataset.getEffectiveStorageDriverId(); @@ -444,9 +491,9 @@ private void monitorTemporaryPermissions(String ruleId, long datasetId) { * files are created in general, some calls may use the * class logger) * @return - * @throws MalformedURLException + * @throws edu.harvard.iq.dataverse.globus.ExpiredTokenException */ - public GlobusTaskState getTask(String accessToken, String taskId, Logger globusLogger) { + public GlobusTaskState getTask(String accessToken, String taskId, Logger globusLogger) throws ExpiredTokenException { Logger myLogger = globusLogger != null ? globusLogger : logger; @@ -462,21 +509,31 @@ public GlobusTaskState getTask(String accessToken, String taskId, Logger globusL MakeRequestResponse result = makeRequest(url, "Bearer", accessToken, "GET", null); - GlobusTaskState task = null; + GlobusTaskState taskState = null; if (result.status == 200) { - task = parseJson(result.jsonResponse, GlobusTaskState.class, false); + taskState = parseJson(result.jsonResponse, GlobusTaskState.class, false); } + if (result.status != 200) { // @todo It should probably retry it 2-3 times before giving up; // similarly, it should probably differentiate between a "no such task" // response and something intermittent like a server/network error or // an expired token... i.e. something that's recoverable (?) - myLogger.warning("Cannot find information for the task " + taskId + " : Reason : " - + result.jsonResponse.toString()); + // edit: yes, but, should be done outside of this method, in the code + // that uses it + myLogger.warning("Cannot find information for the task " + taskId + + " status: " + + result.status + + " : Reason : " + + result.jsonResponse != null ? result.jsonResponse.toString() : "unknown" ); + } + + if (result.status == 401) { + throw new ExpiredTokenException("Http code 401 received. Auth. token must be expired."); } - return task; + return taskState; } /** @@ -639,7 +696,33 @@ public int setPermissionForDownload(Dataset dataset, String principal) { permissions.setPath(endpoint.getBasePath() + "/"); permissions.setPermissions("r"); - return requestPermission(endpoint, dataset, permissions); + int status = requestPermission(endpoint, dataset, permissions); + + if (status == 409) { + // It is possible that the permission already exists. If, for example, + // Dataverse failed to delete it after the last download by this + // user, or if there is another download from the same user + // currently in progress. The latter is now an option when the + // "asynchronous mode" is enabled for task monitoring (since all the + // ongoing tasks are recorded in the database, it is possible to check + // whether it is safe to delete the rule on completion of a task, vs. + // if other tasks are still using it). If that's the case, we'll + // confirm that the rule does exist and assume that it's ok to + // proceed with the download. + String ruleId = getRuleId(endpoint, principal, "r"); + if (ruleId != null) { + if (FeatureFlags.GLOBUS_USE_EXPERIMENTAL_ASYNC_FRAMEWORK.enabled()) { + return 201; + } else { + logger.warning("Attention: potentially stale read access rule found for Globus path " + + endpoint.getBasePath() + + " for the principal " + + principal + + "; check the Globus endpoints for stale rules that are not properly deleted."); + } + } + } + return status; } // Generates the URL to launch the Globus app for upload @@ -730,9 +813,14 @@ public void globusUpload(JsonObject jsonData, Dataset dataset, String httpReques String logTimestamp = logFormatter.format(startDate); Logger globusLogger = Logger.getLogger( - "edu.harvard.iq.dataverse.upload.client.DatasetServiceBean." + "GlobusUpload" + logTimestamp); - String logFileName = System.getProperty("com.sun.aas.instanceRoot") + File.separator + "logs" + File.separator + "globusUpload_" + dataset.getId() + "_" + logTimestamp - + ".log"; + "edu.harvard.iq.dataverse.globus.GlobusServiceBean." + "Globus" + + GlobusTaskInProgress.TaskType.UPLOAD + logTimestamp); + + String logFileName = System.getProperty("com.sun.aas.instanceRoot") + + File.separator + "logs" + + File.separator + "globus" + GlobusTaskInProgress.TaskType.UPLOAD + "_" + + logTimestamp + "_" + dataset.getId() + + ".log"; FileHandler fileHandler; try { @@ -751,25 +839,61 @@ public void globusUpload(JsonObject jsonData, Dataset dataset, String httpReques logger.fine("json: " + JsonUtil.prettyPrint(jsonData)); - globusLogger.info("Globus upload initiated"); - String taskIdentifier = jsonData.getString("taskIdentifier"); + + globusLogger.info("Globus upload initiated, task "+taskIdentifier); + GlobusEndpoint endpoint = getGlobusEndpoint(dataset); - GlobusTaskState taskState = getTask(endpoint.getClientToken(), taskIdentifier, globusLogger); - String ruleId = getRuleId(endpoint, taskState.getOwner_id(), "rw"); - logger.fine("Found rule: " + ruleId); + + // The first check on the status of the task: + // It is important to be careful here, and not give up on the task + // prematurely if anything goes wrong during this initial api call! + + GlobusTaskState taskState = null; + + int retriesLimit = 3; + int retries = 0; + + while (taskState == null && retries++ < retriesLimit) { + // Sleep for 3 seconds before the first check, to make sure the + // task is properly registered on the remote end. Then we'll sleep + // for 3 sec. more if needed, up to retriesLimit number of times. + try { + Thread.sleep(3000); + } catch (InterruptedException ie) { + logger.warning("caught an Interrupted Exception while trying to sleep for 3 sec. in globusDownload()"); + } + + try { + taskState = getTask(endpoint.getClientToken(), taskIdentifier, globusLogger); + } catch (ExpiredTokenException ete) { + // We have just obtained this token milliseconds ago - this shouldn't + // really happen - ? + } + } + + if (taskState != null) { + globusLogger.info("Task owner: "+taskState.getOwner_id()+", human-friendly owner name: "+taskState.getOwner_string()); + } + + String ruleId = taskState != null + ? getRuleId(endpoint, taskState.getOwner_id(), "rw") + : null; + if (ruleId != null) { + logger.fine("Found rule: " + ruleId); Long datasetId = rulesCache.getIfPresent(ruleId); if (datasetId != null) { - // Will not delete rule + // This will only "invalidate" the local cache entry, will not + // delete or invalidate the actual Globus rule rulesCache.invalidate(ruleId); } + } else { + // Something is wrong - the rule should be there + logger.warning("ruleId not found for download taskId: " + taskIdentifier); } - // Wait before first check - Thread.sleep(5000); - if (FeatureFlags.GLOBUS_USE_EXPERIMENTAL_ASYNC_FRAMEWORK.enabled()) { // Save the task information in the database so that the Globus monitoring @@ -806,7 +930,7 @@ public void globusUpload(JsonObject jsonData, Dataset dataset, String httpReques // finish one way or another!) taskState = globusStatusCheck(endpoint, taskIdentifier, globusLogger); // @todo null check, or make sure it's never null - String taskStatus = GlobusUtil.getTaskStatus(taskState); + String taskStatus = GlobusUtil.getCompletedTaskStatus(taskState); boolean taskSuccess = GlobusUtil.isTaskCompleted(taskState); @@ -924,13 +1048,12 @@ private void processCompletedUploadTask(Dataset dataset, datasetSvc.removeDatasetLocks(dataset, DatasetLock.Reason.EditInProgress); } } + + // @todo: this appears to be redundant - it was already deleted above - ? if (ruleId != null) { deletePermission(ruleId, dataset, myLogger); myLogger.info("Removed upload permission: " + ruleId); } - //if (fileHandler != null) { - // fileHandler.close(); - //} } @@ -1157,13 +1280,21 @@ private void processUploadedFiles(JsonArray filesJsonArray, Dataset dataset, Aut } @Asynchronous - public void globusDownload(String jsonData, Dataset dataset, User authUser) throws MalformedURLException { + public void globusDownload(JsonObject jsonObject, Dataset dataset, User authUser) throws MalformedURLException { - String logTimestamp = logFormatter.format(new Date()); + Date startDate = new Date(); + + // the logger initialization method may need to be moved into the GlobusUtil + // eventually, for both this and the monitoring service to use + String logTimestamp = logFormatter.format(startDate); Logger globusLogger = Logger.getLogger( - "edu.harvard.iq.dataverse.upload.client.DatasetServiceBean." + "GlobusDownload" + logTimestamp); + "edu.harvard.iq.dataverse.globus.GlobusServiceBean." + "Globus" + + GlobusTaskInProgress.TaskType.DOWNLOAD + logTimestamp); - String logFileName = System.getProperty("com.sun.aas.instanceRoot") + File.separator + "logs" + File.separator + "globusDownload_id_" + dataset.getId() + "_" + logTimestamp + String logFileName = System.getProperty("com.sun.aas.instanceRoot") + + File.separator + "logs" + + File.separator + "globus" + GlobusTaskInProgress.TaskType.DOWNLOAD + "_" + + logTimestamp + "_" + dataset.getId() + ".log"; FileHandler fileHandler; boolean fileHandlerSuceeded; @@ -1181,73 +1312,104 @@ public void globusDownload(String jsonData, Dataset dataset, User authUser) thro } else { globusLogger = logger; } - - globusLogger.info("Starting a globusDownload "); - - JsonObject jsonObject = null; - try { - jsonObject = JsonUtil.getJsonObject(jsonData); - } catch (Exception jpe) { - jpe.printStackTrace(); - globusLogger.log(Level.SEVERE, "Error parsing dataset json. Json: {0}", jsonData); - // TODO: stop the process after this parsing exception. - } - + String taskIdentifier = jsonObject.getString("taskIdentifier"); + globusLogger.info("Starting monitoring a globus download task "+taskIdentifier); + GlobusEndpoint endpoint = getGlobusEndpoint(dataset); logger.fine("Endpoint path: " + endpoint.getBasePath()); // If the rules_cache times out, the permission will be deleted. Presumably that // doesn't affect a // globus task status check - GlobusTaskState task = getTask(endpoint.getClientToken(), taskIdentifier, globusLogger); - String ruleId = getRuleId(endpoint, task.getOwner_id(), "r"); + + // The first check on the status of the task: + // It is important to be careful here, and not give up on the task + // prematurely if anything goes wrong during this initial api call! + + GlobusTaskState taskState = null; + + int retriesLimit = 3; + int retries = 0; + + while (taskState == null && retries++ < retriesLimit) { + // Sleep for 3 seconds before the first check, to make sure the + // task is properly registered on the remote end. Then we'll sleep + // for 3 sec. more if needed, up to retriesLimit number of times. + try { + Thread.sleep(3000); + } catch (InterruptedException ie) { + logger.warning("caught an Interrupted Exception while trying to sleep for 3 sec. in globusDownload()"); + } + + try { + taskState = getTask(endpoint.getClientToken(), taskIdentifier, globusLogger); + } catch (ExpiredTokenException ete) { + // We have just obtained this token milliseconds ago - this shouldn't + // really happen (?) + endpoint = getGlobusEndpoint(dataset); + } + } + + if (taskState != null) { + globusLogger.info("Task owner: "+taskState.getOwner_id()+", human-friendly owner name: "+taskState.getOwner_string()); + } + + String ruleId = taskState != null + ? getRuleId(endpoint, taskState.getOwner_id(), "r") + : null; + if (ruleId != null) { logger.fine("Found rule: " + ruleId); Long datasetId = rulesCache.getIfPresent(ruleId); if (datasetId != null) { - logger.fine("Deleting from cache: rule: " + ruleId); - // Will not delete rule + logger.fine("Deleting from local cache: rule: " + ruleId); + // This will only "invalidate" the local cache entry, will not + // delete or invalidate the actual Globus rule rulesCache.invalidate(ruleId); } } else { - // Something is wrong - the rule should be there (a race with the cache timing - // out?) - logger.warning("ruleId not found for taskId: " + taskIdentifier); + // Something is wrong - the rule should be there + logger.warning("ruleId not found for download taskId: " + taskIdentifier); + // We will proceed monitoring the transfer, even though the ruleId + // is null at the moment. The whole point of monitoring a download + // task is to remove the rule on the collection side once it's done, + // and we will need the rule id for that. But let's hope this was a + // temporary condition and we will eventually be able to look it up. } - task = globusStatusCheck(endpoint, taskIdentifier, globusLogger); - // @todo null check? - String taskStatus = GlobusUtil.getTaskStatus(task); + + if (FeatureFlags.GLOBUS_USE_EXPERIMENTAL_ASYNC_FRAMEWORK.enabled()) { + + // Save the task information in the database so that the Globus monitoring + // service can continue checking on its progress. + + GlobusTaskInProgress taskInProgress = new GlobusTaskInProgress(taskIdentifier, + GlobusTaskInProgress.TaskType.DOWNLOAD, + dataset, + endpoint.getClientToken(), + authUser instanceof AuthenticatedUser ? (AuthenticatedUser)authUser : null, + ruleId, + new Timestamp(startDate.getTime())); + em.persist(taskInProgress); + + fileHandler.close(); - // Transfer is done (success or failure) so delete the rule - if (ruleId != null) { - logger.fine("Deleting: rule: " + ruleId); - deletePermission(ruleId, dataset, globusLogger); + // return and forget; the Monitoring Service will pick it up on + // the next scheduled check + return; } - - if (taskStatus.startsWith("FAILED") || taskStatus.startsWith("INACTIVE")) { - String comment = "Reason : " + taskStatus.split("#")[1] + "
Short Description : " - + taskStatus.split("#")[2]; - if (authUser != null && authUser instanceof AuthenticatedUser) { - userNotificationService.sendNotification((AuthenticatedUser) authUser, new Timestamp(new Date().getTime()), - UserNotification.Type.GLOBUSDOWNLOADCOMPLETEDWITHERRORS, dataset.getId(), comment, true); - } - - globusLogger.info("Globus task failed during download process: "+comment); - } else if (authUser != null && authUser instanceof AuthenticatedUser) { - boolean taskSkippedFiles = (task.getSkip_source_errors() == null) ? false : task.getSkip_source_errors(); - if (!taskSkippedFiles) { - userNotificationService.sendNotification((AuthenticatedUser) authUser, - new Timestamp(new Date().getTime()), UserNotification.Type.GLOBUSDOWNLOADCOMPLETED, - dataset.getId()); - } else { - userNotificationService.sendNotification((AuthenticatedUser) authUser, - new Timestamp(new Date().getTime()), UserNotification.Type.GLOBUSDOWNLOADCOMPLETEDWITHERRORS, - dataset.getId(), ""); - } - } + // Old implementation: + // globusStatusCheck will loop continuously, until it determines that the + // task has completed - i.e., for the duration of the task + taskState = globusStatusCheck(endpoint, taskIdentifier, globusLogger); + + processCompletedDownloadTask(taskState, + authUser instanceof AuthenticatedUser ? (AuthenticatedUser)authUser : null, + dataset, + ruleId, + globusLogger); } Executor executor = Executors.newFixedThreadPool(10); @@ -1258,6 +1420,7 @@ private GlobusTaskState globusStatusCheck(GlobusEndpoint endpoint, String taskId GlobusTaskState task = null; int pollingInterval = SystemConfig.getIntLimitFromStringOrDefault( settingsSvc.getValueForKey(SettingsServiceBean.Key.GlobusPollingInterval), 50); + int retries = 0; do { try { globusLogger.info("checking globus transfer task " + taskId); @@ -1265,13 +1428,33 @@ private GlobusTaskState globusStatusCheck(GlobusEndpoint endpoint, String taskId // Call the (centralized) Globus API to check on the task state/status: task = getTask(endpoint.getClientToken(), taskId, globusLogger); taskCompleted = GlobusUtil.isTaskCompleted(task); + if (taskCompleted) { + if (task.getStatus().equalsIgnoreCase("ACTIVE")) { + retries++; + // isTaskCompleted() method assumes that a task that is still + // being reported as "ACTIVE" is in fact completed if its + // "nice status" is neither "ok" nor "queued". If that is the + // case, we want it to happen at least 3 times in a row before + // we give up on this task. + globusLogger.fine("Task is reported as \"ACTIVE\", but appears completed (nice_status: " + + task.getNice_status() + + ", " + + retries + + " attempts so far"); + taskCompleted = retries > 3; + } + } else { + retries = 0; + } + } catch (Exception ex) { + logger.warning("Caught exception while in globusStatusCheck(); stack trace below"); ex.printStackTrace(); } } while (!taskCompleted); - globusLogger.info("globus transfer task completed successfully"); + globusLogger.info("globus transfer task completed"); return task; } @@ -1422,14 +1605,27 @@ private GlobusEndpoint getGlobusEndpoint(DvObject dvObject) { logger.fine("endpointId: " + endpointId); - String globusToken = GlobusAccessibleStore.getGlobusToken(driverId); - - AccessToken accessToken = GlobusServiceBean.getClientToken(globusToken); - String clientToken = accessToken.getOtherTokens().get(0).getAccessToken(); + String clientToken = getClientTokenForDataset(dataset); endpoint = new GlobusEndpoint(endpointId, clientToken, directoryPath); return endpoint; } + + public String getClientTokenForDataset(Dataset dataset) { + String clientToken = null; + + String driverId = dataset.getEffectiveStorageDriverId(); + String globusBasicToken = GlobusAccessibleStore.getGlobusToken(driverId); + AccessToken accessToken = GlobusServiceBean.getClientToken(globusBasicToken); + if (accessToken != null) { + clientToken = accessToken.getOtherTokens().get(0).getAccessToken(); + // the above should be safe null pointers-wise, + // if the accessToken returned is not null; i.e., if it should be + // well-structured, with at least one non-null "Other Token", etc. + // - otherwise a null would be returned. + } + return clientToken; + } // This helper method is called from the Download terms/guestbook/etc. popup, // when the user clicks the "ok" button. We use it, instead of calling @@ -1439,6 +1635,9 @@ private GlobusEndpoint getGlobusEndpoint(DvObject dvObject) { public void writeGuestbookAndStartTransfer(GuestbookResponse guestbookResponse, boolean doNotSaveGuestbookResponse) { PrimeFaces.current().executeScript("PF('guestbookAndTermsPopup').hide()"); + + logger.fine("Inside writeGuestbookAndStartTransfer; " + (doNotSaveGuestbookResponse ? "doNotSaveGuestbookResponse" : "DOsaveGuestbookResponse")); + guestbookResponse.setEventType(GuestbookResponse.DOWNLOAD); ApiToken apiToken = null; @@ -1452,6 +1651,8 @@ public void writeGuestbookAndStartTransfer(GuestbookResponse guestbookResponse, apiToken.setTokenString(privUrl.getToken()); } + logger.fine("selected file ids from the guestbookResponse: " +guestbookResponse.getSelectedFileIds()); + DataFile df = guestbookResponse.getDataFile(); if (df != null) { logger.fine("Single datafile case for writeGuestbookAndStartTransfer"); @@ -1495,6 +1696,15 @@ public List findAllOngoingTasks() { return em.createQuery("select object(o) from GlobusTaskInProgress as o order by o.startTime", GlobusTaskInProgress.class).getResultList(); } + public List findAllOngoingTasks(GlobusTaskInProgress.TaskType taskType) { + return em.createQuery("select object(o) from GlobusTaskInProgress as o where o.taskType=:taskType order by o.startTime", GlobusTaskInProgress.class).setParameter("taskType", taskType).getResultList(); + } + + public boolean isRuleInUseByOtherTasks(String ruleId) { + Long numTask = em.createQuery("select count(o) from GlobusTaskInProgress as o where o.ruleId=:ruleId", Long.class).setParameter("ruleId", ruleId).getSingleResult(); + return numTask > 1; + } + public void deleteTask(GlobusTaskInProgress task) { GlobusTaskInProgress mergedTask = em.merge(task); em.remove(mergedTask); @@ -1504,39 +1714,125 @@ public List findExternalUploadsByTaskId(String tas return em.createNamedQuery("ExternalFileUploadInProgress.findByTaskId").setParameter("taskId", taskId).getResultList(); } - public void processCompletedTask(GlobusTaskInProgress globusTask, boolean taskSuccess, String taskStatus, Logger taskLogger) { + public void processCompletedTask(GlobusTaskInProgress globusTask, + GlobusTaskState taskState, + boolean taskSuccess, + String taskStatus, + boolean deleteRule, + Logger taskLogger) { + String ruleId = globusTask.getRuleId(); Dataset dataset = globusTask.getDataset(); AuthenticatedUser authUser = globusTask.getLocalUser(); - if (authUser == null) { - // @todo log error message; do nothing - return; - } + + switch (globusTask.getTaskType()) { - if (GlobusTaskInProgress.TaskType.UPLOAD.equals(globusTask.getTaskType())) { - List fileUploadsInProgress = findExternalUploadsByTaskId(globusTask.getTaskId()); + case UPLOAD: + List fileUploadsInProgress = findExternalUploadsByTaskId(globusTask.getTaskId()); - if (fileUploadsInProgress == null || fileUploadsInProgress.size() < 1) { - // @todo log error message; do nothing - // (will this ever happen though?) - return; - } + if (fileUploadsInProgress == null || fileUploadsInProgress.size() < 1) { + // @todo log error message; do nothing + // (will this ever happen though?) + return; + } + + JsonArrayBuilder filesJsonArrayBuilder = Json.createArrayBuilder(); - JsonArrayBuilder filesJsonArrayBuilder = Json.createArrayBuilder(); + for (ExternalFileUploadInProgress pendingFile : fileUploadsInProgress) { + String jsonInfoString = pendingFile.getFileInfo(); + JsonObject fileObject = JsonUtil.getJsonObject(jsonInfoString); + filesJsonArrayBuilder.add(fileObject); + } - for (ExternalFileUploadInProgress pendingFile : fileUploadsInProgress) { - String jsonInfoString = pendingFile.getFileInfo(); - JsonObject fileObject = JsonUtil.getJsonObject(jsonInfoString); - filesJsonArrayBuilder.add(fileObject); + JsonArray filesJsonArray = filesJsonArrayBuilder.build(); + + processCompletedUploadTask(dataset, filesJsonArray, authUser, ruleId, taskLogger, taskSuccess, taskStatus); + break; + + case DOWNLOAD: + + processCompletedDownloadTask(taskState, authUser, dataset, ruleId, deleteRule, taskLogger); + break; + + default: + logger.warning("Unknown or null TaskType passed to processCompletedTask()"); + } + + } + + private void processCompletedDownloadTask(GlobusTaskState taskState, + AuthenticatedUser authUser, + Dataset dataset, + String ruleId, + Logger taskLogger) { + processCompletedDownloadTask(taskState, authUser, dataset, ruleId, true, taskLogger); + } + + private void processCompletedDownloadTask(GlobusTaskState taskState, + AuthenticatedUser authUser, + Dataset dataset, + String ruleId, + boolean deleteRule, + Logger taskLogger) { + // The only thing to do on completion of a remote download + // transfer is to delete the permission ACL that Dataverse + // had negotiated for the user before the task was initialized ... + + GlobusEndpoint endpoint = getGlobusEndpoint(dataset); + + if (endpoint != null) { + if (deleteRule) { + if (ruleId == null) { + // It is possible that, for whatever reason, we failed to look up + // the rule id when the monitoring of the task was initiated - but + // now that it has completed, let's try and look it up again: + getRuleId(endpoint, taskState.getOwner_id(), "r"); + } + + if (ruleId != null) { + deletePermission(ruleId, endpoint, taskLogger); + } } + } + + String taskStatus = GlobusUtil.getCompletedTaskStatus(taskState); + + // ... plus log the outcome and send any notifications: + if (taskStatus.startsWith("FAILED") || taskStatus.startsWith("INACTIVE")) { + // Outright, unambiguous failure: + String comment = "Reason : " + taskStatus.split("#")[1] + "
Short Description : " + + taskStatus.split("#")[2]; + taskLogger.info("Globus task failed during download process: " + comment); - JsonArray filesJsonArray = filesJsonArrayBuilder.build(); + sendNotification(authUser, UserNotification.Type.GLOBUSDOWNLOADCOMPLETEDWITHERRORS, dataset.getId(), comment); - processCompletedUploadTask(dataset, filesJsonArray, authUser, ruleId, taskLogger, taskSuccess, taskStatus); } else { - // @todo eventually, extend this async. framework to handle Glonus downloads as well - } + // Success, total or partial + boolean taskSkippedFiles = (taskState == null || taskState.getSkip_source_errors() == null) ? false : taskState.getSkip_source_errors(); + + if (!taskSkippedFiles) { + taskLogger.info("Globus task completed successfully"); + + sendNotification(authUser, UserNotification.Type.GLOBUSDOWNLOADCOMPLETED, dataset.getId(), ""); + } else { + taskLogger.info("Globus task completed with partial success (skip source errors)"); + sendNotification(authUser, UserNotification.Type.GLOBUSDOWNLOADCOMPLETEDWITHERRORS, dataset.getId(), ""); + } + } + } + + private void sendNotification(AuthenticatedUser authUser, + UserNotification.Type type, + Long datasetId, + String comment) { + if (authUser != null) { + userNotificationService.sendNotification(authUser, + new Timestamp(new Date().getTime()), + type, + datasetId, + comment); + } } public void deleteExternalUploadRecords(String taskId) { diff --git a/src/main/java/edu/harvard/iq/dataverse/globus/GlobusTaskInProgress.java b/src/main/java/edu/harvard/iq/dataverse/globus/GlobusTaskInProgress.java index 8644bca6143..f1bbd99fa67 100644 --- a/src/main/java/edu/harvard/iq/dataverse/globus/GlobusTaskInProgress.java +++ b/src/main/java/edu/harvard/iq/dataverse/globus/GlobusTaskInProgress.java @@ -86,7 +86,8 @@ public String toString() { @JoinColumn private AuthenticatedUser user; - @Column(nullable=false) + // @Column(nullable=false) @todo we will need a flyway script in order to make + // this field nullable private String ruleId; @JoinColumn(nullable = false) diff --git a/src/main/java/edu/harvard/iq/dataverse/globus/GlobusTaskState.java b/src/main/java/edu/harvard/iq/dataverse/globus/GlobusTaskState.java index b5db20d46c1..08dfb02b0e3 100644 --- a/src/main/java/edu/harvard/iq/dataverse/globus/GlobusTaskState.java +++ b/src/main/java/edu/harvard/iq/dataverse/globus/GlobusTaskState.java @@ -16,6 +16,7 @@ public class GlobusTaskState { private boolean skip_source_errors; private String nice_status; private String nice_status_short_description; + private String owner_string; public String getDestination_endpoint_display_name() { return destination_endpoint_display_name; @@ -92,5 +93,12 @@ public void setNice_status(String nice_status) { public String getNice_status_short_description() { return nice_status_short_description; } - + + public void setOwner_string(String owner_string) { + this.owner_string = owner_string; + } + + public String getOwner_string() { + return owner_string; + } } \ No newline at end of file diff --git a/src/main/java/edu/harvard/iq/dataverse/globus/GlobusUtil.java b/src/main/java/edu/harvard/iq/dataverse/globus/GlobusUtil.java index 652898591ac..c9bb7fb6c4e 100644 --- a/src/main/java/edu/harvard/iq/dataverse/globus/GlobusUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/globus/GlobusUtil.java @@ -36,14 +36,21 @@ public static boolean isTaskCompleted(GlobusTaskState task) { String status = task.getStatus(); if (status != null) { if (status.equalsIgnoreCase("ACTIVE")) { - if (task.getNice_status().equalsIgnoreCase("ok") - || task.getNice_status().equalsIgnoreCase("queued")) { + // We will take "ACTIVE" for face value, i.e., assume that + // this really means that the task is still ongoing. + // (prior to 6.6 we used to assume that was only the case in + // combination with "nice_status" being "ok" or "queued") + /*if (task.getNice_status().equalsIgnoreCase("ok") + || task.getNice_status().equalsIgnoreCase("queued")) {*/ return false; - } } + return true; } } - return true; + // if either task, or status is null - it likely indicates that there + // was an error contacting the task management api, and NOT that it has + // completed one way or another + return false; } public static boolean isTaskSucceeded(GlobusTaskState task) { @@ -53,9 +60,6 @@ public static boolean isTaskSucceeded(GlobusTaskState task) { if (status != null) { status = status.toUpperCase(); if (status.equals("ACTIVE") || status.startsWith("FAILED") || status.startsWith("INACTIVE")) { - // There are cases where a failed task may still be showing - // as "ACTIVE". But it is definitely safe to assume that it - // has not completed *successfully*. return false; } return true; @@ -67,7 +71,7 @@ public static boolean isTaskSucceeded(GlobusTaskState task) { * Produces a human-readable Status label of a completed task * @param GlobusTaskState task - a looked-up state of a task as reported by Globus API */ - public static String getTaskStatus(GlobusTaskState task) { + public static String getCompletedTaskStatus(GlobusTaskState task) { String status = null; if (task != null) { status = task.getStatus(); @@ -75,7 +79,7 @@ public static String getTaskStatus(GlobusTaskState task) { // The task is in progress but is not ok or queued // (L.A.) I think the assumption here is that this method is called // exclusively on tasks that have already completed. So that's why - // it is safe to assume that "ACTIVE" means "FAILED". + // the code below assumes that "ACTIVE" means "FAILED". if (status.equalsIgnoreCase("ACTIVE")) { status = "FAILED" + "#" + task.getNice_status() + "#" + task.getNice_status_short_description(); } else { @@ -86,6 +90,7 @@ public static String getTaskStatus(GlobusTaskState task) { status = "FAILED"; } } else { + // @todo are we sure? status = "FAILED"; } return status; diff --git a/src/main/java/edu/harvard/iq/dataverse/globus/TaskMonitoringServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/globus/TaskMonitoringServiceBean.java index fdb2b222804..63c58f9d422 100644 --- a/src/main/java/edu/harvard/iq/dataverse/globus/TaskMonitoringServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/globus/TaskMonitoringServiceBean.java @@ -1,5 +1,6 @@ package edu.harvard.iq.dataverse.globus; +import edu.harvard.iq.dataverse.Dataset; import edu.harvard.iq.dataverse.settings.JvmSettings; import edu.harvard.iq.dataverse.settings.SettingsServiceBean; import edu.harvard.iq.dataverse.util.SystemConfig; @@ -13,7 +14,9 @@ import java.io.IOException; import java.text.SimpleDateFormat; import java.util.Date; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.concurrent.TimeUnit; import java.util.logging.FileHandler; import java.util.logging.Logger; @@ -38,13 +41,13 @@ public class TaskMonitoringServiceBean { @Resource ManagedScheduledExecutorService scheduler; - @EJB - SystemConfig systemConfig; @EJB SettingsServiceBean settingsSvc; @EJB GlobusServiceBean globusService; + final Map globusClientKeys = new HashMap<>(); + private static final SimpleDateFormat logFormatter = new SimpleDateFormat("yyyy-MM-dd'T'HH-mm-ss"); @PostConstruct @@ -53,43 +56,150 @@ public void init() { logger.info("Starting Globus task monitoring service"); int pollingInterval = SystemConfig.getIntLimitFromStringOrDefault( settingsSvc.getValueForKey(SettingsServiceBean.Key.GlobusPollingInterval), 600); - this.scheduler.scheduleWithFixedDelay(this::checkOngoingTasks, + + // Monitoring service scheduler for ongoing upload tasks: + this.scheduler.scheduleWithFixedDelay(this::checkOngoingUploadTasks, + 0, pollingInterval, + TimeUnit.SECONDS); + + // A separate monitoring service scheduler for ongoing download tasks: + this.scheduler.scheduleWithFixedDelay(this::checkOngoingDownloadTasks, 0, pollingInterval, TimeUnit.SECONDS); + } else { logger.info("Skipping Globus task monitor initialization"); } + + } /** * This method will be executed on a timer-like schedule, continuously - * monitoring all the ongoing external Globus tasks (transfers). + * monitoring all the ongoing external Globus upload tasks (transfers TO remote + * Globus endnodes). */ - public void checkOngoingTasks() { - logger.fine("Performing a scheduled external Globus task check"); - List tasks = globusService.findAllOngoingTasks(); + public void checkOngoingUploadTasks() { + logger.fine("Performing a scheduled external Globus UPLOAD task check"); + List tasks = globusService.findAllOngoingTasks(GlobusTaskInProgress.TaskType.UPLOAD); tasks.forEach(t -> { - FileHandler taskLogHandler = getTaskLogHandler(t); - Logger taskLogger = getTaskLogger(t, taskLogHandler); - - GlobusTaskState retrieved = globusService.getTask(t.getGlobusToken(), t.getTaskId(), taskLogger); + GlobusTaskState retrieved = checkTaskState(t); + if (GlobusUtil.isTaskCompleted(retrieved)) { + FileHandler taskLogHandler = getTaskLogHandler(t); + Logger taskLogger = getTaskLogger(t, taskLogHandler); + // Do our thing, finalize adding the files to the dataset - globusService.processCompletedTask(t, GlobusUtil.isTaskSucceeded(retrieved), GlobusUtil.getTaskStatus(retrieved), taskLogger); + globusService.processCompletedTask(t, retrieved, GlobusUtil.isTaskSucceeded(retrieved), GlobusUtil.getCompletedTaskStatus(retrieved), true, taskLogger); // Whether it finished successfully, or failed in the process, // there's no need to keep monitoring this task, so we can // delete it. //globusService.deleteExternalUploadRecords(t.getTaskId()); globusService.deleteTask(t); + + if (taskLogHandler != null) { + taskLogHandler.close(); + } } - - if (taskLogHandler != null) { - // @todo it should be prudent to cache these loggers and handlers - // between monitoring runs (should be fairly easy to do) - taskLogHandler.close(); + + }); + } + + /** + * This method will be executed on a timer-like schedule, continuously + * monitoring all the ongoing external Globus download tasks (transfers by + * Dataverse users FROM remote, Dataverse-managed Globus endnodes). + */ + public void checkOngoingDownloadTasks() { + logger.fine("Performing a scheduled external Globus DOWNLOAD task check"); + List tasks = globusService.findAllOngoingTasks(GlobusTaskInProgress.TaskType.DOWNLOAD); + + // Unlike with uploads, it is now possible for a user to run several + // download transfers on the same dataset - with several download + // tasks using the same access rule on the corresponding Globus + // pseudofolder. This means that we'll need to be careful not to + // delete any rule, without checking if there are still other + // active tasks using it: + Map rulesInUse = new HashMap<>(); + + tasks.forEach(t -> { + String ruleId = t.getRuleId(); + if (ruleId != null) { + if (rulesInUse.containsKey(ruleId)) { + rulesInUse.put(ruleId, rulesInUse.get(ruleId) + 1); + } else { + rulesInUse.put(ruleId, 1L); + } } }); + + tasks.forEach(t -> { + + GlobusTaskState retrieved = checkTaskState(t); + + if (GlobusUtil.isTaskCompleted(retrieved)) { + FileHandler taskLogHandler = getTaskLogHandler(t); + Logger taskLogger = getTaskLogger(t, taskLogHandler); + + String taskStatus = retrieved == null ? "N/A" : retrieved.getStatus(); + taskLogger.info("Processing completed task " + t.getTaskId() + ", status: " + taskStatus); + + boolean deleteRule = true; + + if (t.getRuleId() == null || rulesInUse.get(t.getRuleId()) > 1) { + taskLogger.info("Access rule " + t.getRuleId() + " is still in use by other tasks."); + deleteRule = false; + rulesInUse.put(t.getRuleId(), rulesInUse.get(t.getRuleId()) - 1); + } else { + taskLogger.info("Access rule " + t.getRuleId() + " is no longer in use by other tasks; will delete."); + } + + globusService.processCompletedTask(t, retrieved, GlobusUtil.isTaskSucceeded(retrieved), GlobusUtil.getCompletedTaskStatus(retrieved), deleteRule, taskLogger); + + // Whether it finished successfully or failed, the entry for the + // task can now be deleted from the database. + globusService.deleteTask(t); + + if (taskLogHandler != null) { + taskLogHandler.close(); + } + } else { + String taskStatus = retrieved == null ? "N/A" : retrieved.getStatus(); + logger.fine("task "+t.getTaskId()+" is still running; " + ", status: " + taskStatus); + } + + }); + } + + private GlobusTaskState checkTaskState(GlobusTaskInProgress task) { + GlobusTaskState retrieved = null; + int attempts = 2; + // we will make an extra attempt to refresh the token and try again + // in the event of an exception indicating the token is stale + + String globusClientToken = getClientTokenForStorageDriver(task.getDataset(), false); + + while (retrieved == null && attempts > 0) { + try { + retrieved = globusService.getTask(globusClientToken, task.getTaskId(), null); + } catch (ExpiredTokenException ete) { + globusClientToken = getClientTokenForStorageDriver(task.getDataset(), true); + } + attempts--; + } + + return retrieved; + } + + private String getClientTokenForStorageDriver(Dataset dataset, boolean forceRefresh) { + String storageDriverId = dataset.getEffectiveStorageDriverId(); + if (globusClientKeys.get(storageDriverId) == null || forceRefresh) { + String clientToken = globusService.getClientTokenForDataset(dataset); + globusClientKeys.put(storageDriverId, clientToken); + } + + return globusClientKeys.get(storageDriverId); } private FileHandler getTaskLogHandler(GlobusTaskInProgress task) { @@ -100,11 +210,14 @@ private FileHandler getTaskLogHandler(GlobusTaskInProgress task) { Date startDate = new Date(task.getStartTime().getTime()); String logTimeStamp = logFormatter.format(startDate); - String logFileName = System.getProperty("com.sun.aas.instanceRoot") + File.separator + "logs" + File.separator + "globusUpload_" + task.getDataset().getId() + "_" + logTimeStamp + String logFileName = System.getProperty("com.sun.aas.instanceRoot") + + File.separator + "logs" + + File.separator + "globus" + task.getTaskType() + "_" + + logTimeStamp + "_" + task.getDataset().getId() + ".log"; FileHandler fileHandler; try { - fileHandler = new FileHandler(logFileName); + fileHandler = new FileHandler(logFileName, true); } catch (IOException | SecurityException ex) { // @todo log this error somehow? fileHandler = null; @@ -120,7 +233,8 @@ private Logger getTaskLogger(GlobusTaskInProgress task, FileHandler logFileHandl String logTimeStamp = logFormatter.format(startDate); Logger taskLogger = Logger.getLogger( - "edu.harvard.iq.dataverse.upload.client.DatasetServiceBean." + "GlobusUpload" + logTimeStamp); + "edu.harvard.iq.dataverse.globus.GlobusServiceBean." + "Globus" + + task.getTaskType() + logTimeStamp); taskLogger.setUseParentHandlers(false); taskLogger.addHandler(logFileHandler); diff --git a/src/main/java/edu/harvard/iq/dataverse/harvest/client/ClientHarvestRun.java b/src/main/java/edu/harvard/iq/dataverse/harvest/client/ClientHarvestRun.java index ba6f5c3dec2..6a85219cc3c 100644 --- a/src/main/java/edu/harvard/iq/dataverse/harvest/client/ClientHarvestRun.java +++ b/src/main/java/edu/harvard/iq/dataverse/harvest/client/ClientHarvestRun.java @@ -6,7 +6,10 @@ package edu.harvard.iq.dataverse.harvest.client; import java.io.Serializable; +import java.util.Arrays; import java.util.Date; + +import edu.harvard.iq.dataverse.util.BundleUtil; import jakarta.persistence.Entity; import jakarta.persistence.GeneratedValue; import jakarta.persistence.GenerationType; @@ -40,13 +43,7 @@ public void setId(Long id) { this.id = id; } - public enum RunResultType { SUCCESS, FAILURE, INPROGRESS, INTERRUPTED }; - - private static String RESULT_LABEL_SUCCESS = "SUCCESS"; - private static String RESULT_LABEL_FAILURE = "FAILED"; - private static String RESULT_LABEL_INPROGRESS = "IN PROGRESS"; - private static String RESULT_DELETE_IN_PROGRESS = "DELETE IN PROGRESS"; - private static String RESULT_LABEL_INTERRUPTED = "INTERRUPTED"; + public enum RunResultType { COMPLETED, COMPLETED_WITH_FAILURES, FAILURE, IN_PROGRESS, INTERRUPTED } @ManyToOne @JoinColumn(nullable = false) @@ -68,36 +65,43 @@ public RunResultType getResult() { public String getResultLabel() { if (harvestingClient != null && harvestingClient.isDeleteInProgress()) { - return RESULT_DELETE_IN_PROGRESS; + return BundleUtil.getStringFromBundle("harvestclients.result.deleteInProgress"); } - - if (isSuccess()) { - return RESULT_LABEL_SUCCESS; + + if (isCompleted()) { + return BundleUtil.getStringFromBundle("harvestclients.result.completed"); + } else if (isCompletedWithFailures()) { + return BundleUtil.getStringFromBundle("harvestclients.result.completedWithFailures"); } else if (isFailed()) { - return RESULT_LABEL_FAILURE; + return BundleUtil.getStringFromBundle("harvestclients.result.failure"); } else if (isInProgress()) { - return RESULT_LABEL_INPROGRESS; + return BundleUtil.getStringFromBundle("harvestclients.result.inProgess"); } else if (isInterrupted()) { - return RESULT_LABEL_INTERRUPTED; + return BundleUtil.getStringFromBundle("harvestclients.result.interrupted"); } return null; } public String getDetailedResultLabel() { if (harvestingClient != null && harvestingClient.isDeleteInProgress()) { - return RESULT_DELETE_IN_PROGRESS; + return BundleUtil.getStringFromBundle("harvestclients.result.deleteInProgress"); } - if (isSuccess() || isInterrupted()) { + if (isCompleted() || isCompletedWithFailures() || isInterrupted()) { String resultLabel = getResultLabel(); - - resultLabel = resultLabel.concat("; "+harvestedDatasetCount+" harvested, "); - resultLabel = resultLabel.concat(deletedDatasetCount+" deleted, "); - resultLabel = resultLabel.concat(failedDatasetCount+" failed."); + + String details = BundleUtil.getStringFromBundle("harvestclients.result.details", Arrays.asList( + harvestedDatasetCount.toString(), + deletedDatasetCount.toString(), + failedDatasetCount.toString() + )); + if(details != null) { + resultLabel = resultLabel + "; " + details; + } return resultLabel; } else if (isFailed()) { - return RESULT_LABEL_FAILURE; + return BundleUtil.getStringFromBundle("harvestclients.result.failure"); } else if (isInProgress()) { - return RESULT_LABEL_INPROGRESS; + return BundleUtil.getStringFromBundle("harvestclients.result.inProgess"); } return null; } @@ -106,12 +110,20 @@ public void setResult(RunResultType harvestResult) { this.harvestResult = harvestResult; } - public boolean isSuccess() { - return RunResultType.SUCCESS == harvestResult; + public boolean isCompleted() { + return RunResultType.COMPLETED == harvestResult; + } + + public void setCompleted() { + harvestResult = RunResultType.COMPLETED; + } + + public boolean isCompletedWithFailures() { + return RunResultType.COMPLETED_WITH_FAILURES == harvestResult; } - public void setSuccess() { - harvestResult = RunResultType.SUCCESS; + public void setCompletedWithFailures() { + harvestResult = RunResultType.COMPLETED_WITH_FAILURES; } public boolean isFailed() { @@ -123,12 +135,12 @@ public void setFailed() { } public boolean isInProgress() { - return RunResultType.INPROGRESS == harvestResult || + return RunResultType.IN_PROGRESS == harvestResult || (harvestResult == null && startTime != null && finishTime == null); } public void setInProgress() { - harvestResult = RunResultType.INPROGRESS; + harvestResult = RunResultType.IN_PROGRESS; } public boolean isInterrupted() { diff --git a/src/main/java/edu/harvard/iq/dataverse/harvest/client/FastGetRecord.java b/src/main/java/edu/harvard/iq/dataverse/harvest/client/FastGetRecord.java index 402d0d8ef91..4f4fd39eb98 100644 --- a/src/main/java/edu/harvard/iq/dataverse/harvest/client/FastGetRecord.java +++ b/src/main/java/edu/harvard/iq/dataverse/harvest/client/FastGetRecord.java @@ -81,8 +81,8 @@ public class FastGetRecord { private static final String XML_METADATA_TAG_OPEN = "<"+XML_METADATA_TAG+">"; private static final String XML_METADATA_TAG_CLOSE = ""; private static final String XML_OAI_PMH_CLOSING_TAGS = ""; - private static final String XML_XMLNS_XSI_ATTRIBUTE_TAG = "xmlns:xsi="; - private static final String XML_XMLNS_XSI_ATTRIBUTE = " "+XML_XMLNS_XSI_ATTRIBUTE_TAG+"\"http://www.w3.org/2001/XMLSchema-instance\">"; + public static final String XML_XMLNS_XSI_ATTRIBUTE_TAG = "xmlns:xsi="; + public static final String XML_XMLNS_XSI_ATTRIBUTE = " "+XML_XMLNS_XSI_ATTRIBUTE_TAG+"\"http://www.w3.org/2001/XMLSchema-instance\">"; private static final String XML_COMMENT_START = ""; diff --git a/src/main/java/edu/harvard/iq/dataverse/harvest/client/HarvesterServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/harvest/client/HarvesterServiceBean.java index e0b5c2dfbfb..d7830991cff 100644 --- a/src/main/java/edu/harvard/iq/dataverse/harvest/client/HarvesterServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/harvest/client/HarvesterServiceBean.java @@ -33,10 +33,14 @@ import org.apache.commons.lang3.mutable.MutableBoolean; import org.xml.sax.SAXException; +import io.gdcc.xoai.model.oaipmh.results.Record; import io.gdcc.xoai.model.oaipmh.results.record.Header; +import io.gdcc.xoai.model.oaipmh.results.record.Metadata; import edu.harvard.iq.dataverse.EjbDataverseEngine; import edu.harvard.iq.dataverse.api.imports.ImportServiceBean; import edu.harvard.iq.dataverse.engine.command.DataverseRequest; +import static edu.harvard.iq.dataverse.harvest.client.FastGetRecord.XML_XMLNS_XSI_ATTRIBUTE_TAG; +import static edu.harvard.iq.dataverse.harvest.client.FastGetRecord.XML_XMLNS_XSI_ATTRIBUTE; import edu.harvard.iq.dataverse.harvest.client.oai.OaiHandler; import edu.harvard.iq.dataverse.harvest.client.oai.OaiHandlerException; import edu.harvard.iq.dataverse.search.IndexServiceBean; @@ -163,7 +167,7 @@ public void doHarvest(DataverseRequest dataverseRequest, Long harvestingClientId try { if (harvestingClientConfig.isHarvestingNow()) { - hdLogger.log(Level.SEVERE, "Cannot start harvest, client " + harvestingClientConfig.getName() + " is already harvesting."); + hdLogger.log(Level.SEVERE, String.format("Cannot start harvest, client %s is already harvesting.", harvestingClientConfig.getName())); } else { harvestingClientService.resetHarvestInProgress(harvestingClientId); @@ -176,9 +180,16 @@ public void doHarvest(DataverseRequest dataverseRequest, Long harvestingClientId } else { throw new IOException("Unsupported harvest type"); } - harvestingClientService.setHarvestSuccess(harvestingClientId, new Date(), harvestedDatasetIds.size(), failedIdentifiers.size(), deletedIdentifiers.size()); - hdLogger.log(Level.INFO, "COMPLETED HARVEST, server=" + harvestingClientConfig.getArchiveUrl() + ", metadataPrefix=" + harvestingClientConfig.getMetadataPrefix()); - hdLogger.log(Level.INFO, "Datasets created/updated: " + harvestedDatasetIds.size() + ", datasets deleted: " + deletedIdentifiers.size() + ", datasets failed: " + failedIdentifiers.size()); + + if (failedIdentifiers.isEmpty()) { + harvestingClientService.setHarvestCompleted(harvestingClientId, new Date(), harvestedDatasetIds.size(), failedIdentifiers.size(), deletedIdentifiers.size()); + hdLogger.log(Level.INFO, String.format("\"COMPLETED HARVEST, server=%s, metadataPrefix=%s", harvestingClientConfig.getArchiveUrl(), harvestingClientConfig.getMetadataPrefix())); + } else { + harvestingClientService.setHarvestCompletedWithFailures(harvestingClientId, new Date(), harvestedDatasetIds.size(), failedIdentifiers.size(), deletedIdentifiers.size()); + hdLogger.log(Level.INFO, String.format("\"COMPLETED HARVEST WITH FAILURES, server=%s, metadataPrefix=%s", harvestingClientConfig.getArchiveUrl(), harvestingClientConfig.getMetadataPrefix())); + } + + hdLogger.log(Level.INFO, String.format("Datasets created/updated: %s, datasets deleted: %s, datasets failed: %s", harvestedDatasetIds.size(), deletedIdentifiers.size(), failedIdentifiers.size())); } } catch (StopHarvestException she) { @@ -232,48 +243,128 @@ private void harvestOAI(DataverseRequest dataverseRequest, HarvestingClient harv httpClient = HttpClient.newBuilder().followRedirects(HttpClient.Redirect.ALWAYS).build(); try { - for (Iterator
idIter = oaiHandler.runListIdentifiers(); idIter.hasNext();) { - // Before each iteration, check if this harvesting job needs to be aborted: - if (checkIfStoppingJob(harvestingClient)) { - throw new StopHarvestException("Harvesting stopped by external request"); - } + if (harvestingClient.isUseListRecords()) { + harvestOAIviaListRecords(oaiHandler, dataverseRequest, harvestingClient, httpClient, failedIdentifiers, deletedIdentifiers, harvestedDatasetIds, hdLogger, importCleanupLog); + } else { + // The default behavior is to use ListIdentifiers: + harvestOAIviaListIdentifiers(oaiHandler, dataverseRequest, harvestingClient, httpClient, failedIdentifiers, deletedIdentifiers, harvestedDatasetIds, hdLogger, importCleanupLog); + } + } catch (OaiHandlerException e) { + throw new IOException("Failed to run ListIdentifiers: " + e.getMessage()); + } - Header h = idIter.next(); - String identifier = h.getIdentifier(); - Date dateStamp = Date.from(h.getDatestamp()); - - hdLogger.info("processing identifier: " + identifier + ", date: " + dateStamp); - - if (h.isDeleted()) { - hdLogger.info("Deleting harvesting dataset for " + identifier + ", per ListIdentifiers."); + logCompletedOaiHarvest(hdLogger, harvestingClient); - deleteHarvestedDatasetIfExists(identifier, oaiHandler.getHarvestingClient().getDataverse(), dataverseRequest, deletedIdentifiers, hdLogger); - continue; - } + } + + private void harvestOAIviaListIdentifiers(OaiHandler oaiHandler, DataverseRequest dataverseRequest, HarvestingClient harvestingClient, HttpClient httpClient, List failedIdentifiers, List deletedIdentifiers, List harvestedDatasetIds, Logger harvesterLogger, PrintWriter importCleanupLog) throws OaiHandlerException, StopHarvestException { + for (Iterator
idIter = oaiHandler.runListIdentifiers(); idIter.hasNext();) { + // Before each iteration, check if this harvesting job needs to be aborted: + if (checkIfStoppingJob(harvestingClient)) { + throw new StopHarvestException("Harvesting stopped by external request"); + } - MutableBoolean getRecordErrorOccurred = new MutableBoolean(false); + Header h = idIter.next(); + String identifier = h.getIdentifier(); + Date dateStamp = Date.from(h.getDatestamp()); - // Retrieve and process this record with a separate GetRecord call: - - Long datasetId = processRecord(dataverseRequest, hdLogger, importCleanupLog, oaiHandler, identifier, getRecordErrorOccurred, deletedIdentifiers, dateStamp, httpClient); + harvesterLogger.info("ListIdentifiers; processing identifier: " + identifier + ", date: " + dateStamp); + + if (h.isDeleted()) { + harvesterLogger.info("ListIdentifiers; deleting harvesting dataset for " + identifier); + + deleteHarvestedDatasetIfExists(identifier, oaiHandler.getHarvestingClient().getDataverse(), dataverseRequest, deletedIdentifiers, harvesterLogger); + continue; + } + + MutableBoolean getRecordErrorOccurred = new MutableBoolean(false); + + // Retrieve and process this record with a separate GetRecord call: + Long datasetId = processRecord(dataverseRequest, harvesterLogger, importCleanupLog, oaiHandler, identifier, getRecordErrorOccurred, deletedIdentifiers, dateStamp, httpClient); + + if (datasetId != null) { + harvestedDatasetIds.add(datasetId); + } + + if (getRecordErrorOccurred.booleanValue() == true) { + failedIdentifiers.add(identifier); + //can be uncommented out for testing failure handling: + //throw new IOException("Exception occured, stopping harvest"); + } + } + } + + private void harvestOAIviaListRecords(OaiHandler oaiHandler, DataverseRequest dataverseRequest, HarvestingClient harvestingClient, HttpClient httpClient, List failedIdentifiers, List deletedIdentifiers, List harvestedDatasetIds, Logger harvesterLogger, PrintWriter importCleanupLog) throws OaiHandlerException, StopHarvestException { + for (Iterator idIter = oaiHandler.runListRecords(); idIter.hasNext();) { + // Before each iteration, check if this harvesting job needs to be aborted: + if (checkIfStoppingJob(harvestingClient)) { + throw new StopHarvestException("Harvesting stopped by external request"); + } + + Record oaiRecord = idIter.next(); + + Header h = oaiRecord.getHeader(); + String identifier = h.getIdentifier(); + Date dateStamp = Date.from(h.getDatestamp()); + + harvesterLogger.info("ListRecords; processing identifier : " + identifier + ", date: " + dateStamp); + + if (h.isDeleted()) { + harvesterLogger.info("ListRecords; Deleting harvested dataset for " + identifier); + + deleteHarvestedDatasetIfExists(identifier, oaiHandler.getHarvestingClient().getDataverse(), dataverseRequest, deletedIdentifiers, harvesterLogger); + continue; + } + + MutableBoolean getRecordErrorOccurred = new MutableBoolean(false); + + Metadata oaiMetadata = oaiRecord.getMetadata(); + String metadataString = oaiMetadata.getMetadataAsString(); + + Long datasetId = null; + + if (metadataString != null) { + Dataset harvestedDataset = null; - if (datasetId != null) { - harvestedDatasetIds.add(datasetId); + // Some xml header sanitation: + if (!metadataString.matches("^<[^>]*" + XML_XMLNS_XSI_ATTRIBUTE_TAG + ".*")) { + metadataString = metadataString.replaceFirst(">", XML_XMLNS_XSI_ATTRIBUTE); } - - if (getRecordErrorOccurred.booleanValue() == true) { - failedIdentifiers.add(identifier); - //can be uncommented out for testing failure handling: - //throw new IOException("Exception occured, stopping harvest"); + + try { + harvestedDataset = importService.doImportHarvestedDataset(dataverseRequest, + oaiHandler.getHarvestingClient(), + identifier, + oaiHandler.getMetadataPrefix(), + metadataString, + dateStamp, + importCleanupLog); + + harvesterLogger.fine("Harvest Successful for identifier " + identifier); + harvesterLogger.fine("Size of this record: " + metadataString.length()); + } catch (Throwable e) { + logGetRecordException(harvesterLogger, oaiHandler, identifier, e); + } + if (harvestedDataset != null) { + datasetId = harvestedDataset.getId(); } + } else { + // Instead of giving up here, let's try to retrieve and process + // this record with a separate GetRecord call: + datasetId = processRecord(dataverseRequest, harvesterLogger, importCleanupLog, oaiHandler, identifier, getRecordErrorOccurred, deletedIdentifiers, dateStamp, httpClient); } - } catch (OaiHandlerException e) { - throw new IOException("Failed to run ListIdentifiers: " + e.getMessage()); - } - logCompletedOaiHarvest(hdLogger, harvestingClient); + if (datasetId != null) { + harvestedDatasetIds.add(datasetId); + } - } + if (getRecordErrorOccurred.booleanValue() == true) { + failedIdentifiers.add(identifier); + //can be uncommented out for testing failure handling: + //throw new IOException("Exception occured, stopping harvest"); + } + } + } private Long processRecord(DataverseRequest dataverseRequest, Logger hdLogger, PrintWriter importCleanupLog, OaiHandler oaiHandler, String identifier, MutableBoolean recordErrorOccurred, List deletedIdentifiers, Date dateStamp, HttpClient httpClient) { String errMessage = null; diff --git a/src/main/java/edu/harvard/iq/dataverse/harvest/client/HarvestingClient.java b/src/main/java/edu/harvard/iq/dataverse/harvest/client/HarvestingClient.java index 7280b6af129..c8b6ebce232 100644 --- a/src/main/java/edu/harvard/iq/dataverse/harvest/client/HarvestingClient.java +++ b/src/main/java/edu/harvard/iq/dataverse/harvest/client/HarvestingClient.java @@ -7,6 +7,7 @@ import edu.harvard.iq.dataverse.Dataset; import edu.harvard.iq.dataverse.Dataverse; +import edu.harvard.iq.dataverse.util.StringUtil; import java.io.Serializable; import java.text.SimpleDateFormat; import java.util.Arrays; @@ -29,13 +30,12 @@ import jakarta.persistence.NamedQueries; import jakarta.persistence.NamedQuery; import jakarta.persistence.OneToMany; -import jakarta.persistence.OneToOne; import jakarta.persistence.OrderBy; import jakarta.persistence.Table; -import jakarta.persistence.Temporal; -import jakarta.persistence.TemporalType; import jakarta.validation.constraints.Pattern; import jakarta.validation.constraints.Size; +import java.text.ParseException; +import org.apache.commons.lang3.StringUtils; import org.hibernate.validator.constraints.NotBlank; /** @@ -192,6 +192,20 @@ public void setHarvestingUrl(String harvestingUrl) { this.harvestingUrl = harvestingUrl.trim(); } } + + private String sourceName; + + public String getSourceName() { + return sourceName; + } + + public void setSourceName(String sourceName) { + this.sourceName = sourceName; + } + + public String getMetadataSource() { + return StringUtils.isNotBlank(this.sourceName) ? this.sourceName : this.name; + } private String archiveUrl; @@ -214,6 +228,7 @@ public void setArchiveDescription(String archiveDescription) { this.archiveDescription = archiveDescription; } + @Column(columnDefinition="TEXT") private String harvestingSet; public String getHarvestingSet() { @@ -252,6 +267,16 @@ public void setAllowHarvestingMissingCVV(boolean allowHarvestingMissingCVV) { this.allowHarvestingMissingCVV = allowHarvestingMissingCVV; } + private boolean useListRecords; + + public boolean isUseListRecords() { + return useListRecords; + } + + public void setUseListrecords(boolean useListRecords) { + this.useListRecords = useListRecords; + } + private boolean useOaiIdAsPid; public boolean isUseOaiIdentifiersAsPids() { @@ -297,7 +322,7 @@ public ClientHarvestRun getLastSuccessfulRun() { int i = harvestHistory.size() - 1; while (i > -1) { - if (harvestHistory.get(i).isSuccess()) { + if (harvestHistory.get(i).isCompleted() || harvestHistory.get(i).isCompletedWithFailures()) { return harvestHistory.get(i); } i--; @@ -314,7 +339,7 @@ ClientHarvestRun getLastNonEmptyRun() { int i = harvestHistory.size() - 1; while (i > -1) { - if (harvestHistory.get(i).isSuccess()) { + if (harvestHistory.get(i).isCompleted() || harvestHistory.get(i).isCompletedWithFailures()) { if (harvestHistory.get(i).getHarvestedDatasetCount().longValue() > 0 || harvestHistory.get(i).getDeletedDatasetCount().longValue() > 0) { return harvestHistory.get(i); @@ -423,14 +448,55 @@ public String getScheduleDescription() { if (schedulePeriod!=null && schedulePeriod!="") { cal.set(Calendar.HOUR_OF_DAY, scheduleHourOfDay); if (schedulePeriod.equals(this.SCHEDULE_PERIOD_WEEKLY)) { - cal.set(Calendar.DAY_OF_WEEK,scheduleDayOfWeek); - desc="Weekly, "+weeklyFormat.format(cal.getTime()); + cal.set(Calendar.DAY_OF_WEEK,scheduleDayOfWeek + 1); + desc="Weekly, "+weeklyFormat.format(cal.getTime()); } else { desc="Daily, "+dailyFormat.format(cal.getTime()); } } return desc; } + + public void readScheduleDescription(String description) { + this.setScheduled(false); + if (description == null || "none".equals(description)) { + return; + } + + if (StringUtil.nonEmpty(description)) { + Date parsed = null; + Calendar cal = new GregorianCalendar(); + + if (description.startsWith("Weekly, ")) { + description = description.replaceFirst("^Weekly, *", ""); + SimpleDateFormat weeklyFormat = new SimpleDateFormat("E h a"); + try { + parsed = weeklyFormat.parse(description); + cal.setTime(parsed); + this.setScheduled(true); + this.setScheduleDayOfWeek(cal.get(Calendar.DAY_OF_WEEK) - 1); + this.setScheduleHourOfDay(cal.get(Calendar.HOUR_OF_DAY)); + this.setSchedulePeriod(this.SCHEDULE_PERIOD_WEEKLY); + } catch (ParseException pex) { + // return; no need; the client will simply stay unscheduled + } + } else if (description.startsWith("Daily, ")) { + description = description.replaceFirst("^Daily, *", ""); + SimpleDateFormat dailyFormat = new SimpleDateFormat("h a"); + try { + parsed = dailyFormat.parse(description); + cal.setTime(parsed); + this.setScheduled(true); + this.setScheduleHourOfDay(cal.get(Calendar.HOUR_OF_DAY)); + this.setSchedulePeriod(this.SCHEDULE_PERIOD_DAILY); + + } catch (ParseException pex) { + // return; no need; the client will simply stay unscheduled + } + } + } + } + private boolean harvestingNow; public boolean isHarvestingNow() { @@ -476,5 +542,4 @@ public boolean equals(Object object) { public String toString() { return "edu.harvard.iq.dataverse.harvest.client.HarvestingClient[ id=" + id + " ]"; } - } diff --git a/src/main/java/edu/harvard/iq/dataverse/harvest/client/HarvestingClientServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/harvest/client/HarvestingClientServiceBean.java index 7ec6d75a41c..2f76fed1a11 100644 --- a/src/main/java/edu/harvard/iq/dataverse/harvest/client/HarvestingClientServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/harvest/client/HarvestingClientServiceBean.java @@ -164,8 +164,13 @@ public void deleteClient(Long clientId) { } @TransactionAttribute(TransactionAttributeType.REQUIRES_NEW) - public void setHarvestSuccess(Long hcId, Date currentTime, int harvestedCount, int failedCount, int deletedCount) { - recordHarvestJobStatus(hcId, currentTime, harvestedCount, failedCount, deletedCount, ClientHarvestRun.RunResultType.SUCCESS); + public void setHarvestCompleted(Long hcId, Date currentTime, int harvestedCount, int failedCount, int deletedCount) { + recordHarvestJobStatus(hcId, currentTime, harvestedCount, failedCount, deletedCount, ClientHarvestRun.RunResultType.COMPLETED); + } + + @TransactionAttribute(TransactionAttributeType.REQUIRES_NEW) + public void setHarvestCompletedWithFailures(Long hcId, Date currentTime, int harvestedCount, int failedCount, int deletedCount) { + recordHarvestJobStatus(hcId, currentTime, harvestedCount, failedCount, deletedCount, ClientHarvestRun.RunResultType.COMPLETED_WITH_FAILURES); } @TransactionAttribute(TransactionAttributeType.REQUIRES_NEW) diff --git a/src/main/java/edu/harvard/iq/dataverse/harvest/client/oai/OaiHandler.java b/src/main/java/edu/harvard/iq/dataverse/harvest/client/oai/OaiHandler.java index bb3dc06972c..0a71dc4ea15 100644 --- a/src/main/java/edu/harvard/iq/dataverse/harvest/client/oai/OaiHandler.java +++ b/src/main/java/edu/harvard/iq/dataverse/harvest/client/oai/OaiHandler.java @@ -1,9 +1,11 @@ package edu.harvard.iq.dataverse.harvest.client.oai; import io.gdcc.xoai.model.oaipmh.Granularity; +import io.gdcc.xoai.model.oaipmh.results.Record; import io.gdcc.xoai.model.oaipmh.results.record.Header; import io.gdcc.xoai.model.oaipmh.results.MetadataFormat; import io.gdcc.xoai.model.oaipmh.results.Set; +import io.gdcc.xoai.model.oaipmh.verbs.Identify; import io.gdcc.xoai.serviceprovider.ServiceProvider; import io.gdcc.xoai.serviceprovider.exceptions.BadArgumentException; import io.gdcc.xoai.serviceprovider.exceptions.InvalidOAIResponse; @@ -11,6 +13,7 @@ import io.gdcc.xoai.serviceprovider.exceptions.IdDoesNotExistException; import io.gdcc.xoai.serviceprovider.model.Context; import io.gdcc.xoai.serviceprovider.parameters.ListIdentifiersParameters; +import io.gdcc.xoai.serviceprovider.parameters.ListRecordsParameters; import edu.harvard.iq.dataverse.harvest.client.FastGetRecord; import static edu.harvard.iq.dataverse.harvest.client.HarvesterServiceBean.DATAVERSE_PROPRIETARY_METADATA_API; import edu.harvard.iq.dataverse.harvest.client.HarvestingClient; @@ -153,6 +156,7 @@ public ServiceProvider getServiceProvider() throws OaiHandlerException { xoaiClientBuilder = xoaiClientBuilder.withCustomHeaders(getCustomHeaders()); } context.withOAIClient(xoaiClientBuilder.build()); + context.withSaveUnparsedMetadata(); serviceProvider = new ServiceProvider(context); } @@ -258,6 +262,16 @@ public Iterator
runListIdentifiers() throws OaiHandlerException { } + public Iterator runListRecords() throws OaiHandlerException { + ListRecordsParameters parameters = buildListRecordsParams(); + try { + return getServiceProvider().listRecords(parameters); + } catch (BadArgumentException bae) { + throw new OaiHandlerException("BadArgumentException thrown when attempted to run ListRecords"); + } + + } + public FastGetRecord runGetRecord(String identifier, HttpClient httpClient) throws OaiHandlerException { if (StringUtils.isEmpty(this.baseOaiUrl)) { throw new OaiHandlerException("Attempted to execute GetRecord without server URL specified."); @@ -288,6 +302,27 @@ private ListIdentifiersParameters buildListIdentifiersParams() throws OaiHandler } mip.withMetadataPrefix(metadataPrefix); + if (this.fromDate != null) { + Identify identify = runIdentify(); + mip.withGranularity(identify.getGranularity().toString()); + mip.withFrom(this.fromDate.toInstant()); + } + + if (!StringUtils.isEmpty(this.setName)) { + mip.withSetSpec(this.setName); + } + + return mip; + } + + private ListRecordsParameters buildListRecordsParams() throws OaiHandlerException { + ListRecordsParameters mip = ListRecordsParameters.request(); + + if (StringUtils.isEmpty(this.metadataPrefix)) { + throw new OaiHandlerException("Attempted to create a ListRecords request without metadataPrefix specified"); + } + mip.withMetadataPrefix(metadataPrefix); + if (this.fromDate != null) { mip.withFrom(this.fromDate.toInstant()); } @@ -311,10 +346,13 @@ public String getProprietaryDataverseMetadataURL(String identifier) { return requestURL.toString(); } - public void runIdentify() { - // not implemented yet - // (we will need it, both for validating the remote server, - // and to learn about its extended capabilities) + public Identify runIdentify() throws OaiHandlerException { + ServiceProvider sp = getServiceProvider(); + try { + return sp.identify(); + } catch (InvalidOAIResponse ior) { + throw new OaiHandlerException("No valid response received from the OAI server."); + } } public Map makeCustomHeaders(String headersString) { diff --git a/src/main/java/edu/harvard/iq/dataverse/ingest/IngestMessageBean.java b/src/main/java/edu/harvard/iq/dataverse/ingest/IngestMessageBean.java index f56fe608a52..c46599e83b5 100644 --- a/src/main/java/edu/harvard/iq/dataverse/ingest/IngestMessageBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/ingest/IngestMessageBean.java @@ -23,6 +23,7 @@ import edu.harvard.iq.dataverse.*; import edu.harvard.iq.dataverse.authorization.AuthenticationServiceBean; import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser; +import edu.harvard.iq.dataverse.search.IndexServiceBean; import edu.harvard.iq.dataverse.util.BundleUtil; import java.sql.Timestamp; @@ -60,6 +61,7 @@ public class IngestMessageBean implements MessageListener { @EJB IngestServiceBean ingestService; @EJB UserNotificationServiceBean userNotificationService; @EJB AuthenticationServiceBean authenticationServiceBean; + @EJB IndexServiceBean indexService; public IngestMessageBean() { @@ -111,6 +113,7 @@ public void onMessage(Message message) { // and "mixed success and failure" emails. Now we never list successfully // ingested files so this line is commented out. // sbIngestedFiles.append(String.format("
  • %s
    • ", datafile.getCurrentName())); + indexService.asyncIndexDataset(datafile.getOwner(), true); } else { logger.warning("Error occurred during ingest job for file id " + datafile_id + "!"); sbIngestedFiles.append(String.format("
  • %s
    • ", datafile.getCurrentName())); diff --git a/src/main/java/edu/harvard/iq/dataverse/ingest/IngestableDataChecker.java b/src/main/java/edu/harvard/iq/dataverse/ingest/IngestableDataChecker.java index fa83552a9ec..4752022b570 100644 --- a/src/main/java/edu/harvard/iq/dataverse/ingest/IngestableDataChecker.java +++ b/src/main/java/edu/harvard/iq/dataverse/ingest/IngestableDataChecker.java @@ -143,13 +143,29 @@ public String[] getTestFormatSet() { return this.testFormatSet; } + /*ToDo + * Rather than making these tests just methods, perhaps they could be implemented as + * classes inheriting a common interface. In addition to the existing ~test*format methods, + * the interface could include a method indicating whether the test requires + * the whole file or, if not, how many bytes are needed. That would make it easier to + * decide whether to use the test on direct/remote uploads (where retrieving a big file may not be worth it, + * but retrieving the 42 bytes needed for a stata check or the ~491 bytes needed for a por check) could be. + * + * Could also add a method to indicate which mimetypes the test can identify/refine which + * might make it possible to replace FileUtil.useRecognizedType(String, String) at some point. + * + * It might also make sense to make this interface broader than just the current ingestable types, + * e.g. to support the NetCDF, graphML and other checks in the same framework. (Some of these might only + * support using a file rather than a bytebuffer though.) + */ + // test methods start here ------------------------------------------------ /** * test this byte buffer against SPSS-SAV spec * * */ - public String testSAVformat(MappedByteBuffer buff) { + public String testSAVformat(ByteBuffer buff) { String result = null; buff.rewind(); boolean DEBUG = false; @@ -192,7 +208,7 @@ public String testSAVformat(MappedByteBuffer buff) { * test this byte buffer against STATA DTA spec * */ - public String testDTAformat(MappedByteBuffer buff) { + public String testDTAformat(ByteBuffer buff) { String result = null; buff.rewind(); boolean DEBUG = false; @@ -311,7 +327,7 @@ public String testDTAformat(MappedByteBuffer buff) { * test this byte buffer against SAS Transport(XPT) spec * */ - public String testXPTformat(MappedByteBuffer buff) { + public String testXPTformat(ByteBuffer buff) { String result = null; buff.rewind(); boolean DEBUG = false; @@ -359,7 +375,7 @@ public String testXPTformat(MappedByteBuffer buff) { * test this byte buffer against SPSS Portable (POR) spec * */ - public String testPORformat(MappedByteBuffer buff) { + public String testPORformat(ByteBuffer buff) { String result = null; buff.rewind(); boolean DEBUG = false; @@ -525,7 +541,7 @@ public String testPORformat(MappedByteBuffer buff) { * test this byte buffer against R data file * */ - public String testRDAformat(MappedByteBuffer buff) { + public String testRDAformat(ByteBuffer buff) { String result = null; buff.rewind(); @@ -607,11 +623,10 @@ public String testRDAformat(MappedByteBuffer buff) { // public instance methods ------------------------------------------------ public String detectTabularDataFormat(File fh) { - boolean DEBUG = false; - String readableFormatType = null; + FileChannel srcChannel = null; FileInputStream inp = null; - + try { // set-up a FileChannel instance for a given file object inp = new FileInputStream(fh); @@ -621,63 +636,7 @@ public String detectTabularDataFormat(File fh) { // create a read-only MappedByteBuffer MappedByteBuffer buff = srcChannel.map(FileChannel.MapMode.READ_ONLY, 0, buffer_size); - - //this.printHexDump(buff, "hex dump of the byte-buffer"); - - buff.rewind(); - dbgLog.fine("before the for loop"); - for (String fmt : this.getTestFormatSet()) { - - // get a test method - Method mthd = testMethods.get(fmt); - //dbgLog.info("mthd: " + mthd.getName()); - - try { - // invoke this method - Object retobj = mthd.invoke(this, buff); - String result = (String) retobj; - - if (result != null) { - dbgLog.fine("result for (" + fmt + ")=" + result); - if (DEBUG) { - out.println("result for (" + fmt + ")=" + result); - } - if (readableFileTypes.contains(result)) { - readableFormatType = result; - } - dbgLog.fine("readableFormatType=" + readableFormatType); - } else { - dbgLog.fine("null was returned for " + fmt + " test"); - if (DEBUG) { - out.println("null was returned for " + fmt + " test"); - } - } - } catch (InvocationTargetException e) { - Throwable cause = e.getCause(); - // added null check because of "homemade.zip" from https://redmine.hmdc.harvard.edu/issues/3273 - if (cause.getMessage() != null) { - err.format(cause.getMessage()); - e.printStackTrace(); - } else { - dbgLog.info("cause.getMessage() was null for " + e); - e.printStackTrace(); - } - } catch (IllegalAccessException e) { - e.printStackTrace(); - } catch (BufferUnderflowException e){ - dbgLog.info("BufferUnderflowException " + e); - e.printStackTrace(); - } - - if (readableFormatType != null) { - break; - } - } - - // help garbage-collect the mapped buffer sooner, to avoid the jvm - // holding onto the underlying file unnecessarily: - buff = null; - + return detectTabularDataFormat(buff); } catch (FileNotFoundException fe) { dbgLog.fine("exception detected: file was not foud"); fe.printStackTrace(); @@ -688,8 +647,73 @@ public String detectTabularDataFormat(File fh) { IOUtils.closeQuietly(srcChannel); IOUtils.closeQuietly(inp); } + return null; + } + + public String detectTabularDataFormat(ByteBuffer buff) { + boolean DEBUG = false; + String readableFormatType = null; + + // this.printHexDump(buff, "hex dump of the byte-buffer"); + + buff.rewind(); + dbgLog.fine("before the for loop"); + for (String fmt : this.getTestFormatSet()) { + + // get a test method + Method mthd = testMethods.get(fmt); + // dbgLog.info("mthd: " + mthd.getName()); + + try { + // invoke this method + Object retobj = mthd.invoke(this, buff); + String result = (String) retobj; + + if (result != null) { + dbgLog.fine("result for (" + fmt + ")=" + result); + if (DEBUG) { + out.println("result for (" + fmt + ")=" + result); + } + if (readableFileTypes.contains(result)) { + readableFormatType = result; + } + dbgLog.fine("readableFormatType=" + readableFormatType); + } else { + dbgLog.fine("null was returned for " + fmt + " test"); + if (DEBUG) { + out.println("null was returned for " + fmt + " test"); + } + } + } catch (InvocationTargetException e) { + Throwable cause = e.getCause(); + // added null check because of "homemade.zip" from + // https://redmine.hmdc.harvard.edu/issues/3273 + if (cause.getMessage() != null) { + err.format(cause.getMessage()); + e.printStackTrace(); + } else { + dbgLog.info("cause.getMessage() was null for " + e); + e.printStackTrace(); + } + } catch (IllegalAccessException e) { + e.printStackTrace(); + } catch (BufferUnderflowException e) { + dbgLog.info("BufferUnderflowException " + e); + e.printStackTrace(); + } + + if (readableFormatType != null) { + break; + } + } + + // help garbage-collect the mapped buffer sooner, to avoid the jvm + // holding onto the underlying file unnecessarily: + buff = null; + return readableFormatType; } + /** * identify the first 5 bytes @@ -737,7 +761,7 @@ private long getBufferSize(FileChannel fileChannel) { return BUFFER_SIZE; } - private int getGzipBufferSize(MappedByteBuffer buff) { + private int getGzipBufferSize(ByteBuffer buff) { int GZIP_BUFFER_SIZE = 120; /* note: diff --git a/src/main/java/edu/harvard/iq/dataverse/license/License.java b/src/main/java/edu/harvard/iq/dataverse/license/License.java index fe19073ab8d..88766cde73b 100644 --- a/src/main/java/edu/harvard/iq/dataverse/license/License.java +++ b/src/main/java/edu/harvard/iq/dataverse/license/License.java @@ -52,22 +52,22 @@ @UniqueConstraint(columnNames = "uri")} ) public class License { - public static String CC0 = "http://creativecommons.org/publicdomain/zero/1.0"; + public static String CC0 = "http://creativecommons.org/publicdomain/zero/1.0"; @Id @GeneratedValue(strategy = GenerationType.IDENTITY) private Long id; - @Column(columnDefinition="TEXT", nullable = false, unique = true) + @Column(columnDefinition = "TEXT", nullable = false, unique = true) private String name; - @Column(columnDefinition="TEXT") + @Column(columnDefinition = "TEXT") private String shortDescription; - @Column(columnDefinition="TEXT", nullable = false, unique = true) + @Column(columnDefinition = "TEXT", nullable = false, unique = true) private String uri; - @Column(columnDefinition="TEXT") + @Column(columnDefinition = "TEXT") private String iconUrl; @Column(nullable = false) @@ -78,8 +78,20 @@ public class License { @Column(nullable = false, columnDefinition = "BIGINT NOT NULL DEFAULT 0") private Long sortOrder; + + @Column(name = "rights_identifier") + private String rightsIdentifier; + + @Column(name = "rights_identifier_scheme") + private String rightsIdentifierScheme; + + @Column(name = "scheme_uri") + private String schemeUri; - @OneToMany(mappedBy="license") + @Column(name = "language_code") + private String languageCode; + + @OneToMany(mappedBy = "license") private List termsOfUseAndAccess; public License() { @@ -186,18 +198,55 @@ public void setSortOrder(Long sortOrder) { this.sortOrder = sortOrder; } + public String getRightsIdentifier() { + return rightsIdentifier; + } + + public void setRightsIdentifier(String rightsIdentifier) { + this.rightsIdentifier = rightsIdentifier; + } + + public String getRightsIdentifierScheme() { + return rightsIdentifierScheme; + } + + public void setRightsIdentifierScheme(String rightsIdentifierScheme) { + this.rightsIdentifierScheme = rightsIdentifierScheme; + } + + public String getSchemeUri() { + return schemeUri; + } + + public void setSchemeUri(String schemeUri) { + this.schemeUri = schemeUri; + } + + public String getLanguageCode() { + return languageCode; + } + + public void setLanguageCode(String languageCode) { + this.languageCode = languageCode; + } + @Override public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; License license = (License) o; - return active == license.active && id.equals(license.id) && name.equals(license.name) && shortDescription.equals(license.shortDescription) && uri.equals(license.uri) && Objects.equals(iconUrl, license.iconUrl) - && Objects.equals(sortOrder, license.sortOrder); + return active == license.active && id.equals(license.id) && name.equals(license.name) + && shortDescription.equals(license.shortDescription) && uri.equals(license.uri) + && Objects.equals(iconUrl, license.iconUrl) && Objects.equals(sortOrder, license.sortOrder) + && Objects.equals(rightsIdentifier, license.rightsIdentifier) + && Objects.equals(rightsIdentifierScheme, license.rightsIdentifierScheme) + && Objects.equals(schemeUri, license.schemeUri); } @Override public int hashCode() { - return Objects.hash(id, name, shortDescription, uri, iconUrl, active, sortOrder); + return Objects.hash(id, name, shortDescription, uri, iconUrl, active, sortOrder, rightsIdentifier, + rightsIdentifierScheme, schemeUri); } @Override @@ -211,7 +260,10 @@ public String toString() { ", active=" + active + ", isDefault=" + isDefault + ", sortOrder=" + sortOrder + + ", rightsIdentifier='" + rightsIdentifier + '\'' + + ", rightsIdentifierScheme='" + rightsIdentifierScheme + '\'' + + ", schemeUri='" + schemeUri + '\'' + '}'; } - + } diff --git a/src/main/java/edu/harvard/iq/dataverse/makedatacount/MakeDataCountProcessState.java b/src/main/java/edu/harvard/iq/dataverse/makedatacount/MakeDataCountProcessState.java index 2241a2c4ca8..f1005af30e2 100644 --- a/src/main/java/edu/harvard/iq/dataverse/makedatacount/MakeDataCountProcessState.java +++ b/src/main/java/edu/harvard/iq/dataverse/makedatacount/MakeDataCountProcessState.java @@ -42,11 +42,14 @@ public String toString() { private MDCProcessState state; @Column(nullable = true) private Timestamp stateChangeTimestamp; + @Column(nullable = true) + private String server; public MakeDataCountProcessState() { } - public MakeDataCountProcessState (String yearMonth, String state) { + public MakeDataCountProcessState (String yearMonth, String state, String server) { this.setYearMonth(yearMonth); this.setState(state); + this.setServer(server); } public void setYearMonth(String yearMonth) throws IllegalArgumentException { @@ -72,4 +75,10 @@ public MDCProcessState getState() { public Timestamp getStateChangeTime() { return stateChangeTimestamp; } + public void setServer(String server) { + this.server = server; + } + public String getServer() { + return server; + } } diff --git a/src/main/java/edu/harvard/iq/dataverse/makedatacount/MakeDataCountProcessStateServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/makedatacount/MakeDataCountProcessStateServiceBean.java index 5d7ec8ff047..74d95cab324 100644 --- a/src/main/java/edu/harvard/iq/dataverse/makedatacount/MakeDataCountProcessStateServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/makedatacount/MakeDataCountProcessStateServiceBean.java @@ -31,10 +31,10 @@ public MakeDataCountProcessState getMakeDataCountProcessState(String yearMonth) return mdcps; } - public MakeDataCountProcessState setMakeDataCountProcessState(String yearMonth, String state) { + public MakeDataCountProcessState setMakeDataCountProcessState(String yearMonth, String state, String server) { MakeDataCountProcessState mdcps = getMakeDataCountProcessState(yearMonth); if (mdcps == null) { - mdcps = new MakeDataCountProcessState(yearMonth, state); + mdcps = new MakeDataCountProcessState(yearMonth, state, server); } else { mdcps.setState(state); } diff --git a/src/main/java/edu/harvard/iq/dataverse/metrics/MetricsServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/metrics/MetricsServiceBean.java index 5bdbeac031d..9090ef05918 100644 --- a/src/main/java/edu/harvard/iq/dataverse/metrics/MetricsServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/metrics/MetricsServiceBean.java @@ -288,11 +288,18 @@ public JsonArray filesTimeSeries(Dataverse d) { + "from (\n" + "select min(to_char(COALESCE(releasetime, createtime), 'YYYY-MM')) as date, filemetadata.id as id\n" + "from datasetversion, filemetadata\n" - + "where datasetversion.id=filemetadata.datasetversion_id\n" - + "and versionstate='RELEASED' \n" - + "and dataset_id in (select dataset.id from dataset, dvobject where dataset.id=dvobject.id\n" + + "where datasetversion.id = filemetadata.datasetversion_id\n" + + "and datasetversion.versionstate = 'RELEASED'\n" + + "and dataset_id in (select dataset.id from dataset, dvobject where dataset.id = dvobject.id\n" + "and dataset.harvestingclient_id IS NULL and publicationdate is not null\n " + ((d == null) ? ")" : "and dvobject.owner_id in (" + getCommaSeparatedIdStringForSubtree(d, "Dataverse") + "))\n ") + + "and filemetadata.id = (\n" + + " select min(fm.id)\n" + + " from filemetadata fm\n" + + " join datasetversion dv on dv.id = fm.datasetversion_id\n" + + " where fm.datafile_id = filemetadata.datafile_id\n" + + " and dv.versionstate = 'RELEASED'\n" + + ")\n" + "group by filemetadata.id) as subq group by subq.date order by date;"); logger.log(Level.FINE, "Metric query: {0}", query); List results = query.getResultList(); @@ -314,8 +321,9 @@ public long filesToMonth(String yyyymm, Dataverse d) { + "select DISTINCT ON (datasetversion.dataset_id) datasetversion.id \n" + "from datasetversion\n" + "join dataset on dataset.id = datasetversion.dataset_id\n" + + "join filemetadata fm on fm.datasetversion_id = datasetversion.id\n" + ((d == null) ? "" : "join dvobject on dvobject.id = dataset.id\n") - + "where versionstate='RELEASED'\n" + + "where datasetversion.versionstate='RELEASED' and filemetadata.datafile_id=fm.datafile_id\n" + ((d == null) ? "" : "and dvobject.owner_id in (" + getCommaSeparatedIdStringForSubtree(d, "Dataverse") + ")\n") + "and date_trunc('month', releasetime) <= to_date('" + yyyymm + "','YYYY-MM')\n" + "and dataset.harvestingclient_id is null\n" @@ -353,12 +361,14 @@ public long filesPastDays(int days, Dataverse d) { public JsonArray filesByType(Dataverse d) { // SELECT DISTINCT df.contenttype, sum(df.filesize) FROM datafile df, dvObject ob where ob.id = df.id and dob.owner_id< group by df.contenttype - // ToDo - published only? Query query = em.createNativeQuery("SELECT DISTINCT df.contenttype, count(df.id), coalesce(sum(df.filesize), 0) " - + " FROM DataFile df, DvObject ob" - + " where ob.id = df.id " - + ((d == null) ? "" : "and ob.owner_id in (" + getCommaSeparatedIdStringForSubtree(d, "Dataset") + ")\n") - + "group by df.contenttype;"); + + " FROM DataFile df " + + " JOIN DvObject ob ON ob.id = df.id " + + " JOIN FileMetadata fm ON fm.datafile_id = df.id " + + " JOIN DatasetVersion dv ON dv.id = fm.datasetversion_id " + + " WHERE dv.versionstate = 'RELEASED' " + + ((d == null) ? "" : "AND ob.owner_id in (" + getCommaSeparatedIdStringForSubtree(d, "Dataset") + ") ") + + "GROUP BY df.contenttype;"); JsonArrayBuilder jab = Json.createArrayBuilder(); try { List results = query.getResultList(); diff --git a/src/main/java/edu/harvard/iq/dataverse/mydata/MyDataFinder.java b/src/main/java/edu/harvard/iq/dataverse/mydata/MyDataFinder.java index 5626a442762..091fbde484e 100644 --- a/src/main/java/edu/harvard/iq/dataverse/mydata/MyDataFinder.java +++ b/src/main/java/edu/harvard/iq/dataverse/mydata/MyDataFinder.java @@ -439,19 +439,6 @@ private boolean runStep1RoleAssignments() { if (results == null) { this.addErrorMessage(BundleUtil.getStringFromBundle("myDataFinder.error.result.null")); return false; - } else if (results.isEmpty()) { - List roleNames = this.rolePermissionHelper.getRoleNamesByIdList(this.filterParams.getRoleIds()); - if ((roleNames == null) || (roleNames.isEmpty())) { - this.addErrorMessage(BundleUtil.getStringFromBundle("myDataFinder.error.result.no.role")); - } else { - final List args = Arrays.asList(StringUtils.join(roleNames, ", ")); - if (roleNames.size() == 1) { - this.addErrorMessage(BundleUtil.getStringFromBundle("myDataFinder.error.result.role.empty", args)); - } else { - this.addErrorMessage(BundleUtil.getStringFromBundle("myDataFinder.error.result.roles.empty", args)); - } - } - return false; } // Iterate through assigned objects, a single object may end up in @@ -485,6 +472,21 @@ private boolean runStep1RoleAssignments() { } directDvObjectIds.add(dvId); } + + if (directDvObjectIds.isEmpty()) { + List roleNames = this.rolePermissionHelper.getRoleNamesByIdList(this.filterParams.getRoleIds()); + if ((roleNames == null) || (roleNames.isEmpty())) { + this.addErrorMessage(BundleUtil.getStringFromBundle("myDataFinder.error.result.no.role")); + } else { + final List args = Arrays.asList(StringUtils.join(roleNames, ", ")); + if (roleNames.size() == 1) { + this.addErrorMessage(BundleUtil.getStringFromBundle("myDataFinder.error.result.role.empty", args)); + } else { + this.addErrorMessage(BundleUtil.getStringFromBundle("myDataFinder.error.result.roles.empty", args)); + } + } + return false; + } return true; } diff --git a/src/main/java/edu/harvard/iq/dataverse/pidproviders/AbstractPidProvider.java b/src/main/java/edu/harvard/iq/dataverse/pidproviders/AbstractPidProvider.java index 250eae7e5fc..df2dc965ed9 100644 --- a/src/main/java/edu/harvard/iq/dataverse/pidproviders/AbstractPidProvider.java +++ b/src/main/java/edu/harvard/iq/dataverse/pidproviders/AbstractPidProvider.java @@ -1,8 +1,10 @@ package edu.harvard.iq.dataverse.pidproviders; +import edu.harvard.iq.dataverse.DataCitation; import edu.harvard.iq.dataverse.DataFile; import edu.harvard.iq.dataverse.Dataset; import edu.harvard.iq.dataverse.DatasetField; +import edu.harvard.iq.dataverse.DatasetVersion; import edu.harvard.iq.dataverse.DvObject; import edu.harvard.iq.dataverse.GlobalId; import edu.harvard.iq.dataverse.util.SystemConfig; @@ -204,6 +206,16 @@ public DvObject generatePid(DvObject dvObject) { + ") doesn't match that of the provider, id: " + getId()); } } + if (dvObject.getSeparator() == null) { + dvObject.setSeparator(getSeparator()); + } else { + if (!dvObject.getSeparator().equals(getSeparator())) { + logger.warning("The separator of the DvObject (" + dvObject.getSeparator() + + ") does not match the configured separator (" + getSeparator() + ")"); + throw new IllegalArgumentException("The separator of the DvObject (" + dvObject.getSeparator() + + ") doesn't match that of the provider, id: " + getId()); + } + } if (dvObject.isInstanceofDataset()) { dvObject.setIdentifier(generateDatasetIdentifier((Dataset) dvObject)); } else { @@ -562,4 +574,10 @@ public boolean updateIdentifier(DvObject dvObject) { //By default, these are the same return publicizeIdentifier(dvObject); } + + /** By default, this is not implemented */ + @Override + public JsonObject getCSLJson(DatasetVersion datasetVersion) { + return new DataCitation(datasetVersion).getCSLJsonFormat(); + } } diff --git a/src/main/java/edu/harvard/iq/dataverse/pidproviders/PidProvider.java b/src/main/java/edu/harvard/iq/dataverse/pidproviders/PidProvider.java index 194a51eeae0..ecc13d17279 100644 --- a/src/main/java/edu/harvard/iq/dataverse/pidproviders/PidProvider.java +++ b/src/main/java/edu/harvard/iq/dataverse/pidproviders/PidProvider.java @@ -1,10 +1,9 @@ package edu.harvard.iq.dataverse.pidproviders; +import edu.harvard.iq.dataverse.DatasetVersion; import edu.harvard.iq.dataverse.DvObject; import edu.harvard.iq.dataverse.GlobalId; import jakarta.json.JsonObject; -import jakarta.json.JsonValue; - import java.util.*; import java.util.logging.Logger; @@ -187,5 +186,15 @@ static boolean checkDOIAuthority(String doiAuthority){ * @return */ public JsonObject getProviderSpecification(); + + /** + * Returns a the Citation Style Language (CSL) JSON representation of the pid. + * For some providers, this could be a call to the service API. For others, it + * may involve generating a local copy. + * + * @param datasetVersion + * @return - the CSL Json for the PID + */ + public JsonObject getCSLJson(DatasetVersion datasetVersion); } \ No newline at end of file diff --git a/src/main/java/edu/harvard/iq/dataverse/pidproviders/PidProviderFactoryBean.java b/src/main/java/edu/harvard/iq/dataverse/pidproviders/PidProviderFactoryBean.java index b01fb5e7eba..c4d6aa4ea21 100644 --- a/src/main/java/edu/harvard/iq/dataverse/pidproviders/PidProviderFactoryBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/pidproviders/PidProviderFactoryBean.java @@ -205,7 +205,7 @@ private void loadProviders() { passphrase); break; case "perma": - String baseUrl = JvmSettings.LEGACY_PERMALINK_BASEURL.lookup(); + String baseUrl = JvmSettings.LEGACY_PERMALINK_BASEURL.lookupOptional().orElse(SystemConfig.getDataverseSiteUrlStatic()); legacy = new PermaLinkPidProvider("legacy", "legacy", authority, shoulder, identifierGenerationStyle, dataFilePidFormat, "", "", baseUrl, PermaLinkPidProvider.SEPARATOR); diff --git a/src/main/java/edu/harvard/iq/dataverse/pidproviders/PidUtil.java b/src/main/java/edu/harvard/iq/dataverse/pidproviders/PidUtil.java index 279f18dcd0e..003b4e3f61c 100644 --- a/src/main/java/edu/harvard/iq/dataverse/pidproviders/PidUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/pidproviders/PidUtil.java @@ -3,6 +3,7 @@ import edu.harvard.iq.dataverse.GlobalId; import edu.harvard.iq.dataverse.pidproviders.doi.AbstractDOIProvider; import edu.harvard.iq.dataverse.pidproviders.handle.HandlePidProvider; +import edu.harvard.iq.dataverse.pidproviders.perma.PermaLinkPidProvider; import edu.harvard.iq.dataverse.util.BundleUtil; import java.io.IOException; import java.io.InputStream; @@ -252,7 +253,12 @@ public static void clearPidProviders() { * Get a PidProvider by protocol/authority/shoulder. */ public static PidProvider getPidProvider(String protocol, String authority, String shoulder) { - return getPidProvider(protocol, authority, shoulder, AbstractPidProvider.SEPARATOR); + switch(protocol) { + case PermaLinkPidProvider.PERMA_PROTOCOL: + return getPidProvider(protocol, authority, shoulder, PermaLinkPidProvider.SEPARATOR); + default: + return getPidProvider(protocol, authority, shoulder, AbstractPidProvider.SEPARATOR); + } } public static PidProvider getPidProvider(String protocol, String authority, String shoulder, String separator) { diff --git a/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java index 8199b7d9c9f..baf8302437d 100644 --- a/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java +++ b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/XmlMetadataTemplate.java @@ -69,8 +69,6 @@ public class XmlMetadataTemplate { public static final String XML_SCHEMA_VERSION = "4.5"; private DoiMetadata doiMetadata; - //QDR - used to get ROR name from ExternalVocabularyValue via pidProvider.get - private PidProvider pidProvider = null; public XmlMetadataTemplate() { } @@ -98,13 +96,6 @@ private void generateXML(DvObject dvObject, OutputStream outputStream) throws XM String language = null; // machine locale? e.g. for Publisher which is global String metadataLanguage = null; // when set, otherwise = language? - //QDR - used to get ROR name from ExternalVocabularyValue via pidProvider.get - GlobalId pid = null; - pid = dvObject.getGlobalId(); - if ((pid == null) && (dvObject instanceof DataFile df)) { - pid = df.getOwner().getGlobalId(); - } - pidProvider = PidUtil.getPidProvider(pid.getProviderId()); XMLStreamWriter xmlw = XMLOutputFactory.newInstance().createXMLStreamWriter(outputStream); xmlw.writeStartElement("resource"); boolean deaccessioned=false; @@ -639,7 +630,7 @@ private void writeEntityElements(XMLStreamWriter xmlw, String elementName, Strin attributeMap.put("schemeURI", "https://ror.org"); attributeMap.put("affiliationIdentifierScheme", "ROR"); - attributeMap.put("affiliationIdentifier", orgName); + attributeMap.put("affiliationIdentifier", affiliation); } XmlWriterUtil.writeFullElementWithAttributes(xmlw, "affiliation", attributeMap, StringEscapeUtils.escapeXml10(orgName)); @@ -695,7 +686,7 @@ private void writeDates(XMLStreamWriter xmlw, DvObject dvObject) throws XMLStrea } else if (dvObject instanceof Dataset d) { DatasetVersion dv = d.getLatestVersionForCopy(); Long versionNumber = dv.getVersionNumber(); - if (versionNumber != null && !(versionNumber.equals(1) && dv.getMinorVersionNumber().equals(0))) { + if (versionNumber != null && !(versionNumber.equals(1L) && dv.getMinorVersionNumber().equals(0L))) { isAnUpdate = true; } releaseDate = dv.getReleaseTime(); @@ -1251,10 +1242,31 @@ private void writeAccessRights(XMLStreamWriter xmlw, DvObject dvObject) throws X } xmlw.writeEndElement(); // xmlw.writeStartElement("rights"); // - + if (license != null) { xmlw.writeAttribute("rightsURI", license.getUri().toString()); - xmlw.writeCharacters(license.getName()); + String label = license.getShortDescription(); + if(StringUtils.isBlank(label)) { + //Use name as a backup in case the license has no short description + label = license.getName(); + } + + if (license.getRightsIdentifier() != null) { + xmlw.writeAttribute("rightsIdentifier", license.getRightsIdentifier()); + } + if (license.getRightsIdentifierScheme() != null) { + xmlw.writeAttribute("rightsIdentifierScheme", license.getRightsIdentifierScheme()); + } + if (license.getSchemeUri() != null) { + xmlw.writeAttribute("schemeURI", license.getSchemeUri()); + } + String langCode = license.getLanguageCode(); + if (StringUtils.isBlank(langCode)) { + langCode = "en"; + } + xmlw.writeAttribute("xml:lang", langCode); + xmlw.writeCharacters(license.getShortDescription()); + } else { xmlw.writeAttribute("rightsURI", DatasetUtil.getLicenseURI(dv)); xmlw.writeCharacters(BundleUtil.getStringFromBundle("license.custom.description")); @@ -1367,7 +1379,13 @@ private void writeDescriptions(XMLStreamWriter xmlw, DvObject dvObject, boolean } } - + String versionNote = dv.getVersionNote(); + if(!StringUtils.isBlank(versionNote)) { + attributes.clear(); + attributes.put("descriptionType", "TechnicalInfo"); + descriptionsWritten = XmlWriterUtil.writeOpenTagIfNeeded(xmlw, "descriptions", descriptionsWritten); + XmlWriterUtil.writeFullElementWithAttributes(xmlw, "description", attributes, versionNote); + } } if (descriptionsWritten) { diff --git a/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/datacite/DataCiteDOIProvider.java b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/datacite/DataCiteDOIProvider.java index b07cd027a01..bb63c4fe08f 100644 --- a/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/datacite/DataCiteDOIProvider.java +++ b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/datacite/DataCiteDOIProvider.java @@ -1,7 +1,12 @@ package edu.harvard.iq.dataverse.pidproviders.doi.datacite; +import java.io.BufferedReader; import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; import java.net.HttpURLConnection; +import java.net.URI; +import java.net.URISyntaxException; import java.net.URL; import java.util.Base64; import java.util.HashMap; @@ -17,6 +22,7 @@ import edu.harvard.iq.dataverse.FileMetadata; import edu.harvard.iq.dataverse.GlobalId; import edu.harvard.iq.dataverse.pidproviders.doi.AbstractDOIProvider; +import edu.harvard.iq.dataverse.util.json.JsonUtil; import jakarta.json.JsonObject; import org.apache.commons.httpclient.HttpException; @@ -342,4 +348,51 @@ public boolean updateIdentifier(DvObject dvObject) { } } + /** Retrieve the CSL JSON - used in cases where this is not directly available from https://doi.org/ + * i.e. for test DOIs and non-findable DOIs. + * + */ + @Override + public JsonObject getCSLJson(DatasetVersion dsv) { + if (dsv.isLatestVersion() && dsv.isReleased()) { + String doi = dsv.getDataset().getGlobalId().asRawIdentifier(); + try { + URL url = new URI(getApiUrl() + "/dois/" + doi).toURL(); + + HttpURLConnection connection = null; + connection = (HttpURLConnection) url.openConnection(); + connection.setRequestMethod("GET"); + String userpass = getUsername() + ":" + getPassword(); + String basicAuth = "Basic " + new String(Base64.getEncoder().encode(userpass.getBytes())); + connection.setRequestProperty("Authorization", basicAuth); + connection.addRequestProperty("Accept", "application/vnd.citationstyles.csl+json"); + int status = connection.getResponseCode(); + if (status != HttpStatus.SC_OK) { + logger.warning("Incorrect Response Status from DataCite: " + status + " : " + + connection.getResponseMessage()); + throw new HttpException("Status: " + status); + } + logger.fine("getCSLJson status for " + doi + ": " + status); + try (BufferedReader in = new BufferedReader( + new InputStreamReader((InputStream) connection.getContent()))) { + String cslString = ""; + String current; + while ((current = in.readLine()) != null) { + cslString += current; + } + logger.fine(cslString); + JsonObject csl = JsonUtil.getJsonObject(cslString); + return csl; + } catch (IOException e) { + logger.log(Level.WARNING, "Error reading DataCite response when getting CSL JSON for " + doi, e); + return super.getCSLJson(dsv); + } + } catch (IOException | URISyntaxException e) { + logger.log(Level.WARNING, "Unable to get CSL JSON for " + doi, e); + return super.getCSLJson(dsv); + } + } else { + return super.getCSLJson(dsv); + } + } } diff --git a/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/fake/FakeDOIProvider.java b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/fake/FakeDOIProvider.java index a967fb40620..023b766f2ac 100644 --- a/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/fake/FakeDOIProvider.java +++ b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/fake/FakeDOIProvider.java @@ -44,8 +44,11 @@ public List getProviderInformation() { } @Override - public String createIdentifier(DvObject dvo) throws Throwable { - return "fakeIdentifier"; + public String createIdentifier(DvObject dvObject) throws Throwable { + if(dvObject.getIdentifier() == null || dvObject.getIdentifier().isEmpty() ){ + dvObject = generatePid(dvObject); + } + return dvObject.getIdentifier(); } @Override diff --git a/src/main/java/edu/harvard/iq/dataverse/search/AbstractSolrClientService.java b/src/main/java/edu/harvard/iq/dataverse/search/AbstractSolrClientService.java new file mode 100644 index 00000000000..1ae236d348f --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/search/AbstractSolrClientService.java @@ -0,0 +1,51 @@ +package edu.harvard.iq.dataverse.search; + +import java.io.IOException; +import java.util.logging.Logger; + +import org.apache.solr.client.solrj.SolrClient; + +import edu.harvard.iq.dataverse.settings.JvmSettings; +import edu.harvard.iq.dataverse.util.SystemConfig; +import jakarta.ejb.EJB; + +/** + * Generics methods for Solr clients implementations + * + * @author jeromeroucou + */ +public abstract class AbstractSolrClientService { + private static final Logger logger = Logger.getLogger(AbstractSolrClientService.class.getCanonicalName()); + + @EJB + SystemConfig systemConfig; + + public abstract void init(); + public abstract void close(); + public abstract SolrClient getSolrClient(); + public abstract void setSolrClient(SolrClient solrClient); + + public void close(SolrClient solrClient) { + if (solrClient != null) { + try { + solrClient.close(); + } catch (IOException e) { + logger.warning("Solr closing error: " + e); + } + solrClient = null; + } + } + + public void reInitialize() { + close(); + init(); + } + + public String getSolrUrl() { + // Get from MPCONFIG. Might be configured by a sysadmin or simply return the + // default shipped with resources/META-INF/microprofile-config.properties. + final String protocol = JvmSettings.SOLR_PROT.lookup(); + final String path = JvmSettings.SOLR_PATH.lookup(); + return protocol + "://" + this.systemConfig.getSolrHostColonPort() + path; + } +} diff --git a/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java index 4efd339ee46..a8e6c0661d7 100644 --- a/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java @@ -1,8 +1,34 @@ package edu.harvard.iq.dataverse.search; -import edu.harvard.iq.dataverse.*; +import edu.harvard.iq.dataverse.ControlledVocabularyValue; +import edu.harvard.iq.dataverse.DataFile; +import edu.harvard.iq.dataverse.DataFileServiceBean; +import edu.harvard.iq.dataverse.DataFileTag; +import edu.harvard.iq.dataverse.Dataset; +import edu.harvard.iq.dataverse.DatasetField; +import edu.harvard.iq.dataverse.DatasetFieldCompoundValue; +import edu.harvard.iq.dataverse.DatasetFieldConstant; +import edu.harvard.iq.dataverse.DatasetFieldServiceBean; +import edu.harvard.iq.dataverse.DatasetFieldType; +import edu.harvard.iq.dataverse.DatasetFieldValue; +import edu.harvard.iq.dataverse.DatasetFieldValueValidator; +import edu.harvard.iq.dataverse.DatasetLinkingServiceBean; +import edu.harvard.iq.dataverse.DatasetServiceBean; +import edu.harvard.iq.dataverse.DatasetVersion; import edu.harvard.iq.dataverse.DatasetVersion.VersionState; +import edu.harvard.iq.dataverse.DatasetVersionFilesServiceBean; +import edu.harvard.iq.dataverse.DatasetVersionServiceBean; +import edu.harvard.iq.dataverse.Dataverse; +import edu.harvard.iq.dataverse.DataverseLinkingServiceBean; +import edu.harvard.iq.dataverse.DataverseServiceBean; +import edu.harvard.iq.dataverse.DvObject; import edu.harvard.iq.dataverse.DvObject.DType; +import edu.harvard.iq.dataverse.DvObjectServiceBean; +import edu.harvard.iq.dataverse.Embargo; +import edu.harvard.iq.dataverse.FileMetadata; +import edu.harvard.iq.dataverse.GlobalId; +import edu.harvard.iq.dataverse.PermissionServiceBean; +import edu.harvard.iq.dataverse.Retention; import edu.harvard.iq.dataverse.authorization.AuthenticationServiceBean; import edu.harvard.iq.dataverse.authorization.providers.builtin.BuiltinUserServiceBean; import edu.harvard.iq.dataverse.batch.util.LoggingUtil; @@ -27,6 +53,8 @@ import java.sql.Timestamp; import java.text.SimpleDateFormat; import java.time.LocalDate; +import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeParseException; import java.util.ArrayList; import java.util.Calendar; import java.util.Collection; @@ -44,9 +72,8 @@ import java.util.function.Function; import java.util.logging.Level; import java.util.logging.Logger; +import java.util.regex.Pattern; import java.util.stream.Collectors; -import jakarta.annotation.PostConstruct; -import jakarta.annotation.PreDestroy; import jakarta.ejb.AsyncResult; import jakarta.ejb.Asynchronous; import jakarta.ejb.EJB; @@ -63,11 +90,9 @@ import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.StringUtils; -import org.apache.solr.client.solrj.SolrClient; import org.apache.solr.client.solrj.SolrQuery; import org.apache.solr.client.solrj.SolrQuery.SortClause; import org.apache.solr.client.solrj.SolrServerException; -import org.apache.solr.client.solrj.impl.HttpSolrClient; import org.apache.solr.client.solrj.response.QueryResponse; import org.apache.solr.client.solrj.response.UpdateResponse; import org.apache.solr.common.SolrDocument; @@ -122,16 +147,15 @@ public class IndexServiceBean { @EJB SettingsServiceBean settingsService; @EJB - SolrClientService solrClientService; + SolrClientService solrClientService; // only for query index on Solr + @EJB + SolrClientIndexService solrClientIndexService; // only for add, update, or remove index on Solr @EJB DataFileServiceBean dataFileService; @EJB VariableServiceBean variableService; - - @EJB - IndexBatchServiceBean indexBatchService; - + @EJB DatasetFieldServiceBean datasetFieldService; @@ -154,37 +178,10 @@ public class IndexServiceBean { private static final String IN_REVIEW_STRING = "In Review"; private static final String DEACCESSIONED_STRING = "Deaccessioned"; public static final String HARVESTED = "Harvested"; - private String rootDataverseName; private Dataverse rootDataverseCached; - SolrClient solrServer; private VariableMetadataUtil variableMetadataUtil; - @PostConstruct - public void init() { - // Get from MPCONFIG. Might be configured by a sysadmin or simply return the default shipped with - // resources/META-INF/microprofile-config.properties. - String protocol = JvmSettings.SOLR_PROT.lookup(); - String path = JvmSettings.SOLR_PATH.lookup(); - - String urlString = protocol + "://" + systemConfig.getSolrHostColonPort() + path; - solrServer = new HttpSolrClient.Builder(urlString).build(); - - rootDataverseName = findRootDataverseCached().getName(); - } - - @PreDestroy - public void close() { - if (solrServer != null) { - try { - solrServer.close(); - } catch (IOException e) { - logger.warning("Solr closing error: " + e); - } - solrServer = null; - } - } - @TransactionAttribute(REQUIRES_NEW) public Future indexDataverseInNewTransaction(Dataverse dataverse) throws SolrServerException, IOException{ return indexDataverse(dataverse, false); @@ -323,7 +320,7 @@ public Future indexDataverse(Dataverse dataverse, boolean processPaths) String status; try { if (dataverse.getId() != null) { - solrClientService.getSolrClient().add(docs); + solrClientIndexService.getSolrClient().add(docs); } else { logger.info("WARNING: indexing of a dataverse with no id attempted"); } @@ -950,32 +947,36 @@ public SolrInputDocuments toSolrDocs(IndexableDataset indexableDataset, Set indexableValues = dsf.getValuesWithoutNaValues().stream() + .filter(s -> intPattern.matcher(s).find()) + .collect(Collectors.toList()); + solrInputDocument.addField(solrFieldSearchable, indexableValues); + if (dsfType.getSolrField().isFacetable()) { + solrInputDocument.addField(solrFieldFacetable, indexableValues); + } + } else if (dsfType.getSolrField().getSolrType().equals(SolrField.SolrType.FLOAT)) { + // same as for integer values, we need to filter invalid float values + List indexableValues = dsf.getValuesWithoutNaValues().stream() + .filter(s -> { + try { + Double.parseDouble(s); + return true; + } catch (NumberFormatException e) { + return false; + } + }) + .collect(Collectors.toList()); + solrInputDocument.addField(solrFieldSearchable, indexableValues); + if (dsfType.getSolrField().isFacetable()) { + solrInputDocument.addField(solrFieldFacetable, indexableValues); + } } else if (dsfType.getSolrField().getSolrType().equals(SolrField.SolrType.DATE)) { + // Solr accepts dates in the ISO-8601 format, e.g. YYYY-MM-DDThh:mm:ssZ, YYYYY-MM-DD, YYYY-MM, YYYY + // See: https://solr.apache.org/guide/solr/latest/indexing-guide/date-formatting-math.html + // If dates have been entered in other formats, we need to skip or convert them + // TODO at the moment we are simply skipping, but converting them would offer more value for search + // For use in facets, we index only the year (YYYY) String dateAsString = ""; if (!dsf.getValues_nondisplay().isEmpty()) { - dateAsString = dsf.getValues_nondisplay().get(0); - } + dateAsString = dsf.getValues_nondisplay().get(0).trim(); + } + logger.fine("date as string: " + dateAsString); + if (dateAsString != null && !dateAsString.isEmpty()) { - SimpleDateFormat inputDateyyyy = new SimpleDateFormat("yyyy", Locale.ENGLISH); - try { - /** - * @todo when bean validation is working we - * won't have to convert strings into dates - */ - logger.fine("Trying to convert " + dateAsString + " to a YYYY date from dataset " + dataset.getId()); - Date dateAsDate = inputDateyyyy.parse(dateAsString); - SimpleDateFormat yearOnly = new SimpleDateFormat("yyyy"); - String datasetFieldFlaggedAsDate = yearOnly.format(dateAsDate); - logger.fine("YYYY only: " + datasetFieldFlaggedAsDate); - // solrInputDocument.addField(solrFieldSearchable, - // Integer.parseInt(datasetFieldFlaggedAsDate)); - solrInputDocument.addField(solrFieldSearchable, datasetFieldFlaggedAsDate); - if (dsfType.getSolrField().isFacetable()) { - // solrInputDocument.addField(solrFieldFacetable, + boolean dateValid = false; + + DateTimeFormatter[] possibleFormats = { + DateTimeFormatter.ISO_INSTANT, + DateTimeFormatter.ofPattern("yyyy-MM-dd"), + DateTimeFormatter.ofPattern("yyyy-MM"), + DateTimeFormatter.ofPattern("yyyy") + }; + for (DateTimeFormatter format : possibleFormats){ + try { + format.parse(dateAsString); + dateValid = true; + } catch (DateTimeParseException e) { + // no-op, date is invalid + } + } + + if (!dateValid) { + logger.fine("couldn't index " + dsf.getDatasetFieldType().getName() + ":" + dsf.getValues() + " because it's not a valid date format according to Solr"); + } else { + SimpleDateFormat inputDateyyyy = new SimpleDateFormat("yyyy", Locale.ENGLISH); + try { + /** + * @todo when bean validation is working we + * won't have to convert strings into dates + */ + logger.fine("Trying to convert " + dateAsString + " to a YYYY date from dataset " + dataset.getId()); + Date dateAsDate = inputDateyyyy.parse(dateAsString); + SimpleDateFormat yearOnly = new SimpleDateFormat("yyyy"); + String datasetFieldFlaggedAsDate = yearOnly.format(dateAsDate); + logger.fine("YYYY only: " + datasetFieldFlaggedAsDate); + // solrInputDocument.addField(solrFieldSearchable, // Integer.parseInt(datasetFieldFlaggedAsDate)); - solrInputDocument.addField(solrFieldFacetable, datasetFieldFlaggedAsDate); + solrInputDocument.addField(solrFieldSearchable, dateAsString); + if (dsfType.getSolrField().isFacetable()) { + // solrInputDocument.addField(solrFieldFacetable, + // Integer.parseInt(datasetFieldFlaggedAsDate)); + solrInputDocument.addField(solrFieldFacetable, datasetFieldFlaggedAsDate); + } + } catch (Exception ex) { + logger.info("unable to convert " + dateAsString + " into YYYY format and couldn't index it (" + dsfType.getName() + ")"); } - } catch (Exception ex) { - logger.info("unable to convert " + dateAsString + " into YYYY format and couldn't index it (" + dsfType.getName() + ")"); } } } else { @@ -1294,11 +1355,15 @@ public SolrInputDocuments toSolrDocs(IndexableDataset indexableDataset, Set variables = fileMetadata.getDataFile().getDataTable().getDataVariables(); + Long observations = fileMetadata.getDataFile().getDataTable().getCaseQuantity(); + datafileSolrInputDocument.addField(SearchFields.OBSERVATIONS, observations); + datafileSolrInputDocument.addField(SearchFields.VARIABLE_COUNT, variables.size()); Map variableMap = null; List variablesByMetadata = variableService.findVarMetByFileMetaId(fileMetadata.getId()); @@ -1682,7 +1752,7 @@ private String addOrUpdateDataset(IndexableDataset indexableDataset, Set d final SolrInputDocuments docs = toSolrDocs(indexableDataset, datafilesInDraftVersion); try { - solrClientService.getSolrClient().add(docs.getDocuments()); + solrClientIndexService.getSolrClient().add(docs.getDocuments()); } catch (SolrServerException | IOException ex) { if (ex.getCause() instanceof SolrServerException) { throw new SolrServerException(ex); @@ -1944,7 +2014,7 @@ private void updatePathForExistingSolrDocs(DvObject object) throws SolrServerExc sid.removeField(SearchFields.SUBTREE); sid.addField(SearchFields.SUBTREE, paths); - UpdateResponse addResponse = solrClientService.getSolrClient().add(sid); + UpdateResponse addResponse = solrClientIndexService.getSolrClient().add(sid); if (object.isInstanceofDataset()) { for (DataFile df : dataset.getFiles()) { solrQuery.setQuery(SearchUtil.constructQuery(SearchFields.ENTITY_ID, df.getId().toString())); @@ -1957,7 +2027,7 @@ private void updatePathForExistingSolrDocs(DvObject object) throws SolrServerExc } sid.removeField(SearchFields.SUBTREE); sid.addField(SearchFields.SUBTREE, paths); - addResponse = solrClientService.getSolrClient().add(sid); + addResponse = solrClientIndexService.getSolrClient().add(sid); } } } @@ -1999,7 +2069,7 @@ public String delete(Dataverse doomed) { logger.fine("deleting Solr document for dataverse " + doomed.getId()); UpdateResponse updateResponse; try { - updateResponse = solrClientService.getSolrClient().deleteById(solrDocIdentifierDataverse + doomed.getId()); + updateResponse = solrClientIndexService.getSolrClient().deleteById(solrDocIdentifierDataverse + doomed.getId()); } catch (SolrServerException | IOException ex) { return ex.toString(); } @@ -2019,7 +2089,7 @@ public String removeSolrDocFromIndex(String doomed) { logger.fine("deleting Solr document: " + doomed); UpdateResponse updateResponse; try { - updateResponse = solrClientService.getSolrClient().deleteById(doomed); + updateResponse = solrClientIndexService.getSolrClient().deleteById(doomed); } catch (SolrServerException | IOException ex) { return ex.toString(); } @@ -2222,7 +2292,7 @@ public List findPermissionsInSolrOnly() throws SearchException { boolean done = false; while (!done) { q.set(CursorMarkParams.CURSOR_MARK_PARAM, cursorMark); - QueryResponse rsp = solrServer.query(q); + QueryResponse rsp = solrClientService.getSolrClient().query(q); String nextCursorMark = rsp.getNextCursorMark(); logger.fine("Next cursor mark (1K entries): " + nextCursorMark); SolrDocumentList list = rsp.getResults(); @@ -2232,7 +2302,7 @@ public List findPermissionsInSolrOnly() throws SearchException { String dtype = dvObjectService.getDtype(id); if (dtype == null) { permissionInSolrOnly.add(docId); - }else if (dtype.equals(DType.Dataset.getDType())) { + } else if (dtype.equals(DType.Dataset.getDType())) { List states = datasetService.getVersionStates(id); if (states != null) { String latestState = states.get(states.size() - 1); @@ -2304,7 +2374,7 @@ private List findDvObjectInSolrOnly(String type) throws SearchException solrQuery.set(CursorMarkParams.CURSOR_MARK_PARAM, cursorMark); QueryResponse rsp = null; try { - rsp = solrServer.query(solrQuery); + rsp = solrClientService.getSolrClient().query(solrQuery); } catch (SolrServerException | IOException ex) { throw new SearchException("Error searching Solr type: " + type, ex); diff --git a/src/main/java/edu/harvard/iq/dataverse/search/SearchFields.java b/src/main/java/edu/harvard/iq/dataverse/search/SearchFields.java index 1f1137016f2..109c17d6ef9 100644 --- a/src/main/java/edu/harvard/iq/dataverse/search/SearchFields.java +++ b/src/main/java/edu/harvard/iq/dataverse/search/SearchFields.java @@ -171,6 +171,7 @@ public class SearchFields { public static final String FILE_CHECKSUM_TYPE = "fileChecksumType"; public static final String FILE_CHECKSUM_VALUE = "fileChecksumValue"; public static final String FILENAME_WITHOUT_EXTENSION = "fileNameWithoutExtension"; + public static final String FILE_RESTRICTED = "fileRestricted"; /** * Indexed as a string so we can facet on it. */ @@ -257,6 +258,7 @@ public class SearchFields { public static final String DATASET_CITATION = "citation"; public static final String DATASET_CITATION_HTML = "citationHtml"; public static final String DATASET_DEACCESSION_REASON = "deaccessionReason"; + public static final String DATASET_VERSION_NOTE = "versionNote"; /** * In contrast to PUBLICATION_YEAR, this field applies only to datasets for more targeted results for just datasets. The format is YYYY (i.e. @@ -270,6 +272,8 @@ more targeted results for just datasets. The format is YYYY (i.e. */ public static final String DATASET_TYPE = "datasetType"; + public static final String OBSERVATIONS = "observations"; + public static final String VARIABLE_COUNT = "variableCount"; public static final String VARIABLE_NAME = "variableName"; public static final String VARIABLE_LABEL = "variableLabel"; public static final String LITERAL_QUESTION = "literalQuestion"; diff --git a/src/main/java/edu/harvard/iq/dataverse/search/SearchServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/search/SearchServiceBean.java index 3fd97d418c0..08ddbab202a 100644 --- a/src/main/java/edu/harvard/iq/dataverse/search/SearchServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/search/SearchServiceBean.java @@ -1,6 +1,7 @@ package edu.harvard.iq.dataverse.search; import edu.harvard.iq.dataverse.*; +import edu.harvard.iq.dataverse.authorization.Permission; import edu.harvard.iq.dataverse.authorization.groups.Group; import edu.harvard.iq.dataverse.authorization.groups.GroupServiceBean; import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser; @@ -18,6 +19,7 @@ import java.util.Calendar; import java.util.Collections; import java.util.Date; +import java.util.EnumSet; import java.util.HashMap; import java.util.LinkedList; import java.util.List; @@ -75,6 +77,8 @@ public class SearchServiceBean { SystemConfig systemConfig; @EJB SolrClientService solrClientService; + @EJB + PermissionServiceBean permissionService; @Inject ThumbnailServiceWrapper thumbnailServiceWrapper; @@ -276,7 +280,7 @@ public SolrQueryResponse search( List datasetFields = datasetFieldService.findAllOrderedById(); Map solrFieldsToHightlightOnMap = new HashMap<>(); if (addHighlights) { - solrQuery.setHighlight(true).setHighlightSnippets(1); + solrQuery.setHighlight(true).setHighlightSnippets(1).setHighlightRequireFieldMatch(true); Integer fragSize = systemConfig.getSearchHighlightFragmentSize(); if (fragSize != null) { solrQuery.setHighlightFragsize(fragSize); @@ -331,9 +335,13 @@ public SolrQueryResponse search( // ----------------------------------- // PERMISSION FILTER QUERY // ----------------------------------- - String permissionFilterQuery = this.getPermissionFilterQuery(dataverseRequest, solrQuery, onlyDatatRelatedToMe, addFacets); - if (!StringUtils.isBlank(permissionFilterQuery)) { - solrQuery.addFilterQuery(permissionFilterQuery); + String permissionFilterQuery = getPermissionFilterQuery(dataverseRequest, solrQuery, onlyDatatRelatedToMe, addFacets); + if (!permissionFilterQuery.isEmpty()) { + String[] filterParts = permissionFilterQuery.split("&q1="); + solrQuery.addFilterQuery(filterParts[0]); + if(filterParts.length > 1 ) { + solrQuery.add("q1", filterParts[1]); + } } /** @@ -616,7 +624,7 @@ public SolrQueryResponse search( if (datasetDescriptions != null) { String firstDatasetDescription = datasetDescriptions.get(0); if (firstDatasetDescription != null) { - solrSearchResult.setDescriptionNoSnippet(firstDatasetDescription); + solrSearchResult.setDescriptionNoSnippet(String.join(" ", datasetDescriptions)); } } solrSearchResult.setDatasetVersionId(datasetVersionId); @@ -677,6 +685,15 @@ public SolrQueryResponse search( logger.info("Exception setting setFileChecksumType: " + ex); } solrSearchResult.setFileChecksumValue((String) solrDocument.getFieldValue(SearchFields.FILE_CHECKSUM_VALUE)); + + if (solrDocument.getFieldValue(SearchFields.FILE_RESTRICTED) != null) { + solrSearchResult.setFileRestricted((Boolean) solrDocument.getFieldValue(SearchFields.FILE_RESTRICTED)); + } + + if (solrSearchResult.getEntity() != null) { + solrSearchResult.setCanDownloadFile(permissionService.hasPermissionsFor(dataverseRequest, solrSearchResult.getEntity(), EnumSet.of(Permission.DownloadFile))); + } + solrSearchResult.setUnf((String) solrDocument.getFieldValue(SearchFields.UNF)); solrSearchResult.setDatasetVersionId(datasetVersionId); List fileCategories = (List) solrDocument.getFieldValues(SearchFields.FILE_TAG); @@ -688,6 +705,10 @@ public SolrQueryResponse search( Collections.sort(tabularDataTags); solrSearchResult.setTabularDataTags(tabularDataTags); } + Long observations = (Long) solrDocument.getFieldValue(SearchFields.OBSERVATIONS); + solrSearchResult.setObservations(observations); + Long tabCount = (Long) solrDocument.getFieldValue(SearchFields.VARIABLE_COUNT); + solrSearchResult.setTabularDataCount(tabCount); String filePID = (String) solrDocument.getFieldValue(SearchFields.FILE_PERSISTENT_ID); if(null != filePID && !"".equals(filePID) && !"".equals("null")) { solrSearchResult.setFilePersistentId(filePID); @@ -1082,9 +1103,9 @@ private String buildPermissionFilterQuery(boolean avoidJoin, String permissionFi String query = (avoidJoin&& !isAllGroups(permissionFilterGroups)) ? SearchFields.PUBLIC_OBJECT + ":" + true : ""; if (permissionFilterGroups != null && !isAllGroups(permissionFilterGroups)) { if (!query.isEmpty()) { - query = "(" + query + " OR " + "{!join from=" + SearchFields.DEFINITION_POINT + " to=id}" + SearchFields.DISCOVERABLE_BY + ":" + permissionFilterGroups + ")"; + query = "(" + query + " OR " + "{!join from=" + SearchFields.DEFINITION_POINT + " to=id v=$q1})&q1=" + SearchFields.DISCOVERABLE_BY + ":" + permissionFilterGroups; } else { - query = "{!join from=" + SearchFields.DEFINITION_POINT + " to=id}" + SearchFields.DISCOVERABLE_BY + ":" + permissionFilterGroups; + query = "{!join from=" + SearchFields.DEFINITION_POINT + " to=id v=$q1}&q1=" + SearchFields.DISCOVERABLE_BY + ":" + permissionFilterGroups; } } return query; diff --git a/src/main/java/edu/harvard/iq/dataverse/search/SolrClientIndexService.java b/src/main/java/edu/harvard/iq/dataverse/search/SolrClientIndexService.java new file mode 100644 index 00000000000..0b7f1aae798 --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/search/SolrClientIndexService.java @@ -0,0 +1,49 @@ +package edu.harvard.iq.dataverse.search; + +import java.util.logging.Logger; + +import org.apache.solr.client.solrj.SolrClient; +import org.apache.solr.client.solrj.impl.ConcurrentUpdateHttp2SolrClient; +import org.apache.solr.client.solrj.impl.Http2SolrClient; + +import jakarta.annotation.PostConstruct; +import jakarta.annotation.PreDestroy; +import jakarta.ejb.Singleton; +import jakarta.inject.Named; + +/** + * Solr client to provide insert/update/delete operations. + * Don't use this service with queries to Solr, use {@link SolrClientService} instead. + */ +@Named +@Singleton +public class SolrClientIndexService extends AbstractSolrClientService { + + private static final Logger logger = Logger.getLogger(SolrClientIndexService.class.getCanonicalName()); + + private SolrClient solrClient; + + @PostConstruct + public void init() { + solrClient = new ConcurrentUpdateHttp2SolrClient.Builder( + getSolrUrl(), new Http2SolrClient.Builder().build()).build(); + } + + @PreDestroy + public void close() { + close(solrClient); + } + + public SolrClient getSolrClient() { + // Should never happen - but? + if (solrClient == null) { + init(); + } + return solrClient; + } + + public void setSolrClient(SolrClient solrClient) { + this.solrClient = solrClient; + } + +} diff --git a/src/main/java/edu/harvard/iq/dataverse/search/SolrClientService.java b/src/main/java/edu/harvard/iq/dataverse/search/SolrClientService.java index b36130de7c8..f9d94b8c6d3 100644 --- a/src/main/java/edu/harvard/iq/dataverse/search/SolrClientService.java +++ b/src/main/java/edu/harvard/iq/dataverse/search/SolrClientService.java @@ -1,65 +1,39 @@ -/* - * To change this license header, choose License Headers in Project Properties. - * To change this template file, choose Tools | Templates - * and open the template in the editor. - */ package edu.harvard.iq.dataverse.search; -import edu.harvard.iq.dataverse.settings.JvmSettings; -import edu.harvard.iq.dataverse.util.SystemConfig; import org.apache.solr.client.solrj.SolrClient; -import org.apache.solr.client.solrj.impl.HttpSolrClient; +import org.apache.solr.client.solrj.impl.Http2SolrClient; import jakarta.annotation.PostConstruct; import jakarta.annotation.PreDestroy; -import jakarta.ejb.EJB; import jakarta.ejb.Singleton; import jakarta.inject.Named; -import java.io.IOException; import java.util.logging.Logger; /** * * @author landreev * - * This singleton is dedicated to initializing the HttpSolrClient used by the - * application to talk to the search engine, and serving it to all the other - * classes that need it. + * This singleton is dedicated to initializing the Http2SolrClient, used by + * the application to talk to the search engine, and serving it to all the + * other classes that need it. * This ensures that we are using one client only - as recommended by the * documentation. */ @Named @Singleton -public class SolrClientService { +public class SolrClientService extends AbstractSolrClientService { private static final Logger logger = Logger.getLogger(SolrClientService.class.getCanonicalName()); - @EJB - SystemConfig systemConfig; - private SolrClient solrClient; @PostConstruct public void init() { - // Get from MPCONFIG. Might be configured by a sysadmin or simply return the default shipped with - // resources/META-INF/microprofile-config.properties. - String protocol = JvmSettings.SOLR_PROT.lookup(); - String path = JvmSettings.SOLR_PATH.lookup(); - - String urlString = protocol + "://" + systemConfig.getSolrHostColonPort() + path; - solrClient = new HttpSolrClient.Builder(urlString).build(); + solrClient = new Http2SolrClient.Builder(getSolrUrl()).build(); } @PreDestroy public void close() { - if (solrClient != null) { - try { - solrClient.close(); - } catch (IOException e) { - logger.warning("Solr closing error: " + e); - } - - solrClient = null; - } + close(solrClient); } public SolrClient getSolrClient() { @@ -73,9 +47,4 @@ public SolrClient getSolrClient() { public void setSolrClient(SolrClient solrClient) { this.solrClient = solrClient; } - - public void reInitialize() { - close(); - init(); - } } diff --git a/src/main/java/edu/harvard/iq/dataverse/search/SolrField.java b/src/main/java/edu/harvard/iq/dataverse/search/SolrField.java index ca9805b6c57..7092a01beb1 100644 --- a/src/main/java/edu/harvard/iq/dataverse/search/SolrField.java +++ b/src/main/java/edu/harvard/iq/dataverse/search/SolrField.java @@ -63,7 +63,7 @@ public enum SolrType { * support range queries) in * https://github.com/IQSS/dataverse/issues/370 */ - STRING("string"), TEXT_EN("text_en"), INTEGER("int"), LONG("long"), DATE("text_en"), EMAIL("text_en"); + STRING("string"), TEXT_EN("text_en"), INTEGER("plong"), FLOAT("pdouble"), DATE("date_range"), EMAIL("text_en"); private String type; diff --git a/src/main/java/edu/harvard/iq/dataverse/search/SolrIndexServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/search/SolrIndexServiceBean.java index e4d885276d0..2b4f08807ef 100644 --- a/src/main/java/edu/harvard/iq/dataverse/search/SolrIndexServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/search/SolrIndexServiceBean.java @@ -46,9 +46,7 @@ public class SolrIndexServiceBean { @EJB DataverseRoleServiceBean rolesSvc; @EJB - IndexServiceBean indexService; - @EJB - SolrClientService solrClientService; + SolrClientIndexService solrClientService; public static String numRowsClearedByClearAllIndexTimes = "numRowsClearedByClearAllIndexTimes"; public static String messageString = "message"; @@ -155,7 +153,15 @@ private List constructDatafileSolrDocs(DataFile dataFile, Map desiredCards = searchPermissionsService.getDesiredCards(dataFile.getOwner()); for (DatasetVersion datasetVersionFileIsAttachedTo : datasetVersionsToBuildCardsFor(dataFile.getOwner())) { boolean cardShouldExist = desiredCards.get(datasetVersionFileIsAttachedTo.getVersionState()); - if (cardShouldExist) { + /* + * Since datasetVersionFileIsAttachedTo should be a draft or the most recent + * released one, it could be more efficient to stop the search through + * FileMetadatas after those two (versus continuing through all prior versions + * as in isInDatasetVersion). Alternately, perhaps filesToReIndexPermissionsFor + * should not combine the list of files for the different datsetversions into a + * single list to start with. + */ + if (cardShouldExist && dataFile.isInDatasetVersion(datasetVersionFileIsAttachedTo)) { String solrIdStart = IndexServiceBean.solrDocIdentifierFile + dataFile.getId(); String solrIdEnd = getDatasetOrDataFileSolrEnding(datasetVersionFileIsAttachedTo.getVersionState()); String solrId = solrIdStart + solrIdEnd; @@ -375,6 +381,12 @@ public IndexResponse indexPermissionsOnSelfAndChildren(long definitionPointId) { * inheritance */ public IndexResponse indexPermissionsOnSelfAndChildren(DvObject definitionPoint) { + + if (definitionPoint == null) { + logger.log(Level.WARNING, "Cannot perform indexPermissionsOnSelfAndChildren with a definitionPoint null"); + return null; + } + List filesToReindexAsBatch = new ArrayList<>(); /** * @todo Re-indexing the definition point itself seems to be necessary diff --git a/src/main/java/edu/harvard/iq/dataverse/search/SolrSearchResult.java b/src/main/java/edu/harvard/iq/dataverse/search/SolrSearchResult.java index 8802555affd..2250a245dab 100644 --- a/src/main/java/edu/harvard/iq/dataverse/search/SolrSearchResult.java +++ b/src/main/java/edu/harvard/iq/dataverse/search/SolrSearchResult.java @@ -97,6 +97,8 @@ public class SolrSearchResult { private String fileMd5; private DataFile.ChecksumType fileChecksumType; private String fileChecksumValue; + private Boolean fileRestricted; + private Boolean canDownloadFile; private String dataverseAlias; private String dataverseParentAlias; private String dataverseParentName; @@ -122,6 +124,8 @@ public class SolrSearchResult { private String harvestingDescription = null; private List fileCategories = null; private List tabularDataTags = null; + private Long tabularDataCount; + private Long observations; private String identifierOfDataverse = null; private String nameOfDataverse = null; @@ -565,7 +569,12 @@ public JsonObjectBuilder json(boolean showRelevance, boolean showEntityIds, bool .add("citationHtml", this.citationHtml) .add("identifier_of_dataverse", this.identifierOfDataverse) .add("name_of_dataverse", this.nameOfDataverse) - .add("citation", this.citation); + .add("citation", this.citation) + .add("restricted", this.fileRestricted) + .add("variables", this.tabularDataCount) + .add("observations", this.observations) + .add("canDownloadFile", this.canDownloadFile); + // Now that nullSafeJsonBuilder has been instatiated, check for null before adding to it! if (showRelevance) { nullSafeJsonBuilder.add("matches", getRelevance()); @@ -579,6 +588,12 @@ public JsonObjectBuilder json(boolean showRelevance, boolean showEntityIds, bool if (!getPublicationStatuses().isEmpty()) { nullSafeJsonBuilder.add("publicationStatuses", getPublicationStatusesAsJSON()); } + if (this.fileCategories != null && !this.fileCategories.isEmpty()) { + nullSafeJsonBuilder.add("categories", JsonPrinter.asJsonArray(this.fileCategories)); + } + if (this.tabularDataTags != null && !this.tabularDataTags.isEmpty()) { + nullSafeJsonBuilder.add("tabularTags", JsonPrinter.asJsonArray(this.tabularDataTags)); + } if (this.entity == null) { @@ -956,6 +971,18 @@ public List getTabularDataTags() { public void setTabularDataTags(List tabularDataTags) { this.tabularDataTags = tabularDataTags; } + public void setTabularDataCount(Long tabularDataCount) { + this.tabularDataCount = tabularDataCount; + } + public Long getTabularDataCount() { + return tabularDataCount; + } + public Long getObservations() { + return observations; + } + public void setObservations(Long observations) { + this.observations = observations; + } public Map getParent() { return parent; @@ -1078,6 +1105,21 @@ public void setFileChecksumValue(String fileChecksumValue) { this.fileChecksumValue = fileChecksumValue; } + public Boolean getFileRestricted() { + return fileRestricted; + } + + public void setFileRestricted(Boolean fileRestricted) { + this.fileRestricted = fileRestricted; + } + public Boolean getCanDownloadFile() { + return canDownloadFile; + } + + public void setCanDownloadFile(Boolean canDownloadFile) { + this.canDownloadFile = canDownloadFile; + } + public String getNameSort() { return nameSort; } diff --git a/src/main/java/edu/harvard/iq/dataverse/settings/FeatureFlags.java b/src/main/java/edu/harvard/iq/dataverse/settings/FeatureFlags.java index 20632c170e4..4326dea6e1c 100644 --- a/src/main/java/edu/harvard/iq/dataverse/settings/FeatureFlags.java +++ b/src/main/java/edu/harvard/iq/dataverse/settings/FeatureFlags.java @@ -33,9 +33,32 @@ public enum FeatureFlags { /** * Enables API authentication via Bearer Token. * @apiNote Raise flag by setting "dataverse.feature.api-bearer-auth" - * @since Dataverse @TODO: + * @since Dataverse 5.14: */ API_BEARER_AUTH("api-bearer-auth"), + /** + * Enables sending the missing user claims in the request JSON provided during OIDC user registration + * (see API endpoint /users/register) when these claims are not returned by the identity provider + * but are necessary for registering the user in Dataverse. + * + *

    The value of this feature flag is only considered when the feature flag + * {@link #API_BEARER_AUTH} is enabled.

    + * + * @apiNote Raise flag by setting "dataverse.feature.api-bearer-auth-provide-missing-claims" + * @since Dataverse @TODO: + */ + API_BEARER_AUTH_PROVIDE_MISSING_CLAIMS("api-bearer-auth-provide-missing-claims"), + /** + * Specifies that Terms of Service acceptance is handled by the IdP, eliminating the need to include + * ToS acceptance boolean parameter (termsAccepted) in the OIDC user registration request body. + * + *

    The value of this feature flag is only considered when the feature flag + * {@link #API_BEARER_AUTH} is enabled.

    + * + * @apiNote Raise flag by setting "dataverse.feature.api-bearer-auth-handle-tos-acceptance-in-idp" + * @since Dataverse @TODO: + */ + API_BEARER_AUTH_HANDLE_TOS_ACCEPTANCE_IN_IDP("api-bearer-auth-handle-tos-acceptance-in-idp"), /** * For published (public) objects, don't use a join when searching Solr. * Experimental! Requires a reindex with the following feature flag enabled, @@ -67,7 +90,6 @@ public enum FeatureFlags { * @since Dataverse 6.3 */ INDEX_HARVESTED_METADATA_SOURCE("index-harvested-metadata-source"), - /** * Dataverse normally deletes all solr documents related to a dataset's files * when the dataset is reindexed. With this flag enabled, additional logic is @@ -109,6 +131,15 @@ public enum FeatureFlags { * @since Dataverse 6.4 */ GLOBUS_USE_EXPERIMENTAL_ASYNC_FRAMEWORK("globus-use-experimental-async-framework"), + /** + * This flag adds a note field to input/display a reason explaining why a version was created. + * + * @apiNote Raise flag by setting + * "dataverse.feature.enable-version-creation-note" + * @since Dataverse 6.5 + */ + VERSION_NOTE("enable-version-note"), + ; final String flag; diff --git a/src/main/java/edu/harvard/iq/dataverse/settings/JvmSettings.java b/src/main/java/edu/harvard/iq/dataverse/settings/JvmSettings.java index d7eea970b8a..bc32e250be5 100644 --- a/src/main/java/edu/harvard/iq/dataverse/settings/JvmSettings.java +++ b/src/main/java/edu/harvard/iq/dataverse/settings/JvmSettings.java @@ -52,6 +52,9 @@ public enum JvmSettings { GUESTBOOK_AT_REQUEST(SCOPE_FILES, "guestbook-at-request"), GLOBUS_CACHE_MAXAGE(SCOPE_FILES, "globus-cache-maxage"), GLOBUS_TASK_MONITORING_SERVER(SCOPE_FILES, "globus-monitoring-server"), + SCOPE_FEATURED_ITEMS(SCOPE_FILES, "featured-items"), + FEATURED_ITEMS_IMAGE_MAXSIZE(SCOPE_FEATURED_ITEMS, "image-maxsize"), + FEATURED_ITEMS_IMAGE_UPLOADS_DIRECTORY(SCOPE_FEATURED_ITEMS, "image-uploads"), //STORAGE DRIVER SETTINGS SCOPE_DRIVER(SCOPE_FILES), @@ -256,6 +259,10 @@ public enum JvmSettings { // STORAGE USE SETTINGS SCOPE_STORAGEUSE(PREFIX, "storageuse"), STORAGEUSE_DISABLE_UPDATES(SCOPE_STORAGEUSE, "disable-storageuse-increments"), + + //CSL CITATION SETTINGS + SCOPE_CSL(PREFIX, "csl"), + CSL_COMMON_STYLES(SCOPE_CSL, "common-styles"), ; private static final String SCOPE_SEPARATOR = "."; diff --git a/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java index b5eb483c2c8..5b0a178969b 100644 --- a/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java @@ -684,7 +684,9 @@ Whether Harvesting (OAI) service is enabled * When ingesting tabular data files, store the generated tab-delimited * files *with* the variable names line up top. */ - StoreIngestedTabularFilesWithVarHeaders + StoreIngestedTabularFilesWithVarHeaders, + + ContactFeedbackMessageSizeLimit ; @Override @@ -749,6 +751,23 @@ public Long getValueForKeyAsLong(Key key){ return null; } + } + + /** + * Attempt to convert the value to an integer + * - Applicable for keys such as MaxFileUploadSizeInBytes + * + * On failure (key not found or string not convertible to a long), returns defaultValue + * @param key + * @param defaultValue + * @return + */ + public Long getValueForKeyAsLong(Key key, Long defaultValue) { + Long val = getValueForKeyAsLong(key); + if (val == null) { + return defaultValue; + } + return val; } /** diff --git a/src/main/java/edu/harvard/iq/dataverse/util/CSLUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/CSLUtil.java new file mode 100644 index 00000000000..fe9e00bd837 --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/util/CSLUtil.java @@ -0,0 +1,128 @@ +package edu.harvard.iq.dataverse.util; + +import java.io.FileNotFoundException; +import java.io.IOException; +import java.net.URL; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Comparator; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.logging.Logger; + +import org.apache.commons.lang3.LocaleUtils; + +import de.undercouch.citeproc.CSL; +import de.undercouch.citeproc.helper.CSLUtils; +import edu.harvard.iq.dataverse.settings.JvmSettings; +import jakarta.faces.model.SelectItem; +import jakarta.faces.model.SelectItemGroup; + +public class CSLUtil { + private static final Logger logger = Logger.getLogger(CSLUtil.class.getName()); + + static ArrayList supportedStyles; + static Map> groupedStylesCache = new ConcurrentHashMap<>(); + + + public static String getDefaultStyle() { + return getCommonStyles()[0]; + } + + public static List getSupportedStyles(String localeCode) { + Locale locale = LocaleUtils.toLocale(localeCode); + if (locale == null) { + locale = Locale.getDefault(); + } + String languageKey = locale.getLanguage(); + + if (groupedStylesCache.containsKey(languageKey)) { + return groupedStylesCache.get(languageKey); + } + + List groupedStyles = new ArrayList<>(); + supportedStyles = new ArrayList<>(); + try { + Set styleSet = CSL.getSupportedStyles(); + // Remove styles starting with "dependent/" + styleSet.removeIf(style -> style.startsWith("dependent/")); + supportedStyles = new ArrayList<>(styleSet); + } catch (IOException e) { + logger.warning("Unable to retrieve supported CSL styles: " + e.getMessage()); + e.printStackTrace(); + } + supportedStyles.sort(Comparator.naturalOrder()); + + String commonTitle = BundleUtil.getStringFromBundle("dataset.cite.cslDialog.commonStyles", locale); + SelectItemGroup commonStyles = new SelectItemGroup(commonTitle); + ArrayList commonArray = new ArrayList<>(); + String[] styleStrings = getCommonStyles(); + Arrays.stream(styleStrings).forEach(style -> { + logger.fine("Found style: " + style); + commonArray.add(new SelectItem(style, style)); + supportedStyles.remove(style); + }); + commonStyles.setSelectItems(commonArray); + + String otherTitle = BundleUtil.getStringFromBundle("dataset.cite.cslDialog.otherStyles", locale); + SelectItemGroup otherStyles = new SelectItemGroup(otherTitle); + + ArrayList otherArray = new ArrayList<>(supportedStyles.size()); + supportedStyles.forEach(style -> { + otherArray.add(new SelectItem(style, style)); + }); + otherStyles.setSelectItems(otherArray); + + groupedStyles.add(commonStyles); + groupedStyles.add(otherStyles); + + groupedStylesCache.put(languageKey, groupedStyles); + return groupedStyles; + } + + /** + * Adapted from private retrieveStyle method in de.undercouch.citeproc.CSL + * Retrieves a CSL style from the classpath. For example, if the given name is + * ieee this method will load the file /ieee.csl + * + * @param styleName the style's name + * @return the serialized XML representation of the style + * @throws IOException if the style could not be loaded + */ + public static String getCitationFormat(String styleName) throws IOException { + URL url; + + // normalize file name + if (!styleName.endsWith(".csl")) { + styleName = styleName + ".csl"; + } + if (!styleName.startsWith("/")) { + styleName = "/" + styleName; + } + + // try to find style in classpath + url = CSL.class.getResource(styleName); + if (url == null) { + throw new FileNotFoundException("Could not find style in " + "classpath: " + styleName); + } + + // load style + String result = CSLUtils.readURLToString(url, "UTF-8"); + result = result.replace("\"", "\\\"").replace("\r", "").replace("\n", ""); + return result; + } + + private static String[] commonStyles = null; + + private static String[] getCommonStyles() { + if (commonStyles == null) { + commonStyles = JvmSettings.CSL_COMMON_STYLES.lookupOptional().orElse("chicago-author-date, ieee") + .split("\\s*,\\s*"); + } + return commonStyles; + } + +} diff --git a/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java index 991682ec8e8..924566cc0ba 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java @@ -62,6 +62,7 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.nio.ByteBuffer; import java.nio.charset.Charset; import java.nio.file.Files; import java.nio.file.Path; @@ -103,6 +104,7 @@ import java.util.Arrays; import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.StringUtils; +import org.apache.tika.Tika; import ucar.nc2.NetcdfFile; import ucar.nc2.NetcdfFiles; @@ -426,7 +428,43 @@ public static String retestIngestableFileType(File file, String fileType) { return newType != null ? newType : fileType; } - public static String determineFileType(File f, String fileName) throws IOException{ + public static String determineRemoteFileType(DataFile df, String fileName) { + String fileType = determineFileTypeByNameAndExtension(fileName); + + if (!StringUtils.isBlank(fileType) && fileType.startsWith("application/x-stata")) { + String driverId = DataAccess + .getStorageDriverFromIdentifier(df.getStorageIdentifier()); + if (StorageIO.isDataverseAccessible(driverId)) { + try { + StorageIO storage = df.getStorageIO(); + storage.open(DataAccessOption.READ_ACCESS); + try (InputStream is = storage.getInputStream()) { + + // Read the first 42 bytes of the file to determine the file type + byte[] buffer = new byte[42]; + is.read(buffer, 0, 42); + ByteBuffer bb = ByteBuffer.allocate(42); + bb.put(buffer); + + // step 1: + // Apply our custom methods to try and recognize data files that can be + // converted to tabular data + logger.fine("Attempting to identify potential tabular data files;"); + IngestableDataChecker tabChk = new IngestableDataChecker(new String[] { "DTA" }); + fileType = tabChk.detectTabularDataFormat(bb); + } catch (IOException ex) { + logger.warning("Unable to getInputStream for storageIdentifier: " + df.getStorageIdentifier()); + } + } catch (IOException ex) { + logger.warning("Unable to open storageIO for storageIdentifier: " + df.getStorageIdentifier()); + } + } + } + return fileType; + + } + + public static String determineFileType(File f, String fileName) throws IOException { String fileType = lookupFileTypeByFileName(fileName); if (fileType != null) { return fileType; @@ -495,6 +533,7 @@ public static String determineFileType(File f, String fileName) throws IOExcepti logger.fine("mime type recognized by extension: "+fileType); } } else { + //ToDo - if the extension is null, how can this call do anything logger.fine("fileExtension is null"); final String fileTypeByExtension = lookupFileTypeByExtensionFromPropertiesFile(fileName); if(!StringUtil.isEmpty(fileTypeByExtension)) { @@ -568,21 +607,23 @@ private static String lookupFileTypeByExtension(final String fileName) { } private static String lookupFileTypeByFileName(final String fileName) { - return lookupFileTypeFromPropertiesFile("MimeTypeDetectionByFileName", fileName); + return lookupFileTypeFromPropertiesFile(fileName, false); } private static String lookupFileTypeByExtensionFromPropertiesFile(final String fileName) { final String fileKey = FilenameUtils.getExtension(fileName); - return lookupFileTypeFromPropertiesFile("MimeTypeDetectionByFileExtension", fileKey); + return lookupFileTypeFromPropertiesFile(fileKey, true); } - private static String lookupFileTypeFromPropertiesFile(final String propertyFileName, final String fileKey) { + private static String lookupFileTypeFromPropertiesFile(final String fileKey, boolean byExtension) { + final String propertyFileName = byExtension ? "MimeTypeDetectionByFileExtension" : "MimeTypeDetectionByFileName"; final String propertyFileNameOnDisk = propertyFileName + ".properties"; try { logger.fine("checking " + propertyFileNameOnDisk + " for file key " + fileKey); return BundleUtil.getStringFromPropertyFile(fileKey, propertyFileName); } catch (final MissingResourceException ex) { - logger.info(fileKey + " is a filename/extension Dataverse doesn't know about. Consider adding it to the " + propertyFileNameOnDisk + " file."); + //Only use info level if it's for an extension + logger.log(byExtension ? Level.INFO : Level.FINE, fileKey + " is a filename/extension Dataverse doesn't know about. Consider adding it to the " + propertyFileNameOnDisk + " file."); return null; } } @@ -1828,4 +1869,16 @@ public static String getStorageDriver(DataFile dataFile) { public static String sanitizeFileName(String fileNameIn) { return fileNameIn == null ? null : fileNameIn.replace(' ', '_').replaceAll("[\\\\/:*?\"<>|,;]", ""); } + + public static Path createDirStructure(String rootDirectory, String... subdirectories) throws IOException { + Path path = Path.of(rootDirectory, subdirectories); + Files.createDirectories(path); + return path; + } + + public static boolean isFileOfImageType(File file) throws IOException { + Tika tika = new Tika(); + String mimeType = tika.detect(file); + return mimeType != null && mimeType.startsWith("image/"); + } } diff --git a/src/main/java/edu/harvard/iq/dataverse/util/MarkupChecker.java b/src/main/java/edu/harvard/iq/dataverse/util/MarkupChecker.java index ef74819f073..02055ad60e9 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/MarkupChecker.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/MarkupChecker.java @@ -1,8 +1,3 @@ -/* - * To change this license header, choose License Headers in Project Properties. - * To change this template file, choose Tools | Templates - * and open the template in the editor. - */ package edu.harvard.iq.dataverse.util; import org.apache.commons.text.StringEscapeUtils; @@ -11,56 +6,105 @@ import org.jsoup.parser.Parser; /** - * Wrapper for Jsoup clean - * + * Provides utility methods for sanitizing and processing HTML content. + *

    + * This class serves as a wrapper for the {@code Jsoup.clean} method and offers + * multiple configurations for cleaning HTML input. It also provides a method + * for escaping HTML entities and stripping all HTML tags. + *

    + * * @author rmp553 */ public class MarkupChecker { - - - + /** - * Wrapper around Jsoup clean method with the basic Safe list - * http://jsoup.org/cookbook/cleaning-html/safelist-sanitizer - * @param unsafe - * @return + * Sanitizes the provided HTML content using a customizable configuration. + *

    + * This method uses the {@code Jsoup.clean} method with a configurable {@code Safelist}. + * For more details, see the + * Jsoup SafeList Sanitizer. + *

    + *

    + * It supports preserving class attributes and optionally adding "noopener noreferrer nofollow" + * attributes to anchor tags to enhance security and usability. + *

    + * + * @param unsafe the HTML content to be sanitized; may contain unsafe or untrusted elements. + * @param keepClasses whether to preserve class attributes in the sanitized HTML. + * @param includeNoopenerNoreferrer whether to add "noopener noreferrer nofollow" to tags. + * @return a sanitized HTML string, free from potentially harmful content. */ - public static String sanitizeBasicHTML(String unsafe) { - + private static String sanitizeHTML(String unsafe, boolean keepClasses, boolean includeNoopenerNoreferrer) { if (unsafe == null) { return null; } - // basic includes: a, b, blockquote, br, cite, code, dd, dl, dt, em, i, li, ol, p, pre, q, small, span, strike, strong, sub, sup, u, ul - //Whitelist wl = Whitelist.basic().addTags("img", "h1", "h2", "h3", "kbd", "hr", "s", "del"); - Safelist sl = Safelist.basicWithImages().addTags("h1", "h2", "h3", "kbd", "hr", "s", "del", "map", "area").addAttributes("img", "usemap") - .addAttributes("map", "name").addAttributes("area", "shape", "coords", "href", "title", "alt") + // Create a base Safelist configuration + Safelist sl = Safelist.basicWithImages() + .addTags("h1", "h2", "h3", "kbd", "hr", "s", "del", "map", "area") + .addAttributes("img", "usemap") + .addAttributes("map", "name") + .addAttributes("area", "shape", "coords", "href", "title", "alt") .addEnforcedAttribute("a", "target", "_blank"); + // Add class attributes if requested + if (keepClasses) { + sl.addAttributes(":all", "class"); + } + + // Add "noopener noreferrer nofollow" to tags if requested + if (includeNoopenerNoreferrer) { + sl.addEnforcedAttribute("a", "rel", "noopener noreferrer nofollow"); + } + return Jsoup.clean(unsafe, sl); + } + /** + * Sanitizes the provided HTML content using a basic configuration. + * + * @param unsafe the HTML content to be sanitized; may contain unsafe or untrusted elements. + * @return a sanitized HTML string, free from potentially harmful content. + */ + public static String sanitizeBasicHTML(String unsafe) { + return sanitizeHTML(unsafe, false, false); } - + /** - * Strip all HTMl tags - * - * http://jsoup.org/apidocs/org/jsoup/safety/Safelist.html#none - * - * @param unsafe - * @return + * Sanitizes the provided HTML content using an advanced configuration. + *

    + * This configuration preserves class attributes and adds "noopener noreferrer nofollow" + * attributes to tags to enhance security and usability. + *

    + * + * @param unsafe the HTML content to be sanitized; may contain unsafe or untrusted elements. + * @return a sanitized HTML string, free from potentially harmful content. */ - public static String stripAllTags(String unsafe) { + public static String sanitizeAdvancedHTML(String unsafe) { + return sanitizeHTML(unsafe, true, true); + } + /** + * Removes all HTML tags from the provided content, leaving only plain text. + * + * @param unsafe the HTML content to process; may contain HTML tags. + * @return the plain text content with all HTML tags removed, or {@code null} if the input is {@code null}. + */ + public static String stripAllTags(String unsafe) { if (unsafe == null) { return null; } return Parser.unescapeEntities(Jsoup.clean(unsafe, Safelist.none()), true); - } - + + /** + * Escapes special characters in the provided string into their corresponding HTML entities. + * + * @param unsafe the string to escape; may contain special characters. + * @return a string with HTML entities escaped. + */ public static String escapeHtml(String unsafe) { - return StringEscapeUtils.escapeHtml4(unsafe); + return StringEscapeUtils.escapeHtml4(unsafe); } - } diff --git a/src/main/java/edu/harvard/iq/dataverse/util/ShapefileHandler.java b/src/main/java/edu/harvard/iq/dataverse/util/ShapefileHandler.java index 2b54f7a3bfe..345a2d3cccc 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/ShapefileHandler.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/ShapefileHandler.java @@ -8,6 +8,7 @@ import java.util.Date; import java.util.ArrayList; import java.util.List; +import java.util.stream.Collectors; import java.util.zip.ZipFile; import java.util.HashMap; import java.util.*; @@ -561,7 +562,7 @@ private boolean isShapefileExtension(String ext_name){ if (ext_name == null){ return false; } - return SHAPEFILE_ALL_EXTENSIONS.contains(ext_name); + return SHAPEFILE_ALL_EXTENSIONS.contains(ext_name.toLowerCase()); } /* Does a list of file extensions match those required for a shapefile set? @@ -570,7 +571,10 @@ private boolean doesListContainShapefileExtensions(List ext_list){ if (ext_list == null){ return false; } - return ext_list.containsAll(SHAPEFILE_MANDATORY_EXTENSIONS); + var lowerCaseExtensions = ext_list.stream() + .map(String::toLowerCase) + .toList(); + return lowerCaseExtensions.containsAll(SHAPEFILE_MANDATORY_EXTENSIONS); } diff --git a/src/main/java/edu/harvard/iq/dataverse/util/SignpostingResources.java b/src/main/java/edu/harvard/iq/dataverse/util/SignpostingResources.java index b6f8870aa2d..8bebcf4d438 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/SignpostingResources.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/SignpostingResources.java @@ -16,6 +16,7 @@ Two configurable options allow changing the limit for the number of authors or d import edu.harvard.iq.dataverse.*; import edu.harvard.iq.dataverse.dataset.DatasetUtil; +import edu.harvard.iq.dataverse.export.ExportService; import jakarta.json.Json; import jakarta.json.JsonArrayBuilder; import jakarta.json.JsonObjectBuilder; @@ -28,6 +29,8 @@ Two configurable options allow changing the limit for the number of authors or d import java.util.logging.Logger; import static edu.harvard.iq.dataverse.util.json.NullSafeJsonBuilder.jsonObjectBuilder; +import io.gdcc.spi.export.ExportException; +import io.gdcc.spi.export.Exporter; public class SignpostingResources { private static final Logger logger = Logger.getLogger(SignpostingResources.class.getCanonicalName()); @@ -72,8 +75,17 @@ public String getLinks() { } String describedby = "<" + ds.getGlobalId().asURL().toString() + ">;rel=\"describedby\"" + ";type=\"" + "application/vnd.citationstyles.csl+json\""; - describedby += ",<" + systemConfig.getDataverseSiteUrl() + "/api/datasets/export?exporter=schema.org&persistentId=" - + ds.getProtocol() + ":" + ds.getAuthority() + "/" + ds.getIdentifier() + ">;rel=\"describedby\"" + ";type=\"application/ld+json\""; + ExportService instance = ExportService.getInstance(); + for (String[] labels : instance.getExportersLabels()) { + String formatName = labels[1]; + Exporter exporter; + try { + exporter = ExportService.getInstance().getExporter(formatName); + describedby += ",<" + getExporterUrl(formatName, ds) + ">;rel=\"describedby\"" + ";type=\"" + exporter.getMediaType() + "\""; + } catch (ExportException ex) { + logger.warning("Could not look up exporter based on " + formatName + ". Exception: " + ex); + } + } valueList.add(describedby); String type = ";rel=\"type\""; @@ -85,7 +97,7 @@ public String getLinks() { String linkset = "<" + systemConfig.getDataverseSiteUrl() + "/api/datasets/:persistentId/versions/" + workingDatasetVersion.getVersionNumber() + "." + workingDatasetVersion.getMinorVersionNumber() - + "/linkset?persistentId=" + ds.getProtocol() + ":" + ds.getAuthority() + "/" + ds.getIdentifier() + "> ; rel=\"linkset\";type=\"application/linkset+json\""; + + "/linkset?persistentId=" + ds.getGlobalId().asString() + "> ; rel=\"linkset\";type=\"application/linkset+json\""; valueList.add(linkset); logger.fine(String.format("valueList is: %s", valueList)); @@ -95,7 +107,7 @@ public String getLinks() { public JsonArrayBuilder getJsonLinkset() { Dataset ds = workingDatasetVersion.getDataset(); GlobalId gid = ds.getGlobalId(); - String landingPage = systemConfig.getDataverseSiteUrl() + "/dataset.xhtml?persistentId=" + ds.getProtocol() + ":" + ds.getAuthority() + "/" + ds.getIdentifier(); + String landingPage = systemConfig.getDataverseSiteUrl() + "/dataset.xhtml?persistentId=" + ds.getGlobalId().asString(); JsonArrayBuilder authors = getJsonAuthors(getAuthorURLs(false)); JsonArrayBuilder items = getJsonItems(); @@ -112,15 +124,24 @@ public JsonArrayBuilder getJsonLinkset() { ) ); - mediaTypes.add( - jsonObjectBuilder().add( - "href", - systemConfig.getDataverseSiteUrl() + "/api/datasets/export?exporter=schema.org&persistentId=" + ds.getProtocol() + ":" + ds.getAuthority() + "/" + ds.getIdentifier() - ).add( - "type", - "application/ld+json" - ) - ); + ExportService instance = ExportService.getInstance(); + for (String[] labels : instance.getExportersLabels()) { + String formatName = labels[1]; + Exporter exporter; + try { + exporter = ExportService.getInstance().getExporter(formatName); + mediaTypes.add( + jsonObjectBuilder().add( + "href", getExporterUrl(formatName, ds) + ).add( + "type", + exporter.getMediaType() + ) + ); + } catch (ExportException ex) { + logger.warning("Could not look up exporter based on " + formatName + ". Exception: " + ex); + } + } JsonArrayBuilder linksetJsonObj = Json.createArrayBuilder(); JsonObjectBuilder mandatory; @@ -274,4 +295,9 @@ private String getPublicDownloadUrl(DataFile dataFile) { return FileUtil.getPublicDownloadUrl(systemConfig.getDataverseSiteUrl(), ((gid != null) ? gid.asString() : null), dataFile.getId()); } + + private String getExporterUrl(String formatName, Dataset ds) { + return systemConfig.getDataverseSiteUrl() + + "/api/datasets/export?exporter=" + formatName + "&persistentId=" + ds.getGlobalId().asString(); + } } diff --git a/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java b/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java index e769cacfdb1..5a78ee97ce2 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java @@ -1173,4 +1173,8 @@ public String getRateLimitsJson() { public String getRateLimitingDefaultCapacityTiers() { return settingsService.getValueForKey(SettingsServiceBean.Key.RateLimitingDefaultCapacityTiers, ""); } + + public long getContactFeedbackMessageSizeLimit() { + return settingsService.getValueForKeyAsLong(SettingsServiceBean.Key.ContactFeedbackMessageSizeLimit, 0L); + } } diff --git a/src/main/java/edu/harvard/iq/dataverse/util/bagit/OREMap.java b/src/main/java/edu/harvard/iq/dataverse/util/bagit/OREMap.java index 60ab9407269..4cbc2aa7b9a 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/bagit/OREMap.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/bagit/OREMap.java @@ -49,7 +49,8 @@ public class OREMap { public static final String NAME = "OREMap"; //NOTE: Update this value whenever the output of this class is changed - private static final String DATAVERSE_ORE_FORMAT_VERSION = "Dataverse OREMap Format v1.0.0"; + private static final String DATAVERSE_ORE_FORMAT_VERSION = "Dataverse OREMap Format v1.0.1"; + //v1.0.1 - added versionNote private static final String DATAVERSE_SOFTWARE_NAME = "Dataverse"; private static final String DATAVERSE_SOFTWARE_URL = "https://github.com/iqss/dataverse"; @@ -122,13 +123,15 @@ public JsonObjectBuilder getOREMapBuilder(boolean aggregationOnly) { .add(JsonLDTerm.schemaOrg("name").getLabel(), version.getTitle()) .add(JsonLDTerm.schemaOrg("dateModified").getLabel(), version.getLastUpdateTime().toString()); addIfNotNull(aggBuilder, JsonLDTerm.schemaOrg("datePublished"), dataset.getPublicationDateFormattedYYYYMMDD()); + addIfNotNull(aggBuilder, JsonLDTerm.DVCore("versionNote"), version.getVersionNote()); + //Add version state info - DRAFT, RELEASED, DEACCESSIONED, ARCHIVED with extra info for DEACCESIONED VersionState vs = version.getVersionState(); if(vs.equals(VersionState.DEACCESSIONED)) { JsonObjectBuilder deaccBuilder = Json.createObjectBuilder(); deaccBuilder.add(JsonLDTerm.schemaOrg("name").getLabel(), vs.name()); - deaccBuilder.add(JsonLDTerm.DVCore("reason").getLabel(), version.getVersionNote()); - addIfNotNull(deaccBuilder, JsonLDTerm.DVCore("forwardUrl"), version.getArchiveNote()); + deaccBuilder.add(JsonLDTerm.DVCore("reason").getLabel(), version.getDeaccessionNote()); + addIfNotNull(deaccBuilder, JsonLDTerm.DVCore("forwardUrl"), version.getDeaccessionLink()); aggBuilder.add(JsonLDTerm.schemaOrg("creativeWorkStatus").getLabel(), deaccBuilder); } else { @@ -444,39 +447,45 @@ public static JsonValue getJsonLDForField(DatasetField field, Boolean excludeEma for (DatasetField dsf : dscv.getChildDatasetFields()) { DatasetFieldType dsft = dsf.getDatasetFieldType(); - if (excludeEmail && DatasetFieldType.FieldType.EMAIL.equals(dsft.getFieldType())) { - continue; - } - // which may have multiple values - if (!dsf.isEmpty()) { - // Add context entry - // ToDo - also needs to recurse here? - JsonLDTerm subFieldName = dsft.getJsonLDTerm(); - if (subFieldName.inNamespace()) { - localContext.putIfAbsent(subFieldName.getNamespace().getPrefix(), - subFieldName.getNamespace().getUrl()); - } else { - localContext.putIfAbsent(subFieldName.getLabel(), subFieldName.getUrl()); + JsonLDTerm subFieldName = dsft.getJsonLDTerm(); + + if (dsft.isCompound()) { + JsonValue compoundChildVals = getJsonLDForField(dsf, excludeEmail, cvocMap, localContext); + child.add(subFieldName.getLabel(), compoundChildVals); + } else { + if (excludeEmail && DatasetFieldType.FieldType.EMAIL.equals(dsft.getFieldType())) { + continue; } + // which may have multiple values + if (!dsf.isEmpty()) { + // Add context entry + // ToDo - also needs to recurse here? + if (subFieldName.inNamespace()) { + localContext.putIfAbsent(subFieldName.getNamespace().getPrefix(), + subFieldName.getNamespace().getUrl()); + } else { + localContext.putIfAbsent(subFieldName.getLabel(), subFieldName.getUrl()); + } - List values = dsf.getValues_nondisplay(); + List values = dsf.getValues_nondisplay(); - JsonArrayBuilder childVals = Json.createArrayBuilder(); + JsonArrayBuilder childVals = Json.createArrayBuilder(); - for (String val : dsf.getValues_nondisplay()) { - logger.fine("Child name: " + dsft.getName()); - if (cvocMap.containsKey(dsft.getId())) { - logger.fine("Calling addcvocval for: " + dsft.getName()); - addCvocValue(val, childVals, cvocMap.get(dsft.getId()), localContext); + for (String val : dsf.getValues_nondisplay()) { + logger.fine("Child name: " + dsft.getName()); + if (cvocMap.containsKey(dsft.getId())) { + logger.fine("Calling addcvocval for: " + dsft.getName()); + addCvocValue(val, childVals, cvocMap.get(dsft.getId()), localContext); + } else { + childVals.add(val); + } + } + if (values.size() > 1) { + child.add(subFieldName.getLabel(), childVals); } else { - childVals.add(val); + child.add(subFieldName.getLabel(), childVals.build().get(0)); } } - if (values.size() > 1) { - child.add(subFieldName.getLabel(), childVals); - } else { - child.add(subFieldName.getLabel(), childVals.build().get(0)); - } } } vals.add(child); diff --git a/src/main/java/edu/harvard/iq/dataverse/util/cache/CacheFactoryBean.java b/src/main/java/edu/harvard/iq/dataverse/util/cache/CacheFactoryBean.java index 36b2b35b48f..c27d6f8a559 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/cache/CacheFactoryBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/cache/CacheFactoryBean.java @@ -52,6 +52,9 @@ public boolean checkRate(User user, Command command) { int capacity = RateLimitUtil.getCapacity(systemConfig, user, action); if (capacity == RateLimitUtil.NO_LIMIT) { return true; + } else if (capacity == RateLimitUtil.RESET_CACHE) { + rateLimitCache.clear(); + return true; } else { String cacheKey = RateLimitUtil.generateCacheKey(user, action); return (!RateLimitUtil.rateLimited(rateLimitCache, cacheKey, capacity)); diff --git a/src/main/java/edu/harvard/iq/dataverse/util/cache/RateLimitUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/cache/RateLimitUtil.java index b566cd42fe1..572ea8d5601 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/cache/RateLimitUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/cache/RateLimitUtil.java @@ -25,6 +25,8 @@ public class RateLimitUtil { static final List rateLimits = new CopyOnWriteArrayList<>(); static final Map rateLimitMap = new ConcurrentHashMap<>(); public static final int NO_LIMIT = -1; + public static final int RESET_CACHE = -2; + static String settingRateLimitsJson = ""; static String generateCacheKey(final User user, final String action) { return (user != null ? user.getIdentifier() : GuestUser.get().getIdentifier()) + @@ -34,6 +36,15 @@ static int getCapacity(SystemConfig systemConfig, User user, String action) { if (user != null && user.isSuperuser()) { return NO_LIMIT; } + + // If the setting changes then reset the cache + if (!settingRateLimitsJson.equals(systemConfig.getRateLimitsJson())) { + settingRateLimitsJson = systemConfig.getRateLimitsJson(); + logger.fine("Setting RateLimitingCapacityByTierAndAction changed (" + settingRateLimitsJson + "). Resetting cache"); + rateLimits.clear(); + return RESET_CACHE; + } + // get the capacity, i.e. calls per hour, from config return (user instanceof AuthenticatedUser authUser) ? getCapacityByTierAndAction(systemConfig, authUser.getRateLimitTier(), action) : diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/BriefJsonPrinter.java b/src/main/java/edu/harvard/iq/dataverse/util/json/BriefJsonPrinter.java index c16a46a1765..83b9d341d6d 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/BriefJsonPrinter.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/BriefJsonPrinter.java @@ -24,14 +24,14 @@ public JsonObjectBuilder json( DatasetVersion dsv ) { } public JsonObjectBuilder json( MetadataBlock blk ) { - return ( blk==null ) - ? null - : jsonObjectBuilder().add("id", blk.getId()) - .add("displayName", blk.getDisplayName()) - .add("displayOnCreate", blk.isDisplayOnCreate()) - .add("name", blk.getName()) - ; - } + if (blk == null) return null; + boolean displayOnCreate = blk.isDisplayOnCreate(); + return jsonObjectBuilder().add("id", blk.getId()) + .add("displayName", blk.getDisplayName()) + .add("displayOnCreate", displayOnCreate) + .add("name", blk.getName()) + ; + } public JsonObjectBuilder json( Workflow wf ) { return jsonObjectBuilder().add("id", wf.getId()) diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonParser.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonParser.java index 232b7431a24..8fd311f31ea 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonParser.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonParser.java @@ -21,6 +21,7 @@ import edu.harvard.iq.dataverse.api.Util; import edu.harvard.iq.dataverse.api.dto.DataverseDTO; import edu.harvard.iq.dataverse.api.dto.FieldDTO; +import edu.harvard.iq.dataverse.api.dto.UserDTO; import edu.harvard.iq.dataverse.authorization.groups.impl.ipaddress.IpGroup; import edu.harvard.iq.dataverse.authorization.groups.impl.ipaddress.ip.IpAddress; import edu.harvard.iq.dataverse.authorization.groups.impl.ipaddress.ip.IpAddressRange; @@ -31,6 +32,7 @@ import edu.harvard.iq.dataverse.harvest.client.HarvestingClient; import edu.harvard.iq.dataverse.license.License; import edu.harvard.iq.dataverse.license.LicenseServiceBean; +import edu.harvard.iq.dataverse.settings.FeatureFlags; import edu.harvard.iq.dataverse.settings.SettingsServiceBean; import edu.harvard.iq.dataverse.util.BundleUtil; import edu.harvard.iq.dataverse.workflow.Workflow; @@ -49,6 +51,7 @@ import java.util.Map; import java.util.Optional; import java.util.Set; +import java.util.function.Function; import java.util.function.Consumer; import java.util.logging.Logger; import java.util.stream.Collectors; @@ -76,11 +79,11 @@ public class JsonParser { DatasetTypeServiceBean datasetTypeService; HarvestingClient harvestingClient = null; boolean allowHarvestingMissingCVV = false; - + /** * if lenient, we will accept alternate spellings for controlled vocabulary values */ - boolean lenient = false; + boolean lenient = false; @Deprecated public JsonParser(DatasetFieldServiceBean datasetFieldSvc, MetadataBlockServiceBean blockService, SettingsServiceBean settingsService) { @@ -92,7 +95,7 @@ public JsonParser(DatasetFieldServiceBean datasetFieldSvc, MetadataBlockServiceB public JsonParser(DatasetFieldServiceBean datasetFieldSvc, MetadataBlockServiceBean blockService, SettingsServiceBean settingsService, LicenseServiceBean licenseService, DatasetTypeServiceBean datasetTypeService) { this(datasetFieldSvc, blockService, settingsService, licenseService, datasetTypeService, null); } - + public JsonParser(DatasetFieldServiceBean datasetFieldSvc, MetadataBlockServiceBean blockService, SettingsServiceBean settingsService, LicenseServiceBean licenseService, DatasetTypeServiceBean datasetTypeService, HarvestingClient harvestingClient) { this.datasetFieldSvc = datasetFieldSvc; this.blockService = blockService; @@ -106,7 +109,7 @@ public JsonParser(DatasetFieldServiceBean datasetFieldSvc, MetadataBlockServiceB public JsonParser() { this( null,null,null ); } - + public boolean isLenient() { return lenient; } @@ -163,6 +166,9 @@ public Dataverse parseDataverse(JsonObject jobj) throws JsonParseException { if (jobj.containsKey("filePIDsEnabled")) { dv.setFilePIDsEnabled(jobj.getBoolean("filePIDsEnabled")); } + if (jobj.containsKey("requireFilesToPublishDataset")) { + dv.setRequireFilesToPublishDataset(jobj.getBoolean("requireFilesToPublishDataset")); + } /* We decided that subject is not user set, but gotten from the subject of the dataverse's datasets - leavig this code in for now, in case we need to go back to it at some point @@ -282,11 +288,19 @@ public DataverseTheme parseDataverseTheme(JsonObject obj) { return theme; } - private static String getMandatoryString(JsonObject jobj, String name) throws JsonParseException { + private static T getMandatoryField(JsonObject jobj, String name, Function getter) throws JsonParseException { if (jobj.containsKey(name)) { - return jobj.getString(name); + return getter.apply(name); } - throw new JsonParseException("Field " + name + " is mandatory"); + throw new JsonParseException("Field '" + name + "' is mandatory"); + } + + private static String getMandatoryString(JsonObject jobj, String name) throws JsonParseException { + return getMandatoryField(jobj, name, jobj::getString); + } + + private static Boolean getMandatoryBoolean(JsonObject jobj, String name) throws JsonParseException { + return getMandatoryField(jobj, name, jobj::getBoolean); } public IpGroup parseIpGroup(JsonObject obj) { @@ -318,10 +332,10 @@ public IpGroup parseIpGroup(JsonObject obj) { return retVal; } - + public MailDomainGroup parseMailDomainGroup(JsonObject obj) throws JsonParseException { MailDomainGroup grp = new MailDomainGroup(); - + if (obj.containsKey("id")) { grp.setId(obj.getJsonNumber("id").longValue()); } @@ -345,7 +359,7 @@ public MailDomainGroup parseMailDomainGroup(JsonObject obj) throws JsonParseExce } else { throw new JsonParseException("Field domains is mandatory."); } - + return grp; } @@ -383,7 +397,7 @@ public Dataset parseDataset(JsonObject obj) throws JsonParseException { throw new JsonParseException("Invalid dataset type: " + datasetTypeIn); } - DatasetVersion dsv = new DatasetVersion(); + DatasetVersion dsv = new DatasetVersion(); dsv.setDataset(dataset); dsv = parseDatasetVersion(obj.getJsonObject("datasetVersion"), dsv); List versions = new ArrayList<>(1); @@ -396,12 +410,15 @@ public Dataset parseDataset(JsonObject obj) throws JsonParseException { public DatasetVersion parseDatasetVersion(JsonObject obj, DatasetVersion dsv) throws JsonParseException { try { - String archiveNote = obj.getString("archiveNote", null); - if (archiveNote != null) { - dsv.setArchiveNote(archiveNote); - } - dsv.setDeaccessionLink(obj.getString("deaccessionLink", null)); + String deaccessionNote = obj.getString("deaccessionNote", null); + // ToDo - the treatment of null inputs is inconsistent across different fields (either the original value is kept or set to null). + // This is moot for most uses of this method, which start from an empty datasetversion, but use through https://github.com/IQSS/dataverse/blob/3e5a516670c42e019338063516a9d93a61833027/src/main/java/edu/harvard/iq/dataverse/api/datadeposit/ContainerManagerImpl.java#L112 + // starts from an existing version where this inconsistency could be/is a problem. + if (deaccessionNote != null) { + dsv.setDeaccessionNote(deaccessionNote); + } + dsv.setVersionNote(obj.getString("versionNote", null)); int versionNumberInt = obj.getInt("versionNumber", -1); Long versionNumber = null; if (versionNumberInt !=-1) { @@ -414,7 +431,7 @@ public DatasetVersion parseDatasetVersion(JsonObject obj, DatasetVersion dsv) th if (dsv.getId()==null) { dsv.setId(parseLong(obj.getString("id", null))); } - + String versionStateStr = obj.getString("versionState", null); if (versionStateStr != null) { dsv.setVersionState(DatasetVersion.VersionState.valueOf(versionStateStr)); @@ -427,8 +444,8 @@ public DatasetVersion parseDatasetVersion(JsonObject obj, DatasetVersion dsv) th // Terms of Use related fields TermsOfUseAndAccess terms = new TermsOfUseAndAccess(); - License license = null; - + License license = null; + try { // This method will attempt to parse the license in the format // in which it appears in our json exports, as a compound @@ -447,7 +464,7 @@ public DatasetVersion parseDatasetVersion(JsonObject obj, DatasetVersion dsv) th // "license" : "CC0 1.0" license = parseLicense(obj.getString("license", null)); } - + if (license == null) { terms.setLicense(license); terms.setTermsOfUse(obj.getString("termsOfUse", null)); @@ -485,13 +502,13 @@ public DatasetVersion parseDatasetVersion(JsonObject obj, DatasetVersion dsv) th dsv.setFileMetadatas(parseFiles(filesJson, dsv)); } return dsv; - } catch (ParseException ex) { + } catch (ParseException ex) { throw new JsonParseException(BundleUtil.getStringFromBundle("jsonparser.error.parsing.date", Arrays.asList(ex.getMessage())) , ex); } catch (NumberFormatException ex) { throw new JsonParseException(BundleUtil.getStringFromBundle("jsonparser.error.parsing.number", Arrays.asList(ex.getMessage())), ex); } } - + private edu.harvard.iq.dataverse.license.License parseLicense(String licenseNameOrUri) throws JsonParseException { if (licenseNameOrUri == null){ boolean safeDefaultIfKeyNotFound = true; @@ -505,7 +522,7 @@ private edu.harvard.iq.dataverse.license.License parseLicense(String licenseName if (license == null) throw new JsonParseException("Invalid license: " + licenseNameOrUri); return license; } - + private edu.harvard.iq.dataverse.license.License parseLicense(JsonObject licenseObj) throws JsonParseException { if (licenseObj == null){ boolean safeDefaultIfKeyNotFound = true; @@ -515,12 +532,12 @@ private edu.harvard.iq.dataverse.license.License parseLicense(JsonObject license return licenseService.getDefault(); } } - + String licenseName = licenseObj.getString("name", null); String licenseUri = licenseObj.getString("uri", null); - - License license = null; - + + License license = null; + // If uri is provided, we'll try that first. This is an easier lookup // method; the uri is always the same. The name may have been customized // (translated) on this instance, so we may be dealing with such translated @@ -530,17 +547,17 @@ private edu.harvard.iq.dataverse.license.License parseLicense(JsonObject license if (licenseUri != null) { license = licenseService.getByNameOrUri(licenseUri); } - + if (license != null) { return license; } - + if (licenseName == null) { - String exMsg = "Invalid or unsupported license section submitted" + String exMsg = "Invalid or unsupported license section submitted" + (licenseUri != null ? ": " + licenseUri : "."); - throw new JsonParseException("Invalid or unsupported license section submitted."); + throw new JsonParseException("Invalid or unsupported license section submitted."); } - + license = licenseService.getByPotentiallyLocalizedName(licenseName); if (license == null) { throw new JsonParseException("Invalid or unsupported license: " + licenseName); @@ -559,13 +576,13 @@ public List parseMetadataBlocks(JsonObject json) throws JsonParseE } return fields; } - + public List parseMultipleFields(JsonObject json) throws JsonParseException { JsonArray fieldsJson = json.getJsonArray("fields"); List fields = parseFieldsFromArray(fieldsJson, false); return fields; } - + public List parseMultipleFieldsForDelete(JsonObject json) throws JsonParseException { List fields = new LinkedList<>(); for (JsonObject fieldJson : json.getJsonArray("fields").getValuesAs(JsonObject.class)) { @@ -573,7 +590,7 @@ public List parseMultipleFieldsForDelete(JsonObject json) throws J } return fields; } - + private List parseFieldsFromArray(JsonArray fieldsArray, Boolean testType) throws JsonParseException { List fields = new LinkedList<>(); for (JsonObject fieldJson : fieldsArray.getValuesAs(JsonObject.class)) { @@ -585,18 +602,18 @@ private List parseFieldsFromArray(JsonArray fieldsArray, Boolean t } catch (CompoundVocabularyException ex) { DatasetFieldType fieldType = datasetFieldSvc.findByNameOpt(fieldJson.getString("typeName", "")); if (lenient && (DatasetFieldConstant.geographicCoverage).equals(fieldType.getName())) { - fields.add(remapGeographicCoverage( ex)); + fields.add(remapGeographicCoverage( ex)); } else { // if not lenient mode, re-throw exception throw ex; } - } + } } return fields; - + } - + public List parseFiles(JsonArray metadatasJson, DatasetVersion dsv) throws JsonParseException { List fileMetadatas = new LinkedList<>(); if (metadatasJson != null) { @@ -610,7 +627,7 @@ public List parseFiles(JsonArray metadatasJson, DatasetVersion dsv fileMetadata.setDirectoryLabel(directoryLabel); fileMetadata.setDescription(description); fileMetadata.setDatasetVersion(dsv); - + if ( filemetadataJson.containsKey("dataFile") ) { DataFile dataFile = parseDataFile(filemetadataJson.getJsonObject("dataFile")); dataFile.getFileMetadatas().add(fileMetadata); @@ -623,7 +640,7 @@ public List parseFiles(JsonArray metadatasJson, DatasetVersion dsv dsv.getDataset().getFiles().add(dataFile); } } - + fileMetadatas.add(fileMetadata); fileMetadata.setCategories(getCategories(filemetadataJson, dsv.getDataset())); } @@ -631,19 +648,19 @@ public List parseFiles(JsonArray metadatasJson, DatasetVersion dsv return fileMetadatas; } - + public DataFile parseDataFile(JsonObject datafileJson) { DataFile dataFile = new DataFile(); - + Timestamp timestamp = new Timestamp(new Date().getTime()); dataFile.setCreateDate(timestamp); dataFile.setModificationTime(timestamp); dataFile.setPermissionModificationTime(timestamp); - + if ( datafileJson.containsKey("filesize") ) { dataFile.setFilesize(datafileJson.getJsonNumber("filesize").longValueExact()); } - + String contentType = datafileJson.getString("contentType", null); if (contentType == null) { contentType = "application/octet-stream"; @@ -706,21 +723,21 @@ public DataFile parseDataFile(JsonObject datafileJson) { // TODO: // unf (if available)... etc.? - + dataFile.setContentType(contentType); dataFile.setStorageIdentifier(storageIdentifier); - + return dataFile; } /** * Special processing for GeographicCoverage compound field: * Handle parsing exceptions caused by invalid controlled vocabulary in the "country" field by * putting the invalid data in "otherGeographicCoverage" in a new compound value. - * + * * @param ex - contains the invalid values to be processed - * @return a compound DatasetField that contains the newly created values, in addition to + * @return a compound DatasetField that contains the newly created values, in addition to * the original valid values. - * @throws JsonParseException + * @throws JsonParseException */ private DatasetField remapGeographicCoverage(CompoundVocabularyException ex) throws JsonParseException{ List> geoCoverageList = new ArrayList<>(); @@ -747,23 +764,23 @@ private DatasetField remapGeographicCoverage(CompoundVocabularyException ex) thr } return geoCoverageField; } - - + + public DatasetField parseFieldForDelete(JsonObject json) throws JsonParseException{ DatasetField ret = new DatasetField(); - DatasetFieldType type = datasetFieldSvc.findByNameOpt(json.getString("typeName", "")); + DatasetFieldType type = datasetFieldSvc.findByNameOpt(json.getString("typeName", "")); if (type == null) { throw new JsonParseException("Can't find type '" + json.getString("typeName", "") + "'"); } return ret; } - - + + public DatasetField parseField(JsonObject json) throws JsonParseException{ return parseField(json, true); } - - + + public DatasetField parseField(JsonObject json, Boolean testType) throws JsonParseException { if (json == null) { return null; @@ -771,7 +788,7 @@ public DatasetField parseField(JsonObject json, Boolean testType) throws JsonPar DatasetField ret = new DatasetField(); DatasetFieldType type = datasetFieldSvc.findByNameOpt(json.getString("typeName", "")); - + if (type == null) { logger.fine("Can't find type '" + json.getString("typeName", "") + "'"); @@ -789,8 +806,8 @@ public DatasetField parseField(JsonObject json, Boolean testType) throws JsonPar if (testType && type.isControlledVocabulary() && !json.getString("typeClass").equals("controlledVocabulary")) { throw new JsonParseException("incorrect typeClass for field " + json.getString("typeName", "") + ", should be controlledVocabulary"); } - - + + ret.setDatasetFieldType(type); if (type.isCompound()) { @@ -803,11 +820,11 @@ public DatasetField parseField(JsonObject json, Boolean testType) throws JsonPar return ret; } - + public void parseCompoundValue(DatasetField dsf, DatasetFieldType compoundType, JsonObject json) throws JsonParseException { parseCompoundValue(dsf, compoundType, json, true); } - + public void parseCompoundValue(DatasetField dsf, DatasetFieldType compoundType, JsonObject json, Boolean testType) throws JsonParseException { List vocabExceptions = new ArrayList<>(); List vals = new LinkedList<>(); @@ -829,7 +846,7 @@ public void parseCompoundValue(DatasetField dsf, DatasetFieldType compoundType, } catch(ControlledVocabularyException ex) { vocabExceptions.add(ex); } - + if (f!=null) { if (!compoundType.getChildDatasetFieldTypes().contains(f.getDatasetFieldType())) { throw new JsonParseException("field " + f.getDatasetFieldType().getName() + " is not a child of " + compoundType.getName()); @@ -846,10 +863,10 @@ public void parseCompoundValue(DatasetField dsf, DatasetFieldType compoundType, order++; } - + } else { - + DatasetFieldCompoundValue cv = new DatasetFieldCompoundValue(); List fields = new LinkedList<>(); JsonObject value = json.getJsonObject("value"); @@ -870,7 +887,7 @@ public void parseCompoundValue(DatasetField dsf, DatasetFieldType compoundType, cv.setChildDatasetFields(fields); vals.add(cv); } - + } if (!vocabExceptions.isEmpty()) { throw new CompoundVocabularyException( "Invalid controlled vocabulary in compound field ", vocabExceptions, vals); @@ -909,7 +926,7 @@ public void parsePrimitiveValue(DatasetField dsf, DatasetFieldType dft , JsonObj try {json.getString("value");} catch (ClassCastException cce) { throw new JsonParseException("Invalid value submitted for " + dft.getName() + ". It should be a single value."); - } + } DatasetFieldValue datasetFieldValue = new DatasetFieldValue(); datasetFieldValue.setValue(json.getString("value", "").trim()); datasetFieldValue.setDatasetField(dsf); @@ -923,7 +940,7 @@ public void parsePrimitiveValue(DatasetField dsf, DatasetFieldType dft , JsonObj dsf.setDatasetFieldValues(vals); } - + public Workflow parseWorkflow(JsonObject json) throws JsonParseException { Workflow retVal = new Workflow(); validate("", json, "name", ValueType.STRING); @@ -937,12 +954,12 @@ public Workflow parseWorkflow(JsonObject json) throws JsonParseException { retVal.setSteps(steps); return retVal; } - + public WorkflowStepData parseStepData( JsonObject json ) throws JsonParseException { WorkflowStepData wsd = new WorkflowStepData(); validate("step", json, "provider", ValueType.STRING); validate("step", json, "stepType", ValueType.STRING); - + wsd.setProviderId(json.getString("provider")); wsd.setStepType(json.getString("stepType")); if ( json.containsKey("parameters") ) { @@ -959,7 +976,7 @@ public WorkflowStepData parseStepData( JsonObject json ) throws JsonParseExcepti } return wsd; } - + private String jsonValueToString(JsonValue jv) { switch ( jv.getValueType() ) { case STRING: return ((JsonString)jv).getString(); @@ -1038,12 +1055,13 @@ Long parseLong(String str) throws NumberFormatException { int parsePrimitiveInt(String str, int defaultValue) { return str == null ? defaultValue : Integer.parseInt(str); } - + public String parseHarvestingClient(JsonObject obj, HarvestingClient harvestingClient) throws JsonParseException { - + String dataverseAlias = obj.getString("dataverseAlias",null); - + harvestingClient.setName(obj.getString("nickName",null)); + harvestingClient.setSourceName(obj.getString("sourceName",null)); harvestingClient.setHarvestStyle(obj.getString("style", "default")); harvestingClient.setHarvestingUrl(obj.getString("harvestUrl",null)); harvestingClient.setArchiveUrl(obj.getString("archiveUrl",null)); @@ -1052,7 +1070,10 @@ public String parseHarvestingClient(JsonObject obj, HarvestingClient harvestingC harvestingClient.setHarvestingSet(obj.getString("set",null)); harvestingClient.setCustomHttpHeaders(obj.getString("customHeaders", null)); harvestingClient.setAllowHarvestingMissingCVV(obj.getBoolean("allowHarvestingMissingCVV", false)); + harvestingClient.setUseListrecords(obj.getBoolean("useListRecords", false)); harvestingClient.setUseOaiIdentifiersAsPids(obj.getBoolean("useOaiIdentifiersAsPids", false)); + + harvestingClient.readScheduleDescription(obj.getString("schedule", null)); return dataverseAlias; } @@ -1078,7 +1099,7 @@ private List getCategories(JsonObject filemetadataJson, Datase } return dataFileCategories; } - + /** * Validate than a JSON object has a field of an expected type, or throw an * inforamtive exception. @@ -1086,12 +1107,29 @@ private List getCategories(JsonObject filemetadataJson, Datase * @param jobject * @param fieldName * @param expectedValueType - * @throws JsonParseException + * @throws JsonParseException */ private void validate(String objectName, JsonObject jobject, String fieldName, ValueType expectedValueType) throws JsonParseException { - if ( (!jobject.containsKey(fieldName)) + if ( (!jobject.containsKey(fieldName)) || (jobject.get(fieldName).getValueType()!=expectedValueType) ) { throw new JsonParseException( objectName + " missing a field named '"+fieldName+"' of type " + expectedValueType ); } } + + public UserDTO parseUserDTO(JsonObject jobj) throws JsonParseException { + UserDTO userDTO = new UserDTO(); + + userDTO.setUsername(jobj.getString("username", null)); + userDTO.setEmailAddress(jobj.getString("emailAddress", null)); + userDTO.setFirstName(jobj.getString("firstName", null)); + userDTO.setLastName(jobj.getString("lastName", null)); + userDTO.setAffiliation(jobj.getString("affiliation", null)); + userDTO.setPosition(jobj.getString("position", null)); + + if (!FeatureFlags.API_BEARER_AUTH_HANDLE_TOS_ACCEPTANCE_IN_IDP.enabled()) { + userDTO.setTermsAccepted(getMandatoryBoolean(jobj, "termsAccepted")); + } + + return userDTO; + } } diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java index 91af13c79a3..c3f4d347646 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java @@ -26,6 +26,7 @@ import edu.harvard.iq.dataverse.datavariable.VariableCategory; import edu.harvard.iq.dataverse.datavariable.VariableMetadata; import edu.harvard.iq.dataverse.datavariable.VariableRange; +import edu.harvard.iq.dataverse.dataverse.featured.DataverseFeaturedItem; import edu.harvard.iq.dataverse.license.License; import edu.harvard.iq.dataverse.globus.FileDetailsHolder; import edu.harvard.iq.dataverse.harvest.client.HarvestingClient; @@ -72,8 +73,8 @@ public class JsonPrinter { @EJB static DatasetFieldServiceBean datasetFieldService; - - public static void injectSettingsService(SettingsServiceBean ssb, DatasetFieldServiceBean dfsb) { + + public static void injectSettingsService(SettingsServiceBean ssb, DatasetFieldServiceBean dfsb, DataverseFieldTypeInputLevelServiceBean dfils) { settingsService = ssb; datasetFieldService = dfsb; } @@ -258,11 +259,11 @@ public static JsonObjectBuilder json(Workflow wf){ } public static JsonObjectBuilder json(Dataverse dv) { - return json(dv, false, false); + return json(dv, false, false, null); } //TODO: Once we upgrade to Java EE 8 we can remove objects from the builder, and this email removal can be done in a better place. - public static JsonObjectBuilder json(Dataverse dv, Boolean hideEmail, Boolean returnOwners) { + public static JsonObjectBuilder json(Dataverse dv, Boolean hideEmail, Boolean returnOwners, Long childCount) { JsonObjectBuilder bld = jsonObjectBuilder() .add("id", dv.getId()) .add("alias", dv.getAlias()) @@ -294,6 +295,7 @@ public static JsonObjectBuilder json(Dataverse dv, Boolean hideEmail, Boolean re if (dv.getFilePIDsEnabled() != null) { bld.add("filePIDsEnabled", dv.getFilePIDsEnabled()); } + bld.add("effectiveRequiresFilesToPublishDataset", dv.getEffectiveRequiresFilesToPublishDataset()); bld.add("isReleased", dv.isReleased()); List inputLevels = dv.getDataverseFieldTypeInputLevels(); @@ -301,6 +303,10 @@ public static JsonObjectBuilder json(Dataverse dv, Boolean hideEmail, Boolean re bld.add("inputLevels", JsonPrinter.jsonDataverseFieldTypeInputLevels(inputLevels)); } + if (childCount != null) { + bld.add("childCount", childCount); + } + return bld; } @@ -400,6 +406,7 @@ public static JsonObjectBuilder json(Dataset ds, Boolean returnOwners) { .add("persistentUrl", ds.getPersistentURL()) .add("protocol", ds.getProtocol()) .add("authority", ds.getAuthority()) + .add("separator", ds.getSeparator()) .add("publisher", BrandingUtil.getInstallationBrandName()) .add("publicationDate", ds.getPublicationDateFormattedYYYYMMDD()) .add("storageIdentifier", ds.getStorageIdentifier()); @@ -422,11 +429,17 @@ public static JsonObjectBuilder json(FileDetailsHolder ds) { } public static JsonObjectBuilder json(DatasetVersion dsv, boolean includeFiles) { - return json(dsv, null, includeFiles, false); + return json(dsv, null, includeFiles, false,true); + } + public static JsonObjectBuilder json(DatasetVersion dsv, boolean includeFiles, boolean includeMetadataBlocks) { + return json(dsv, null, includeFiles, false, includeMetadataBlocks); + } + public static JsonObjectBuilder json(DatasetVersion dsv, List anonymizedFieldTypeNamesList, + boolean includeFiles, boolean returnOwners) { + return json( dsv, anonymizedFieldTypeNamesList, includeFiles, returnOwners,true); } - public static JsonObjectBuilder json(DatasetVersion dsv, List anonymizedFieldTypeNamesList, - boolean includeFiles, boolean returnOwners) { + boolean includeFiles, boolean returnOwners, boolean includeMetadataBlocks) { Dataset dataset = dsv.getDataset(); JsonObjectBuilder bld = jsonObjectBuilder() .add("id", dsv.getId()).add("datasetId", dataset.getId()) @@ -436,8 +449,7 @@ public static JsonObjectBuilder json(DatasetVersion dsv, List anonymized .add("versionMinorNumber", dsv.getMinorVersionNumber()) .add("versionState", dsv.getVersionState().name()) .add("latestVersionPublishingState", dataset.getLatestVersion().getVersionState().name()) - .add("versionNote", dsv.getVersionNote()) - .add("archiveNote", dsv.getArchiveNote()) + .add("deaccessionNote", dsv.getDeaccessionNote()) .add("deaccessionLink", dsv.getDeaccessionLink()) .add("distributionDate", dsv.getDistributionDate()) .add("productionDate", dsv.getProductionDate()) @@ -447,7 +459,8 @@ public static JsonObjectBuilder json(DatasetVersion dsv, List anonymized .add("createTime", format(dsv.getCreateTime())) .add("alternativePersistentId", dataset.getAlternativePersistentIdentifier()) .add("publicationDate", dataset.getPublicationDateFormattedYYYYMMDD()) - .add("citationDate", dataset.getCitationDateFormattedYYYYMMDD()); + .add("citationDate", dataset.getCitationDateFormattedYYYYMMDD()) + .add("versionNote", dsv.getVersionNote()); License license = DatasetUtil.getLicense(dsv); if (license != null) { @@ -471,11 +484,12 @@ public static JsonObjectBuilder json(DatasetVersion dsv, List anonymized .add("sizeOfCollection", dsv.getTermsOfUseAndAccess().getSizeOfCollection()) .add("studyCompletion", dsv.getTermsOfUseAndAccess().getStudyCompletion()) .add("fileAccessRequest", dsv.getTermsOfUseAndAccess().isFileAccessRequest()); - - bld.add("metadataBlocks", (anonymizedFieldTypeNamesList != null) ? - jsonByBlocks(dsv.getDatasetFields(), anonymizedFieldTypeNamesList) - : jsonByBlocks(dsv.getDatasetFields()) - ); + if(includeMetadataBlocks) { + bld.add("metadataBlocks", (anonymizedFieldTypeNamesList != null) ? + jsonByBlocks(dsv.getDatasetFields(), anonymizedFieldTypeNamesList) + : jsonByBlocks(dsv.getDatasetFields()) + ); + } if(returnOwners){ bld.add("isPartOf", getOwnersFromDvObject(dataset)); } @@ -598,13 +612,13 @@ public static JsonObjectBuilder json(MetadataBlock block, List fie } public static JsonArrayBuilder json(List metadataBlocks, boolean returnDatasetFieldTypes, boolean printOnlyDisplayedOnCreateDatasetFieldTypes) { - return json(metadataBlocks, returnDatasetFieldTypes, printOnlyDisplayedOnCreateDatasetFieldTypes, null); + return json(metadataBlocks, returnDatasetFieldTypes, printOnlyDisplayedOnCreateDatasetFieldTypes, null, null); } - public static JsonArrayBuilder json(List metadataBlocks, boolean returnDatasetFieldTypes, boolean printOnlyDisplayedOnCreateDatasetFieldTypes, Dataverse ownerDataverse) { + public static JsonArrayBuilder json(List metadataBlocks, boolean returnDatasetFieldTypes, boolean printOnlyDisplayedOnCreateDatasetFieldTypes, Dataverse ownerDataverse, DatasetType datasetType) { JsonArrayBuilder arrayBuilder = Json.createArrayBuilder(); for (MetadataBlock metadataBlock : metadataBlocks) { - arrayBuilder.add(returnDatasetFieldTypes ? json(metadataBlock, printOnlyDisplayedOnCreateDatasetFieldTypes, ownerDataverse) : brief.json(metadataBlock)); + arrayBuilder.add(returnDatasetFieldTypes ? json(metadataBlock, printOnlyDisplayedOnCreateDatasetFieldTypes, ownerDataverse, datasetType) : brief.json(metadataBlock)); } return arrayBuilder; } @@ -632,36 +646,58 @@ public static JsonObject json(DatasetField dfv) { } public static JsonObjectBuilder json(MetadataBlock metadataBlock) { - return json(metadataBlock, false, null); + return json(metadataBlock, false, null, null); } - public static JsonObjectBuilder json(MetadataBlock metadataBlock, boolean printOnlyDisplayedOnCreateDatasetFieldTypes, Dataverse ownerDataverse) { + public static JsonObjectBuilder json(MetadataBlock metadataBlock, boolean printOnlyDisplayedOnCreateDatasetFieldTypes, Dataverse ownerDataverse, DatasetType datasetType) { JsonObjectBuilder jsonObjectBuilder = jsonObjectBuilder() .add("id", metadataBlock.getId()) .add("name", metadataBlock.getName()) - .add("displayName", metadataBlock.getDisplayName()) - .add("displayOnCreate", metadataBlock.isDisplayOnCreate()); - - Set datasetFieldTypes; + .add("displayName", metadataBlock.getDisplayName()); + + jsonObjectBuilder.add("displayOnCreate", metadataBlock.isDisplayOnCreate()); - if (ownerDataverse != null) { - datasetFieldTypes = new TreeSet<>(datasetFieldService.findAllInMetadataBlockAndDataverse( - metadataBlock, ownerDataverse, printOnlyDisplayedOnCreateDatasetFieldTypes)); - } else { - datasetFieldTypes = printOnlyDisplayedOnCreateDatasetFieldTypes - ? new TreeSet<>(datasetFieldService.findAllDisplayedOnCreateInMetadataBlock(metadataBlock)) - : new TreeSet<>(metadataBlock.getDatasetFieldTypes()); - } + List datasetFieldTypesList = metadataBlock.getDatasetFieldTypes(); + Set datasetFieldTypes = filterOutDuplicateDatasetFieldTypes(datasetFieldTypesList); JsonObjectBuilder fieldsBuilder = Json.createObjectBuilder(); + for (DatasetFieldType datasetFieldType : datasetFieldTypes) { - fieldsBuilder.add(datasetFieldType.getName(), json(datasetFieldType, ownerDataverse)); + if (!datasetFieldType.isChild()) { + DataverseFieldTypeInputLevel level = null; + datasetFieldType.setInclude(true); + if (ownerDataverse != null) { + level = ownerDataverse.getDatasetFieldTypeInInputLevels(datasetFieldType.getId()); + if (level != null) { + datasetFieldType.setLocalDisplayOnCreate(level.getDisplayOnCreate()); + datasetFieldType.setRequiredDV(level.isRequired()); + datasetFieldType.setInclude(level.isInclude()); + } + } + boolean fieldDisplayOnCreate = datasetFieldType.shouldDisplayOnCreate(); + if (datasetFieldType.isInclude() && (!printOnlyDisplayedOnCreateDatasetFieldTypes + || fieldDisplayOnCreate || datasetFieldType.isRequired() + || (datasetFieldType.isRequiredDV() && (level != null)))) { + fieldsBuilder.add(datasetFieldType.getName(), json(datasetFieldType, ownerDataverse)); + } + } } - + jsonObjectBuilder.add("fields", fieldsBuilder); return jsonObjectBuilder; } + // This will remove datasetFieldTypes that are in the list but also a child of another datasetFieldType in the list + // Prevents duplicate datasetFieldType information from being returned twice + // See: https://github.com/IQSS/dataverse/issues/10472 + private static Set filterOutDuplicateDatasetFieldTypes(List datasetFieldTypesList) { + // making a copy of the list as to not damage the original when we remove items + List datasetFieldTypes = new ArrayList<>(datasetFieldTypesList); + // exclude/remove datasetFieldTypes if datasetFieldType exists as a child of another datasetFieldType + datasetFieldTypesList.forEach(dsft -> dsft.getChildDatasetFieldTypes().forEach(c -> datasetFieldTypes.remove(c))); + return new TreeSet<>(datasetFieldTypes); + } + public static JsonArrayBuilder jsonDatasetFieldTypes(List fields) { JsonArrayBuilder fieldsJson = Json.createArrayBuilder(); for (DatasetFieldType field : fields) { @@ -678,7 +714,7 @@ public static JsonObjectBuilder json(DatasetFieldType fld, Dataverse ownerDatave JsonObjectBuilder fieldsBld = jsonObjectBuilder(); fieldsBld.add("name", fld.getName()); fieldsBld.add("displayName", fld.getDisplayName()); - fieldsBld.add("displayOnCreate", fld.isDisplayOnCreate()); + fieldsBld.add("displayOnCreate", fld.shouldDisplayOnCreate()); fieldsBld.add("title", fld.getTitle()); fieldsBld.add("type", fld.getFieldType().toString()); fieldsBld.add("typeClass", typeClassString(fld)); @@ -689,8 +725,8 @@ public static JsonObjectBuilder json(DatasetFieldType fld, Dataverse ownerDatave fieldsBld.add("displayFormat", fld.getDisplayFormat()); fieldsBld.add("displayOrder", fld.getDisplayOrder()); - boolean requiredInOwnerDataverse = ownerDataverse != null && ownerDataverse.isDatasetFieldTypeRequiredAsInputLevel(fld.getId()); - fieldsBld.add("isRequired", requiredInOwnerDataverse || fld.isRequired()); + boolean inLevel= ownerDataverse != null && ownerDataverse.isDatasetFieldTypeInInputLevels(fld.getId()); + fieldsBld.add("isRequired", (fld.isRequiredDV() && inLevel) || fld.isRequired()); if (fld.isControlledVocabulary()) { // If the field has a controlled vocabulary, @@ -705,7 +741,20 @@ public static JsonObjectBuilder json(DatasetFieldType fld, Dataverse ownerDatave if (!fld.getChildDatasetFieldTypes().isEmpty()) { JsonObjectBuilder subFieldsBld = jsonObjectBuilder(); for (DatasetFieldType subFld : fld.getChildDatasetFieldTypes()) { - subFieldsBld.add(subFld.getName(), JsonPrinter.json(subFld, ownerDataverse)); + subFld.setInclude(true); + if (ownerDataverse != null) { + DataverseFieldTypeInputLevel childLevel = ownerDataverse + .getDatasetFieldTypeInInputLevels(subFld.getId()); + if (childLevel != null) { + subFld.setLocalDisplayOnCreate(childLevel.getDisplayOnCreate()); + subFld.setRequiredDV(childLevel.isRequired()); + subFld.setInclude(childLevel.isInclude()); + } + } + //This assumes a child have can't be displayOnCreate=false when the parent has it true (i.e. we're not excluding children based on testing displayOnCreate (or required) here.) + if(subFld.isInclude()) { + subFieldsBld.add(subFld.getName(), JsonPrinter.json(subFld, ownerDataverse)); + } } fieldsBld.add("childFields", subFieldsBld); } @@ -1001,6 +1050,7 @@ public static JsonObjectBuilder json(HarvestingClient harvestingClient) { } return jsonObjectBuilder().add("nickName", harvestingClient.getName()). + add("sourceName", harvestingClient.getSourceName()). add("dataverseAlias", harvestingClient.getDataverse().getAlias()). add("type", harvestingClient.getHarvestType()). add("style", harvestingClient.getHarvestStyle()). @@ -1013,6 +1063,7 @@ public static JsonObjectBuilder json(HarvestingClient harvestingClient) { add("status", harvestingClient.isHarvestingNow() ? "inProgress" : "inActive"). add("customHeaders", harvestingClient.getCustomHttpHeaders()). add("allowHarvestingMissingCVV", harvestingClient.getAllowHarvestingMissingCVV()). + add("useListRecords", harvestingClient.isUseListRecords()). add("useOaiIdentifiersAsPids", harvestingClient.isUseOaiIdentifiersAsPids()). add("lastHarvest", harvestingClient.getLastHarvestTime() == null ? null : harvestingClient.getLastHarvestTime().toString()). add("lastResult", harvestingClient.getLastResult()). @@ -1222,6 +1273,7 @@ public static JsonObjectBuilder json(Retention retention) { } public static JsonObjectBuilder json(License license) { + return jsonObjectBuilder() .add("id", license.getId()) .add("name", license.getName()) @@ -1230,7 +1282,11 @@ public static JsonObjectBuilder json(License license) { .add("iconUrl", license.getIconUrl() == null ? null : license.getIconUrl().toString()) .add("active", license.isActive()) .add("isDefault", license.isDefault()) - .add("sortOrder", license.getSortOrder()); + .add("sortOrder", license.getSortOrder()) + .add("rightsIdentifier", license.getRightsIdentifier()) + .add("rightsIdentifierScheme", license.getRightsIdentifierScheme()) + .add("schemeUri", license.getSchemeUri() == null ? null : license.getSchemeUri().toString()) + .add("languageCode", license.getLanguageCode()); } public static Collector stringsToJsonArray() { @@ -1384,8 +1440,15 @@ private static JsonObjectBuilder jsonLicense(DatasetVersion dsv) { .add("name", DatasetUtil.getLicenseName(dsv)) .add("uri", DatasetUtil.getLicenseURI(dsv)); String licenseIconUri = DatasetUtil.getLicenseIcon(dsv); - if (licenseIconUri != null) { - licenseJsonObjectBuilder.add("iconUri", licenseIconUri); + licenseJsonObjectBuilder.add("iconUri", licenseIconUri); + License license = DatasetUtil.getLicense(dsv); + if(license != null) { + licenseJsonObjectBuilder.add("rightsIdentifier",license.getRightsIdentifier()) + .add("rightsIdentifierScheme", license.getRightsIdentifierScheme()) + .add("schemeUri", license.getSchemeUri()) + .add("languageCode", license.getLanguageCode()); + } else { + licenseJsonObjectBuilder.add("languageCode", BundleUtil.getDefaultLocale().getLanguage()); } return licenseJsonObjectBuilder; } @@ -1397,6 +1460,7 @@ public static JsonArrayBuilder jsonDataverseFieldTypeInputLevels(List dataverseFeaturedItems) { + JsonArrayBuilder featuredItemsArrayBuilder = Json.createArrayBuilder(); + for (DataverseFeaturedItem dataverseFeaturedItem : dataverseFeaturedItems) { + featuredItemsArrayBuilder.add(json(dataverseFeaturedItem)); + } + return featuredItemsArrayBuilder; + } + + public static JsonObjectBuilder json(DataverseFeaturedItem dataverseFeaturedItem) { + return jsonObjectBuilder() + .add("id", dataverseFeaturedItem.getId()) + .add("content", dataverseFeaturedItem.getContent()) + .add("imageFileName", dataverseFeaturedItem.getImageFileName()) + .add("imageFileUrl", dataverseFeaturedItem.getImageFileUrl()) + .add("displayOrder", dataverseFeaturedItem.getDisplayOrder()); + } } diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinterHelper.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinterHelper.java index 55f9ecb5ce8..43d553c93e7 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinterHelper.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinterHelper.java @@ -1,6 +1,7 @@ package edu.harvard.iq.dataverse.util.json; import edu.harvard.iq.dataverse.DatasetFieldServiceBean; +import edu.harvard.iq.dataverse.DataverseFieldTypeInputLevelServiceBean; import edu.harvard.iq.dataverse.settings.SettingsServiceBean; import jakarta.annotation.PostConstruct; @@ -22,8 +23,11 @@ public class JsonPrinterHelper { @EJB DatasetFieldServiceBean datasetFieldSvc; + @EJB + DataverseFieldTypeInputLevelServiceBean datasetFieldInpuLevelSvc; + @PostConstruct public void injectService() { - JsonPrinter.injectSettingsService(settingsSvc, datasetFieldSvc); + JsonPrinter.injectSettingsService(settingsSvc, datasetFieldSvc, datasetFieldInpuLevelSvc); } } diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/NullSafeJsonBuilder.java b/src/main/java/edu/harvard/iq/dataverse/util/json/NullSafeJsonBuilder.java index ef8ab39122f..21360fcd708 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/NullSafeJsonBuilder.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/NullSafeJsonBuilder.java @@ -85,7 +85,10 @@ public NullSafeJsonBuilder add(String name, boolean value) { delegate.add(name, value); return this; } - + public NullSafeJsonBuilder add(String name, Boolean value) { + return (value != null) ? add(name, value.booleanValue()) : this; + } + @Override public NullSafeJsonBuilder addNull(String name) { delegate.addNull(name); diff --git a/src/main/java/edu/harvard/iq/dataverse/validation/URLValidator.java b/src/main/java/edu/harvard/iq/dataverse/validation/URLValidator.java index 8fde76d84e1..55f69ee1471 100644 --- a/src/main/java/edu/harvard/iq/dataverse/validation/URLValidator.java +++ b/src/main/java/edu/harvard/iq/dataverse/validation/URLValidator.java @@ -7,6 +7,7 @@ * * @author skraffmi */ +//Not currently used except in tests public class URLValidator implements ConstraintValidator { private String[] allowedSchemes; diff --git a/src/main/java/edu/harvard/iq/dataverse/validation/ValidateURL.java b/src/main/java/edu/harvard/iq/dataverse/validation/ValidateURL.java index 3834b119598..7c368ea3934 100644 --- a/src/main/java/edu/harvard/iq/dataverse/validation/ValidateURL.java +++ b/src/main/java/edu/harvard/iq/dataverse/validation/ValidateURL.java @@ -13,6 +13,7 @@ @Retention(RUNTIME) @Constraint(validatedBy = {URLValidator.class}) @Documented +//Not currently used except in tests public @interface ValidateURL { String message() default "'${validatedValue}' {url.invalid}"; String[] schemes() default {"http", "https", "ftp"}; diff --git a/src/main/java/propertyFiles/3dobjects.properties b/src/main/java/propertyFiles/3dobjects.properties new file mode 100644 index 00000000000..97b8b0698dc --- /dev/null +++ b/src/main/java/propertyFiles/3dobjects.properties @@ -0,0 +1,74 @@ +metadatablock.name=3dobjects +metadatablock.displayName=3D Objects Metadata +metadatablock.displayFacet= +datasetfieldtype.3d3DTechnique.title=3D Technique +datasetfieldtype.3dEquipment.title=Equipment +datasetfieldtype.3dLightingSetup.title=Lighting Setup +datasetfieldtype.3dMasterFilePolygonCount.title=Master File Polygon Count +datasetfieldtype.3dExportedFilePolygonCount.title=Exported File Polygon Count +datasetfieldtype.3dExportedFileFormat.title=Exported File Format +datasetfieldtype.3dAltText.title=Alt-Text +datasetfieldtype.3dMaterialComposition.title=Material Composition +datasetfieldtype.3dObjectDimensions.title=Object Dimensions +datasetfieldtype.3dLength.title=Length +datasetfieldtype.3dWidth.title=Width +datasetfieldtype.3dHeight.title=Height +datasetfieldtype.3dWeight.title=Weight +datasetfieldtype.3dUnit.title=Unit +datasetfieldtype.3dHandling.title=Instructions +datasetfieldtype.3d3DTechnique.description=The technique used for capturing the 3D data +datasetfieldtype.3dEquipment.description=The equipment used for capturing the 3D data +datasetfieldtype.3dLightingSetup.description=The lighting used while capturing the 3D data +datasetfieldtype.3dMasterFilePolygonCount.description=The high-resolution polygon count +datasetfieldtype.3dExportedFilePolygonCount.description=The exported mesh polygon count +datasetfieldtype.3dExportedFileFormat.description=The format of the exported mesh +datasetfieldtype.3dAltText.description=A physical description of the object modeled +datasetfieldtype.3dMaterialComposition.description=The material used to create the object, e.g. stone +datasetfieldtype.3dObjectDimensions.description=The general measurements of the physical object +datasetfieldtype.3dLength.description=The rough length of the object +datasetfieldtype.3dWidth.description=The rough width of the object +datasetfieldtype.3dHeight.description=The rough height of the object +datasetfieldtype.3dWeight.description=The rough weight of the object +datasetfieldtype.3dUnit.description=The unit of measurement used for the object dimensions +datasetfieldtype.3dHandling.description=Safety and special handling instructions for the object +datasetfieldtype.3d3DTechnique.watermark= +datasetfieldtype.3dEquipment.watermark= +datasetfieldtype.3dLightingSetup.watermark= +datasetfieldtype.3dMasterFilePolygonCount.watermark= +datasetfieldtype.3dExportedFilePolygonCount.watermark= +datasetfieldtype.3dExportedFileFormat.watermark= +datasetfieldtype.3dAltText.watermark= +datasetfieldtype.3dMaterialComposition.watermark= +datasetfieldtype.3dObjectDimensions.watermark= +datasetfieldtype.3dLength.watermark= +datasetfieldtype.3dWidth.watermark= +datasetfieldtype.3dHeight.watermark= +datasetfieldtype.3dWeight.watermark= +datasetfieldtype.3dUnit.watermark= +datasetfieldtype.3dHandling.watermark= +controlledvocabulary.3d3DTechnique.ir_scanner=IR Scanner +controlledvocabulary.3d3DTechnique.laser=Laser +controlledvocabulary.3d3DTechnique.modelled=Modelled +controlledvocabulary.3d3DTechnique.photogrammetry=Photogrammetry +controlledvocabulary.3d3DTechnique.rti=RTI +controlledvocabulary.3d3DTechnique.structured_light=Structured Light +controlledvocabulary.3d3DTechnique.tomographic=Tomographic +controlledvocabulary.3d3DTechnique.other=Other +controlledvocabulary.3dLightingSetup.natural_light=Natural Light +controlledvocabulary.3dLightingSetup.lightbox=Lightbox +controlledvocabulary.3dLightingSetup.led=LED +controlledvocabulary.3dLightingSetup.fluorescent=Fluorescent +controlledvocabulary.3dLightingSetup.other=Other +controlledvocabulary.3dUnit.cm=cm +controlledvocabulary.3dUnit.m=m +controlledvocabulary.3dUnit.in=in +controlledvocabulary.3dUnit.ft=ft +controlledvocabulary.3dUnit.lbs=lbs +controlledvocabulary.3dExportedFileFormat..fbx=.fbx +controlledvocabulary.3dExportedFileFormat..glb=.glb +controlledvocabulary.3dExportedFileFormat..gltf=.gltf +controlledvocabulary.3dExportedFileFormat..obj=.obj +controlledvocabulary.3dExportedFileFormat..stl=.stl +controlledvocabulary.3dExportedFileFormat..usdz=.usdz +controlledvocabulary.3dExportedFileFormat..x3d=.x3d +controlledvocabulary.3dExportedFileFormat.other=other diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties index c47356008ff..703426ded7c 100644 --- a/src/main/java/propertyFiles/Bundle.properties +++ b/src/main/java/propertyFiles/Bundle.properties @@ -72,7 +72,7 @@ affiliation=Affiliation storage=Storage curationLabels=Curation Labels metadataLanguage=Dataset Metadata Language -guestbookEntryOption=Guestbook Entry Option +guestbookEntryOption=Guestbook Mode pidProviderOption=PID Provider Option createDataverse=Create Dataverse remove=Remove @@ -215,6 +215,13 @@ user.helpShibUserMigrateOffShibAfterLink=for assistance. user.helpOAuthBeforeLink=Your Dataverse account uses {0} for login. If you are interested in changing login methods, please contact user.helpOAuthAfterLink=for assistance. user.lostPasswdTip=If you have lost or forgotten your password, please enter your username or email address below and click Submit. We will send you an e-mail with your new password. + +user.orcid=ORCID +user.orcid.link=Link to ORCID profile +user.orcid.authenticate=Add Authenticated ORCID +user.orcid.remove=Remove ORCID +user.orcid.callback.message=Authentication Error - Dataverse could not authenticate your login at ORCID and will not add your ORCID to your profile. Please make sure you authorize your account to connect with Dataverse. + user.dataRelatedToMe=My Data wasCreatedIn=, was created in wasCreatedTo=, was added to @@ -513,6 +520,8 @@ oauth2.callback.page.title=OAuth Callback oauth2.callback.message=Authentication Error - Dataverse could not authenticate your login with the provider that you selected. Please make sure you authorize your account to connect with Dataverse. For more details about the information being requested, see the User Guide. oauth2.callback.error.providerDisabled=This authentication method ({0}) is currently disabled. Please log in using one of the supported methods. oauth2.callback.error.signupDisabledForProvider=Sorry, signup for new accounts using {0} authentication is currently disabled. +oauth2.callback.error.accountNotFound=You must be logged in to associate an ORCID with your account. +oauth2.callback.error.orcidInUse=This ORCID ({0}) is already associated with another user account. # deactivated user accounts deactivated.error=Sorry, your account has been deactivated. @@ -542,6 +551,9 @@ dashboard.card.harvestingserver.sets={0, choice, 0#Sets|1#Set|2#Sets} dashboard.card.harvestingserver.btn.manage=Manage Server dashboard.card.metadataexport.header=Metadata Export dashboard.card.metadataexport.message=Dataset metadata export is only available through the {0} API. Learn more in the {0} {1}API Guide{2}. +dashboard.card.move.data=Data +dashboard.card.move.dataset.manage=Move Dataset +dashboard.card.move.dataverse.manage=Move Dataverse #harvestclients.xhtml harvestclients.title=Manage Harvesting Clients @@ -576,6 +588,8 @@ harvestclients.newClientDialog.nickname.helptext=Consists of letters, digits, un harvestclients.newClientDialog.nickname.required=Client nickname cannot be empty! harvestclients.newClientDialog.nickname.invalid=Client nickname can contain only letters, digits, underscores (_) and dashes (-); and must be at most 30 characters. harvestclients.newClientDialog.nickname.alreadyused=This nickname is already used. +harvestclients.newClientDialog.sourcename=Source Name +harvestclients.newClientDialog.sourcename.helptext=When the feature flag "index-harvested-metadata-source" is enabled, the source name will override the nickname in the "Metadata Source" facet. You can group multiple harvesting clients together under that facet by using the same source name. harvestclients.newClientDialog.customHeader=Custom HTTP Header harvestclients.newClientDialog.customHeader.helptext=(Optional) Custom HTTP header to add to requests, if required by this OAI server. harvestclients.newClientDialog.customHeader.watermark=Enter an http header, as in header-name: header-value @@ -636,6 +650,13 @@ harvestclients.viewEditDialog.archiveDescription.tip=Description of the archival harvestclients.viewEditDialog.archiveDescription.default.generic=This Dataset is harvested from our partners. Clicking the link will take you directly to the archival source of the data. harvestclients.viewEditDialog.btn.save=Save Changes harvestclients.newClientDialog.title.edit=Edit Group {0} +harvestclients.result.completed=Completed +harvestclients.result.completedWithFailures=Completed with failures +harvestclients.result.failure=FAILED +harvestclients.result.inProgess=IN PROGRESS +harvestclients.result.deleteInProgress=DELETE IN PROGRESS +harvestclients.result.interrupted=INTERRUPTED +harvestclients.result.details={0} harvested, {1} deleted, {2} failed. #harvestset.xhtml harvestserver.title=Manage Harvesting Server @@ -747,32 +768,45 @@ dashboard.list_users.toggleSuperuser.confirmationText.remove=Are you sure you wa dashboard.list_users.api.auth.invalid_apikey=The API key is invalid. dashboard.list_users.api.auth.not_superuser=Forbidden. You must be a superuser. -#dashboard-datamove.xhtml -dashboard.card.datamove=Data -dashboard.card.datamove.header=Dashboard - Move Data -dashboard.card.datamove.manage=Move Data -dashboard.card.datamove.message=Manage and curate your installation by moving datasets from one host dataverse to another. See also Managing Datasets and Dataverses in the Admin Guide. -dashboard.card.datamove.selectdataset.header=Dataset to Move -dashboard.card.datamove.newdataverse.header=New Host Dataverse -dashboard.card.datamove.dataset.label=Dataset -dashboard.card.datamove.dataverse.label=Dataverse -dashboard.card.datamove.confirm.dialog=Are you sure want to move this dataset? -dashboard.card.datamove.confirm.yes=Yes, Move Data -dashboard.card.datamove.message.success=The dataset "{0}" ({1}) has been successfully moved to {2}. -dashboard.card.datamove.message.failure.summary=Failed to moved dataset -dashboard.card.datamove.message.failure.details=The dataset "{0}" ({1}) could not be moved to {2}. {3}{4} -dashboard.card.datamove.dataverse.placeholder=Enter Dataverse Identifier... -dashboard.card.datamove.dataverse.menu.header=Dataverse Name (Affiliate), Identifier -dashboard.card.datamove.dataverse.menu.invalidMsg=No matches found -dashboard.card.datamove.dataset.placeholder=Enter Dataset Persistent ID, doi:... -dashboard.card.datamove.dataset.menu.header=Dataset Persistent ID, Title, Host Dataverse Identifier -dashboard.card.datamove.dataset.menu.invalidMsg=No matches found -dashboard.card.datamove.dataset.command.error.targetDataverseUnpublishedDatasetPublished=A published dataset may not be moved to an unpublished dataverse. You can retry the move after publishing {0}. -dashboard.card.datamove.dataset.command.error.targetDataverseSameAsOriginalDataverse=This dataset is already in this dataverse. -dashboard.card.datamove.dataset.command.error.unforced.datasetGuestbookNotInTargetDataverse=The guestbook would be removed from this dataset if you moved it because the guestbook is not in the new host dataverse. -dashboard.card.datamove.dataset.command.error.unforced.linkedToTargetDataverseOrOneOfItsParents=This dataset is linked to the new host dataverse or one of its parents. This move would remove the link to this dataset. -dashboard.card.datamove.dataset.command.error.unforced.suggestForce=Forcing this move is currently only available via API. Please see "Move a Dataset" under Managing Datasets and Dataverses in the Admin Guide for details. -dashboard.card.datamove.dataset.command.error.indexingProblem=Dataset could not be moved. Indexing failed. +#dashboard-movedataset.xhtml +dashboard.move.dataset.header=Dashboard - Move Data +dashboard.move.dataset.message=Manage and curate your installation by moving datasets from one host dataverse to another. See also Managing Datasets and Dataverses in the Admin Guide. +dashboard.move.dataset.selectdataset.header=Dataset to move +dashboard.move.dataset.newdataverse.header=New dataverse collection host +dashboard.move.dataset.dataset.label=Dataset +dashboard.move.dataset.dataverse.label=Dataverse +dashboard.move.dataset.confirm.dialog=Are you sure you want to move this dataset? +dashboard.move.dataset.confirm.yes=Yes, move this dataset +dashboard.move.dataset.message.success=The dataset "{0}" ({1}) has been successfully moved to {2}. +dashboard.move.dataset.message.failure.summary=Failed to moved dataset +dashboard.move.dataset.message.failure.details=The dataset "{0}" ({1}) could not be moved to {2}. {3}{4} +dashboard.move.dataset.dataverse.placeholder=Enter Dataverse Identifier... +dashboard.move.dataset.dataverse.menu.header=Dataverse Name (Affiliate), Identifier +dashboard.move.dataset.dataverse.menu.invalidMsg=No matches found +dashboard.move.dataset.placeholder=Enter Dataset Persistent ID, doi:... +dashboard.move.dataset.menu.header=Dataset Persistent ID, Title, Host Dataverse Identifier +dashboard.move.dataset.menu.invalidMsg=No matches found +dashboard.move.dataset.command.error.targetDataverseUnpublishedDatasetPublished=A published dataset may not be moved to an unpublished dataverse. You can retry the move after publishing {0}. +dashboard.move.dataset.command.error.targetDataverseSameAsOriginalDataverse=This dataset is already in this dataverse. +dashboard.move.dataset.command.error.unforced.datasetGuestbookNotInTargetDataverse=The guestbook would be removed from this dataset if you moved it because the guestbook is not in the new host dataverse. +dashboard.move.dataset.command.error.unforced.linkedToTargetDataverseOrOneOfItsParents=This dataset is linked to the new host dataverse or one of its parents. This move would remove the link to this dataset. +dashboard.move.dataset.command.error.unforced.suggestForce=Forcing this move is currently only available via API. Please see "Move a Dataset" under Managing Datasets and Dataverses in the Admin Guide for details. + +#dashboard-movedataverse.xhtml +dashboard.move.dataverse.header=Dashboard - Move Data +dashboard.move.dataverse.message.summary=Move Dataverse Collection +dashboard.move.dataverse.message.detail=Manage and curate your installation by moving a dataverse collection from one host dataverse collection to another. See also Managing Datasets and Dataverses in the Admin Guide. +dashboard.move.dataverse.selectdataverse.header=Dataverse collection to move +dashboard.move.dataverse.newdataverse.header=New dataverse collection host +dashboard.move.dataverse.label=Dataverse +dashboard.move.dataverse.confirm.dialog=Are you sure you want to move this dataverse collection? +dashboard.move.dataverse.confirm.yes=Yes, move this collection +dashboard.move.dataverse.message.success=The dataverse "{0}" has been successfully moved to {1}. +dashboard.move.dataverse.message.failure.summary=Failed to moved dataverse +dashboard.move.dataverse.message.failure.details=The dataverse "{0}" could not be moved to {1}. {2} +dashboard.move.dataverse.placeholder=Enter Dataverse Identifier... +dashboard.move.dataverse.menu.header=Dataverse Name (Affiliate), Identifier +dashboard.move.dataverse.menu.invalidMsg=No matches found #MailServiceBean.java notification.email.create.dataverse.subject={0}: Your dataverse has been created @@ -804,7 +838,7 @@ notification.email.greeting.html=Hello,
    # Bundle file editors, please note that "notification.email.welcome" is used in a unit test notification.email.welcome=Welcome to {0}! Get started by adding or finding data. Have questions? Check out the User Guide at {1}/{2}/user or contact {3} at {4} for assistance. notification.email.welcomeConfirmEmailAddOn=\n\nPlease verify your email address at {0} . Note, the verify link will expire after {1}. Send another verification email by visiting your account page. -notification.email.requestFileAccess=File access requested for dataset: {0} by {1} ({2}). Manage permissions at {3}. +notification.email.requestFileAccess=File access requested for dataset: {0} by {1} ({2}). Manage permissions at {3} . notification.email.requestFileAccess.guestbookResponse=

    Guestbook Response:

    {0} notification.email.grantFileAccess=Access granted for files in dataset: {0} (view at {1} ). notification.email.rejectFileAccess=Your request for access was rejected for the requested files in the dataset: {0} (view at {1} ). If you have any questions about why your request was rejected, you may reach the dataset owner using the "Contact" link on the upper right corner of the dataset page. @@ -862,7 +896,7 @@ dataverse.curationLabels.title=A set of curation status labels that are used to dataverse.curationLabels.disabled=Disabled dataverse.category=Category dataverse.category.title=The type that most closely reflects this dataverse. -dataverse.guestbookentryatrequest.title=Whether Guestbooks are displayed to users when they request file access or when they download files. +dataverse.guestbookentryatrequest.title=Whether Guestbooks (when configured for a dataset) are displayed to users when they request file access or when they download files. dataverse.pidProvider.title=The source of PIDs (DOIs, Handles, etc.) when a new PID is created. dataverse.type.selectTab.top=Select one... dataverse.type.selectTab.researchers=Researcher @@ -977,8 +1011,17 @@ dataverse.inputlevels.error.cannotberequiredifnotincluded=The input level for th dataverse.facets.error.fieldtypenotfound=Can't find dataset field type '{0}' dataverse.facets.error.fieldtypenotfacetable=Dataset field type '{0}' is not facetable dataverse.metadatablocks.error.invalidmetadatablockname=Invalid metadata block name: {0} +dataverse.metadatablocks.error.containslistandinheritflag=Metadata block can not contain both {0} and {1}: true dataverse.create.error.jsonparse=Error parsing Json: {0} dataverse.create.error.jsonparsetodataverse=Error parsing the POSTed json into a dataverse: {0} +dataverse.create.featuredItem.error.imageFileProcessing=Error processing featured item file: {0} +dataverse.create.featuredItem.error.fileSizeExceedsLimit=File exceeds the maximum size of {0} +dataverse.create.featuredItem.error.invalidFileType=Invalid image file type +dataverse.create.featuredItem.error.contentShouldBeProvided=Featured item 'content' property should be provided and not empty. +dataverse.create.featuredItem.error.contentExceedsLengthLimit=Featured item content exceeds the maximum allowed length of {0} characters. +dataverse.update.featuredItems.error.missingInputParams=All input parameters (id, content, displayOrder, keepFile, fileName) are required. +dataverse.update.featuredItems.error.inputListsSizeMismatch=All input lists (id, content, displayOrder, keepFile, fileName) must have the same size. +dataverse.delete.featuredItems.success=All featured items of this Dataverse have been successfully deleted. # rolesAndPermissionsFragment.xhtml # advanced.xhtml @@ -1549,6 +1592,7 @@ dataset.mayNotPublish.administrator= This dataset cannot be published until {0} dataset.mayNotPublish.both= This dataset cannot be published until {0} is published. Would you like to publish both right now? dataset.mayNotPublish.twoGenerations= This dataset cannot be published until {0} and {1} are published. dataset.mayNotBePublished.both.button=Yes, Publish Both +dataset.mayNotPublish.FilesRequired=Published datasets should contain at least one data file. dataset.viewVersion.unpublished=View Unpublished Version dataset.viewVersion.published=View Published Version dataset.link.title=Link Dataset @@ -1610,9 +1654,18 @@ dataset.cite.title.draft=DRAFT VERSION will be replaced in the citation with the dataset.cite.title.deassessioned=DEACCESSIONED VERSION has been added to the citation for this version since it is no longer available. dataset.cite.standards.tip=Learn about Data Citation Standards. dataset.cite.downloadBtn=Cite Dataset -dataset.cite.downloadBtn.xml=EndNote XML -dataset.cite.downloadBtn.ris=RIS -dataset.cite.downloadBtn.bib=BibTeX +dataset.cite.downloadBtn.xml=Download EndNote XML +dataset.cite.downloadBtn.ris=Download RIS +dataset.cite.downloadBtn.bib=Download BibTeX +dataset.cite.viewCitation=View Styled Citation +dataset.cite.cslDialog.title=Styled Citation +dataset.cite.cslDialog.select=Select a CSL style +dataset.cite.cslDialog.commonStyles=Common Styles +dataset.cite.cslDialog.otherStyles=More Styles +dataset.cite.cslDialog.citation=Citation in "{0}" style +dataset.cite.cslDialog.generating=Generating Citation... +dataset.cite.cslDialog.copy=Copy Citation to Clipboard +dataset.cite.cslDialog.close=Close dataset.create.authenticatedUsersOnly=Only authenticated users can create datasets. dataset.deaccession.reason=Deaccession Reason dataset.beAccessedAt=The dataset can now be accessed at: @@ -1676,6 +1729,7 @@ dataset.message.createFailure=The dataset could not be created. dataset.message.termsFailure=The dataset terms could not be updated. dataset.message.label.fileAccess=Publicly-accessible storage dataset.message.publicInstall=Files in this dataset may be readable outside Dataverse, restricted and embargoed access are disabled +dataset.message.versionNoteSuccess=Version note successfully updated. dataset.message.parallelUpdateError=Changes cannot be saved. This dataset has been edited since this page was opened. To continue, copy your changes, refresh the page to see the recent updates, and re-enter any changes you want to save. dataset.message.parallelPublishError=Publishing is blocked. This dataset has been edited since this page was opened. To publish it, refresh the page to see the recent updates, and publish again. dataset.metadata.publicationDate=Publication Date @@ -1729,27 +1783,27 @@ dataset.transferUnrestricted=Click Continue to transfer the elligible files. dataset.requestAccessToRestrictedFiles=You may request access to any restricted file(s) by clicking the Request Access button. dataset.requestAccessToRestrictedFilesWithEmbargo=Embargoed files cannot be accessed during the embargo period. If your selection contains restricted files, you may request access to them by clicking the Request Access button. -dataset.privateurl.infoMessageAuthor=Privately share this dataset before it is published: {0} +dataset.privateurl.infoMessageAuthor=Privately share this draft dataset before it is published: {0} dataset.privateurl.infoMessageReviewer=You are viewing a preview of this unpublished dataset version. dataset.privateurl.header=Unpublished Dataset Preview URL dataset.privateurl.tip=To cite this data in publications, use the dataset's persistent ID instead of this URL. For more information about the Preview URL feature, please refer to the User Guide. -dataset.privateurl.onlyone=Only one Preview URL can be active for a single dataset. +dataset.privateurl.onlyone=Only one Preview URL can be active for a single draft dataset. dataset.privateurl.absent=Preview URL has not been created. dataset.privateurl.general.button.label=Create General Preview URL -dataset.privateurl.general.description=Create a URL that others can use to review this dataset version before it is published. They will be able to access all files in the dataset and see all metadata, including metadata that may identify the dataset's authors. +dataset.privateurl.general.description=Create a URL that others can use to review this draft dataset version before it is published. They will be able to access all files in the dataset and see all metadata, including metadata that may identify the dataset's authors. dataset.privateurl.general.title=General Preview dataset.privateurl.anonymous.title=Anonymous Preview +dataset.privateurl.anonymous.tooltip.preface=The following metadata fields will be hidden from the user of this Anonymous Preview URL: dataset.privateurl.anonymous.button.label=Create Anonymous Preview URL -dataset.privateurl.anonymous.description=Create a URL that others can use to access an anonymized view of this unpublished dataset version. Metadata that could identify the dataset author will not be displayed. Non-identifying metadata will be visible. -dataset.privateurl.anonymous.description.paragraph.two=The dataset's files are not changed and will be accessible if they're not restricted. Users of the Anonymous Preview URL will not be able to see the name of the Dataverse that this dataset is in but will be able to see the name of the repository, which might expose the dataset authors' identities. +dataset.privateurl.anonymous.description=Create a URL that others can use to access an anonymized view of this unpublished dataset version. Metadata that could identify the dataset's author will not be displayed. (See Tool Tip for the list of withheld metadata fields.) Non-identifying metadata will be visible. +dataset.privateurl.anonymous.description.paragraph.two=The dataset's files are not changed and users of the Anonymous Preview URL will be able to access them. Users of the Anonymous Preview URL will not be able to see the name of the Dataverse that this dataset is in but will be able to see the name of the repository, which might expose the dataset authors' identities. +dataset.privateurl.anonymous.description.paragraph.three=To verify that all identifying information has been removed or anonymized, it is recommended that you logout and review the dataset as as it would be seen by an Anonymous Preview URL user. See User Guide for more information. dataset.privateurl.createPrivateUrl=Create Preview URL dataset.privateurl.introduction=You can create a Preview URL to copy and share with others who will not need a repository account to review this unpublished dataset version. Once the dataset is published or if the URL is disabled, the URL will no longer work and will point to a "Page not found" page. dataset.privateurl.createPrivateUrl.anonymized=Create URL for Anonymized Access -dataset.privateurl.createPrivateUrl.anonymized.unavailable=Anonymized Access is not available once a version of the dataset has been published -dataset.privateurl.disablePrivateUrl=Disable Preview URL +dataset.privateurl.createPrivateUrl.anonymized.unavailable=You won't be able to create an Anonymous Preview URL once a version of this dataset has been published. dataset.privateurl.disableGeneralPreviewUrl=Disable General Preview URL dataset.privateurl.disableAnonPreviewUrl=Disable Anonymous Preview URL -dataset.privateurl.disablePrivateUrlConfirm=Yes, Disable Preview URL dataset.privateurl.disableGeneralPreviewUrlConfirm=Yes, Disable General Preview URL dataset.privateurl.disableAnonPreviewUrlConfirm=Yes, Disable Anonymous Preview URL dataset.privateurl.disableConfirmationText=Are you sure you want to disable the Preview URL? If you have shared the Preview URL with others they will no longer be able to use it to access your unpublished dataset. @@ -2047,6 +2101,7 @@ file.dataFilesTab.button.direct=Direct file.dataFilesTab.versions=Versions file.dataFilesTab.versions.headers.dataset=Dataset Version file.dataFilesTab.versions.headers.summary=Summary +file.dataFilesTab.versions.headers.versionNote=Version Note file.dataFilesTab.versions.headers.contributors=Contributors file.dataFilesTab.versions.headers.contributors.withheld=Contributor name(s) withheld file.dataFilesTab.versions.headers.published=Published on @@ -2071,6 +2126,7 @@ file.dataFilesTab.versions.description.firstPublished=This is the first publishe file.dataFilesTab.versions.description.deaccessionedReason=Deaccessioned Reason: file.dataFilesTab.versions.description.beAccessedAt=The dataset can now be accessed at: file.dataFilesTab.versions.viewDetails.btn=View Details +file.dataFilesTab.versions.versionNote.btn=Edit Note file.dataFilesTab.versions.widget.viewMoreInfo=To view more information about the versions of this dataset, and to edit it if this is your dataset, please visit the full version of this dataset at the {2}. file.dataFilesTab.versions.preloadmessage=(Loading versions...) file.previewTab.externalTools.header=Available Previews @@ -2169,6 +2225,11 @@ file.auxfiles.types.NcML=XML from NetCDF/HDF5 (NcML) # Add more types here file.auxfiles.unspecifiedTypes=Other Auxiliary Files +dataset.version.versionNote.addEdit=Version Note +dataset.version.versionNote.title=The reason this version was created +dataset.versionNote.header=Add/Edit a Version Note +dataset.versionNote.tip=Enter the reason this version was created. To learn more about Version Notes, visit the Version Notes section of the User Guide. + # dataset-widgets.xhtml dataset.widgets.title=Dataset Thumbnail + Widgets dataset.widgets.notPublished.why.header=Why Use Widgets? @@ -2504,6 +2565,8 @@ permission.addDatasetDataverse=Add a dataset to a dataverse userPage.informationUpdated=Your account information has been successfully updated. userPage.passwordChanged=Your account password has been successfully changed. confirmEmail.changed=Your email address has changed and must be re-verified. Please check your inbox at {0} and follow the link we''ve sent. \n\nAlso, please note that the link will only work for the next {1} before it has expired. +auth.orcid.notConfigured=ORCID authentication is not configured. +auth.orcid.error=An error occurred while starting ORCID authentication. Please try again later. #Dataset.java dataset.category.documentation=Documentation @@ -2767,6 +2830,7 @@ datasets.api.thumbnail.basedOnWrongFileId=Dataset thumbnail should be based on f datasets.api.thumbnail.fileNotFound=Could not find file based on id supplied: {0} datasets.api.thumbnail.fileNotSupplied=A file was not selected to be the new dataset thumbnail. datasets.api.thumbnail.noChange=No changes to save. +datasets.api.citation.invalidFormat=Invalid Format Requested. #Dataverses.java dataverses.api.update.default.contributor.role.failure.role.not.found=Role {0} not found. @@ -2783,13 +2847,14 @@ dataverses.api.delete.featured.collections.successful=Featured dataverses have b dataverses.api.move.dataverse.error.metadataBlock=Dataverse metadata block is not in target dataverse. dataverses.api.move.dataverse.error.dataverseLink=Dataverse is linked to target dataverse or one of its parents. dataverses.api.move.dataverse.error.datasetLink=Dataset is linked to target dataverse or one of its parents. -dataverses.api.move.dataverse.error.forceMove=Please use the parameter ?forceMove=true to complete the move. This will remove anything from the dataverse that is not compatible with the target dataverse. +dataverses.api.move.dataverse.error.forceMove=Please use the API and see "Move a Dataverse Collection" with the parameter ?forceMove=true to complete the move. This will remove anything from the dataverse that is not compatible with the target dataverse. dataverses.api.create.dataset.error.mustIncludeVersion=Please provide initial version in the dataset json dataverses.api.create.dataset.error.superuserFiles=Only a superuser may add files via this api dataverses.api.create.dataset.error.mustIncludeAuthorName=Please provide author name in the dataset json dataverses.api.validate.json.succeeded=The Dataset JSON provided is valid for this Dataverse Collection. dataverses.api.validate.json.failed=The Dataset JSON provided failed validation with the following error: dataverses.api.validate.json.exception=Validation failed with following exception: +dataverses.api.update.featured.items.error.onlyImageFilesAllowed=Invalid file type. Only image files are allowed. #Access.java access.api.allowRequests.failure.noDataset=Could not find Dataset with id: {0} @@ -3085,3 +3150,41 @@ openapi.exception.invalid.format=Invalid format {0}, currently supported formats openapi.exception=Supported format definition not found. openapi.exception.unaligned=Unaligned parameters on Headers [{0}] and Request [{1}] +#Users.java +users.api.errors.bearerAuthFeatureFlagDisabled=This endpoint is only available when bearer authentication feature flag is enabled. +users.api.errors.bearerTokenRequired=Bearer token required. +users.api.errors.jsonParseToUserDTO=Error parsing the POSTed User json: {0} +users.api.userRegistered=User registered. + +#RegisterOidcUserCommand.java +registerOidcUserCommand.errors.userAlreadyRegisteredWithToken=User is already registered with this token. +registerOidcUserCommand.errors.invalidFields=The provided fields are invalid for registering a new user. +registerOidcUserCommand.errors.userShouldAcceptTerms=Terms should be accepted. +registerOidcUserCommand.errors.provideMissingClaimsEnabled.fieldAlreadyPresentInProvider=Unable to set {0} because it conflicts with an existing claim from the OIDC identity provider. +registerOidcUserCommand.errors.provideMissingClaimsEnabled.fieldRequired=It is required to include the field {0} in the request JSON for registering the user. +registerOidcUserCommand.errors.provideMissingClaimsDisabled.unableToSetFieldViaJSON=Unable to set field {0} via JSON because the api-bearer-auth-provide-missing-claims feature flag is disabled. +registerOidcUserCommand.errors.provideMissingClaimsDisabled.fieldRequired=The OIDC identity provider does not provide the user claim {0}, which is required for user registration. Please contact an administrator. +registerOidcUserCommand.errors.emailAddressInUse=Email already in use. +registerOidcUserCommand.errors.usernameInUse=Username already in use. + +#BearerTokenAuthMechanism.java +bearerTokenAuthMechanism.errors.tokenValidatedButNoRegisteredUser=Bearer token is validated, but there is no linked user account. + +#AuthenticationServiceBean.java +authenticationServiceBean.errors.unauthorizedBearerToken=Unauthorized bearer token. +authenticationServiceBean.errors.invalidBearerToken=Could not parse bearer token. +authenticationServiceBean.errors.bearerTokenDetectedNoOIDCProviderConfigured=Bearer token detected, no OIDC provider configured. + +#SendFeedbackAPI.java +sendfeedback.request.error.targetNotFound=Feedback target object not found. +sendfeedback.request.rateLimited=Too many requests to send feedback. +sendfeedback.body.error.exceedsLength=Body exceeds feedback length: {0} > {1}}. +sendfeedback.body.error.isEmpty=Body can not be empty. +sendfeedback.body.error.missingRequiredFields=Body missing required fields. +sendfeedback.fromEmail.error.missing=Missing fromEmail +sendfeedback.fromEmail.error.invalid=Invalid fromEmail: {0} + +#DataverseFeaturedItems.java +dataverseFeaturedItems.errors.notFound=Could not find dataverse featured item with identifier {0} +dataverseFeaturedItems.delete.successful=Successfully deleted dataverse featured item with identifier {0} + diff --git a/src/main/java/propertyFiles/archival.properties b/src/main/java/propertyFiles/archival.properties new file mode 100644 index 00000000000..cc460e1559e --- /dev/null +++ b/src/main/java/propertyFiles/archival.properties @@ -0,0 +1,22 @@ +metadatablock.name=archival +metadatablock.displayName=Archival Metadata +metadatablock.displayFacet=Archival Metadata +datasetfieldtype.submitToArchivalAppraisal.title=Submit to Archival Appraisal +datasetfieldtype.archivedFrom.title=Archived from +datasetfieldtype.holdingArchive.title=Holding Archive +datasetfieldtype.holdingArchiveName.title=Archived at Holding Archive +datasetfieldtype.archivedAt.title=Archived at URL +datasetfieldtype.submitToArchivalAppraisal.description=Your assessment of whether the dataset should be submitted for archival appraisal +datasetfieldtype.archivedFrom.description=A date (YYYY-MM-DD) from which the dataset is archived +datasetfieldtype.holdingArchive.description=Information on the holding archive where the dataset is archived +datasetfieldtype.holdingArchiveName.description=The name of the holding archive +datasetfieldtype.archivedAt.description=URL to holding archive +datasetfieldtype.submitToArchivalAppraisal.watermark= +datasetfieldtype.archivedFrom.watermark=YYYY-MM-DD +datasetfieldtype.archivedFor.watermark= +datasetfieldtype.holdingArchive.watermark= +datasetfieldtype.holdingArchiveName.watermark= +datasetfieldtype.archivedAt.watermark=URL +controlledvocabulary.submitToArchivalAppraisal.true=Yes +controlledvocabulary.submitToArchivalAppraisal.false=No +controlledvocabulary.submitToArchivalAppraisal.unknown=Unknown diff --git a/src/main/java/propertyFiles/citation.properties b/src/main/java/propertyFiles/citation.properties index 5899523da67..9a1e6f280ec 100644 --- a/src/main/java/propertyFiles/citation.properties +++ b/src/main/java/propertyFiles/citation.properties @@ -298,6 +298,7 @@ controlledvocabulary.contributorType.supervisor=Supervisor controlledvocabulary.contributorType.work_package_leader=Work Package Leader controlledvocabulary.contributorType.other=Other controlledvocabulary.authorIdentifierScheme.orcid=ORCID +controlledvocabulary.authorIdentifierScheme.ror=ROR controlledvocabulary.authorIdentifierScheme.isni=ISNI controlledvocabulary.authorIdentifierScheme.lcna=LCNA controlledvocabulary.authorIdentifierScheme.viaf=VIAF diff --git a/src/main/resources/META-INF/microprofile-config.properties b/src/main/resources/META-INF/microprofile-config.properties index b0bc92cf975..95f30b6ba1d 100644 --- a/src/main/resources/META-INF/microprofile-config.properties +++ b/src/main/resources/META-INF/microprofile-config.properties @@ -19,6 +19,8 @@ dataverse.files.directory=${STORAGE_DIR:/tmp/dataverse} dataverse.files.uploads=${STORAGE_DIR:${com.sun.aas.instanceRoot}}/uploads dataverse.files.docroot=${STORAGE_DIR:${com.sun.aas.instanceRoot}}/docroot dataverse.files.globus-cache-maxage=5 +dataverse.files.featured-items.image-maxsize=1000000 +dataverse.files.featured-items.image-uploads=featuredItems # SEARCH INDEX dataverse.solr.host=localhost diff --git a/src/main/resources/db/migration/V6.5.0.1.sql b/src/main/resources/db/migration/V6.5.0.1.sql new file mode 100644 index 00000000000..661924b54af --- /dev/null +++ b/src/main/resources/db/migration/V6.5.0.1.sql @@ -0,0 +1,2 @@ +-- files are required to publish datasets +ALTER TABLE dataverse ADD COLUMN IF NOT EXISTS requirefilestopublishdataset bool; diff --git a/src/main/resources/db/migration/V6.5.0.10.sql b/src/main/resources/db/migration/V6.5.0.10.sql new file mode 100644 index 00000000000..a04517c5eb8 --- /dev/null +++ b/src/main/resources/db/migration/V6.5.0.10.sql @@ -0,0 +1 @@ +ALTER TABLE makedatacountprocessstate ADD COLUMN IF NOT EXISTS server character varying(255) DEFAULT ''; diff --git a/src/main/resources/db/migration/V6.5.0.11.sql b/src/main/resources/db/migration/V6.5.0.11.sql new file mode 100644 index 00000000000..13b554ca54c --- /dev/null +++ b/src/main/resources/db/migration/V6.5.0.11.sql @@ -0,0 +1,4 @@ +-- Store authenticated orcid in URL form +ALTER TABLE authenticateduser ADD COLUMN IF NOT EXISTS authenticatedorcid VARCHAR(45); +ALTER TABLE authenticateduser DROP CONSTRAINT IF EXISTS orcid_unique; +ALTER TABLE authenticateduser ADD CONSTRAINT orcid_unique UNIQUE (authenticatedorcid); diff --git a/src/main/resources/db/migration/V6.5.0.12.sql b/src/main/resources/db/migration/V6.5.0.12.sql new file mode 100644 index 00000000000..c1b21624c8e --- /dev/null +++ b/src/main/resources/db/migration/V6.5.0.12.sql @@ -0,0 +1 @@ +CREATE INDEX IF NOT EXISTS index_dataversefieldtypeinputlevel_displayoncreate ON dataversefieldtypeinputlevel (displayoncreate); diff --git a/src/main/resources/db/migration/V6.5.0.2.sql b/src/main/resources/db/migration/V6.5.0.2.sql new file mode 100644 index 00000000000..804ce3c1ea8 --- /dev/null +++ b/src/main/resources/db/migration/V6.5.0.2.sql @@ -0,0 +1,10 @@ +-- Fixes File Access Requests when upgrading from Dataverse 6.0 +-- See: https://github.com/IQSS/dataverse/issues/10714 +DELETE FROM fileaccessrequests +WHERE creation_time <> (SELECT MIN(creation_time) + FROM fileaccessrequests far2 + WHERE far2.datafile_id = fileaccessrequests.datafile_id + AND far2.authenticated_user_id = fileaccessrequests.authenticated_user_id + AND far2.request_state is NULL); + +UPDATE fileaccessrequests SET request_state='CREATED' WHERE request_state is NULL; diff --git a/src/main/resources/db/migration/V6.5.0.3.sql b/src/main/resources/db/migration/V6.5.0.3.sql new file mode 100644 index 00000000000..e2814139e3d --- /dev/null +++ b/src/main/resources/db/migration/V6.5.0.3.sql @@ -0,0 +1,2 @@ +-- #8739 map publisher tag to distributorName when harvesting +update foreignmetadatafieldmapping set datasetfieldname = 'distributorName' where foreignfieldxpath = ':publisher'; diff --git a/src/main/resources/db/migration/V6.5.0.4.sql b/src/main/resources/db/migration/V6.5.0.4.sql new file mode 100644 index 00000000000..9c3b24712e1 --- /dev/null +++ b/src/main/resources/db/migration/V6.5.0.4.sql @@ -0,0 +1,3 @@ +ALTER TABLE dvobject ADD COLUMN IF NOT EXISTS separator character varying(255) DEFAULT ''; + +UPDATE dvobject SET separator='/' WHERE protocol = 'doi' OR protocol = 'hdl'; \ No newline at end of file diff --git a/src/main/resources/db/migration/V6.5.0.5.sql b/src/main/resources/db/migration/V6.5.0.5.sql new file mode 100644 index 00000000000..d2b03040df8 --- /dev/null +++ b/src/main/resources/db/migration/V6.5.0.5.sql @@ -0,0 +1,70 @@ +ALTER TABLE license +ADD COLUMN IF NOT EXISTS rights_identifier VARCHAR(255), +ADD COLUMN IF NOT EXISTS rights_identifier_scheme VARCHAR(255), +ADD COLUMN IF NOT EXISTS scheme_uri VARCHAR(255), +ADD COLUMN IF NOT EXISTS language_code VARCHAR(5); + +-- Update existing entries + +UPDATE license SET + rights_identifier = 'Apache-2.0', + rights_identifier_scheme = 'SPDX', + scheme_uri = 'https://spdx.org/licenses/', + language_code = 'en' +WHERE uri = 'https://www.apache.org/licenses/LICENSE-2.0'; + +UPDATE license SET + rights_identifier = 'CC0-1.0', + rights_identifier_scheme = 'SPDX', + scheme_uri = 'https://spdx.org/licenses/', + language_code = 'en' +WHERE uri = 'http://creativecommons.org/publicdomain/zero/1.0'; + +UPDATE license SET + rights_identifier = 'CC-BY-4.0', + rights_identifier_scheme = 'SPDX', + scheme_uri = 'https://spdx.org/licenses/', + language_code = 'en' +WHERE uri = 'https://creativecommons.org/licenses/by/4.0/'; + +UPDATE license SET + rights_identifier = 'CC-BY-NC-4.0', + rights_identifier_scheme = 'SPDX', + scheme_uri = 'https://spdx.org/licenses/', + language_code = 'en' +WHERE uri = 'https://creativecommons.org/licenses/by-nc/4.0/'; + +UPDATE license SET + rights_identifier = 'CC-BY-NC-ND-4.0', + rights_identifier_scheme = 'SPDX', + scheme_uri = 'https://spdx.org/licenses/', + language_code = 'en' +WHERE uri = 'https://creativecommons.org/licenses/by-nc-nd/4.0/'; + +UPDATE license SET + rights_identifier = 'CC-BY-NC-SA-4.0', + rights_identifier_scheme = 'SPDX', + scheme_uri = 'https://spdx.org/licenses/', + language_code = 'en' +WHERE uri = 'https://creativecommons.org/licenses/by-nc-sa/4.0/'; + +UPDATE license SET + rights_identifier = 'CC-BY-ND-4.0', + rights_identifier_scheme = 'SPDX', + scheme_uri = 'https://spdx.org/licenses/', + language_code = 'en' +WHERE uri = 'https://creativecommons.org/licenses/by-nd/4.0/'; + +UPDATE license SET + rights_identifier = 'CC-BY-SA-4.0', + rights_identifier_scheme = 'SPDX', + scheme_uri = 'https://spdx.org/licenses/', + language_code = 'en' +WHERE uri = 'https://creativecommons.org/licenses/by-sa/4.0/'; + +UPDATE license SET + rights_identifier = 'MIT', + rights_identifier_scheme = 'SPDX', + scheme_uri = 'https://spdx.org/licenses/', + language_code = 'en' +WHERE uri = 'https://opensource.org/licenses/MIT'; diff --git a/src/main/resources/db/migration/V6.5.0.6.sql b/src/main/resources/db/migration/V6.5.0.6.sql new file mode 100644 index 00000000000..9227ec2d83b --- /dev/null +++ b/src/main/resources/db/migration/V6.5.0.6.sql @@ -0,0 +1,3 @@ +--add column - should have been part of 10476/ PR #11224 + +ALTER TABLE DataverseFieldTypeInputLevel ADD COLUMN IF NOT EXISTS displayOnCreate bool; \ No newline at end of file diff --git a/src/main/resources/db/migration/V6.5.0.7.sql b/src/main/resources/db/migration/V6.5.0.7.sql new file mode 100644 index 00000000000..d7d4404fb66 --- /dev/null +++ b/src/main/resources/db/migration/V6.5.0.7.sql @@ -0,0 +1,2 @@ +-- Add this text will help to customized name in metadata source facet +ALTER TABLE harvestingclient ADD COLUMN IF NOT EXISTS sourcename TEXT; \ No newline at end of file diff --git a/src/main/resources/db/migration/V6.5.0.8.sql b/src/main/resources/db/migration/V6.5.0.8.sql new file mode 100644 index 00000000000..d7cb24855b4 --- /dev/null +++ b/src/main/resources/db/migration/V6.5.0.8.sql @@ -0,0 +1,3 @@ +-- Add a new boolean flag, and make the harvesting set field free text, in order to accommodate DataCite harvesting +ALTER TABLE harvestingclient ADD COLUMN IF NOT EXISTS useListRecords BOOLEAN DEFAULT FALSE; +ALTER TABLE harvestingclient ALTER COLUMN harvestingSet TYPE TEXT; diff --git a/src/main/resources/db/migration/V6.5.0.9.sql b/src/main/resources/db/migration/V6.5.0.9.sql new file mode 100644 index 00000000000..5d00cd88ade --- /dev/null +++ b/src/main/resources/db/migration/V6.5.0.9.sql @@ -0,0 +1,22 @@ +-- Add deaccessionnote column +-- + +ALTER TABLE datasetversion ADD COLUMN IF NOT EXISTS deaccessionnote VARCHAR(1000); +ALTER TABLE datasetversion ALTER COLUMN deaccessionlink TYPE varchar(1260); + +-- Move/merge archivenote contents and remove archivenote column (on existing DBs that have this column) +DO $$ +BEGIN +IF EXISTS (SELECT 1 FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_NAME = 'datasetversion' AND COLUMN_NAME = 'archivenote') THEN +UPDATE datasetversion set deaccessionlink = CONCAT_WS(' ', deaccessionlink, archivenote); +ALTER TABLE datasetversion DROP COLUMN archivenote; + +-- Update deaccessionnote for existing datasetversions +-- Only do this once - if archivenote hasn't been deleted is a convenient trigger + +UPDATE datasetversion set deaccessionnote = versionnote; +UPDATE datasetversion set versionnote = null; + +END IF; +END +$$ diff --git a/src/main/resources/db/migration/afterMigrate__1-7256-upsert-referenceData.sql b/src/main/resources/db/migration/afterMigrate__1-7256-upsert-referenceData.sql index 07e9b2c6266..f63fedba02f 100644 --- a/src/main/resources/db/migration/afterMigrate__1-7256-upsert-referenceData.sql +++ b/src/main/resources/db/migration/afterMigrate__1-7256-upsert-referenceData.sql @@ -31,7 +31,7 @@ INSERT INTO foreignmetadatafieldmapping (id, foreignfieldxpath, datasetfieldname (15, 'affiliation', 'authorAffiliation', TRUE, 3, 1 ), (16, ':contributor', 'contributorName', FALSE, NULL, 1 ), (17, 'type', 'contributorType', TRUE, 16, 1 ), - (18, ':publisher', 'producerName', FALSE, NULL, 1 ), + (18, ':publisher', 'distributorName', FALSE, NULL, 1 ), (19, ':language', 'language', FALSE, NULL, 1 ) ON CONFLICT DO NOTHING; diff --git a/src/main/webapp/ThemeAndWidgets.xhtml b/src/main/webapp/ThemeAndWidgets.xhtml index fd8d7c7cf12..cde8b720aef 100644 --- a/src/main/webapp/ThemeAndWidgets.xhtml +++ b/src/main/webapp/ThemeAndWidgets.xhtml @@ -15,12 +15,14 @@ - + + + diff --git a/src/main/webapp/confirmemail.xhtml b/src/main/webapp/confirmemail.xhtml index 2a071c83817..8eac2d8db00 100644 --- a/src/main/webapp/confirmemail.xhtml +++ b/src/main/webapp/confirmemail.xhtml @@ -14,12 +14,14 @@ - + + + diff --git a/src/main/webapp/dashboard-datamove.xhtml b/src/main/webapp/dashboard-movedataset.xhtml similarity index 79% rename from src/main/webapp/dashboard-datamove.xhtml rename to src/main/webapp/dashboard-movedataset.xhtml index 7f8365c9be3..d41a5ee74a8 100644 --- a/src/main/webapp/dashboard-datamove.xhtml +++ b/src/main/webapp/dashboard-movedataset.xhtml @@ -13,40 +13,40 @@ - + - - + + - + - +
    -
    #{bundle['dashboard.card.datamove.selectdataset.header']}
    +
    #{bundle['dashboard.move.dataset.selectdataset.header']}
    @@ -70,26 +70,26 @@
    -
    #{bundle['dashboard.card.datamove.newdataverse.header']}
    +
    #{bundle['dashboard.move.dataset.newdataverse.header']}
    @@ -113,12 +113,12 @@
    + oncomplete="if (args && !args.validationFailed) PF('moveDatasetConfirmation').show();"> - + +
    + +
    + + + + + + + + + + + + +
    +
    + +
    +
    +
    +
    +
    +
    #{bundle['dashboard.move.dataverse.newdataverse.header']}
    +
    + + +
    + +
    + + + + + + + + + + + + +
    +
    + +
    +
    +
    + +
    + + + + +
    + + +

    #{bundle['dashboard.move.dataverse.confirm.dialog']}

    +
    + + +
    +     + + + + + + diff --git a/src/main/webapp/dashboard-users.xhtml b/src/main/webapp/dashboard-users.xhtml index 11e26335234..9f2f60c113c 100644 --- a/src/main/webapp/dashboard-users.xhtml +++ b/src/main/webapp/dashboard-users.xhtml @@ -15,7 +15,7 @@ - + diff --git a/src/main/webapp/dashboard.xhtml b/src/main/webapp/dashboard.xhtml index 5a72b52937b..577d8c67f63 100644 --- a/src/main/webapp/dashboard.xhtml +++ b/src/main/webapp/dashboard.xhtml @@ -12,13 +12,15 @@ - + + +
    @@ -126,21 +128,26 @@
    -

    #{bundle['dashboard.card.datamove']}

    +

    #{bundle['dashboard.card.move.data']}

    - -

    #{bundle['dataverses']}

    + +

    #{bundle['datasets']}

    - -

    #{bundle['datasets']}

    + +

    #{bundle['dataverses']}

    -
    +
    +

    + #{bundle['dashboard.card.move.dataset.manage']} +

    +
    +

    - #{bundle['dashboard.card.datamove.manage']} + #{bundle['dashboard.card.move.dataverse.manage']}

    diff --git a/src/main/webapp/dataset-citation.xhtml b/src/main/webapp/dataset-citation.xhtml index 346e39dc463..26bc9dca65e 100644 --- a/src/main/webapp/dataset-citation.xhtml +++ b/src/main/webapp/dataset-citation.xhtml @@ -41,6 +41,11 @@
    • +
  • + +
    • diff --git a/src/main/webapp/dataset-versions.xhtml b/src/main/webapp/dataset-versions.xhtml index fe0758d74f0..1bbeff0fb86 100644 --- a/src/main/webapp/dataset-versions.xhtml +++ b/src/main/webapp/dataset-versions.xhtml @@ -77,28 +77,28 @@ - - - - - - - - - + + + + + + + + + - - + + - - - - - + + + + + @@ -120,7 +120,11 @@ #{bundle['file.dataFilesTab.versions.description.firstPublished']} - #{bundle['file.dataFilesTab.versions.description.deaccessionedReason']} #{versionTab.versionNote} #{bundle['file.dataFilesTab.versions.description.beAccessedAt']} #{versionTab.archiveNote} + #{bundle['file.dataFilesTab.versions.description.deaccessionedReason']} #{versionTab.deaccessionNote} + #{bundle['file.dataFilesTab.versions.description.beAccessedAt']} + #{versionTab.deaccessionLink} + + + + + + +
    +     + +     diff --git a/src/main/webapp/dataset-widgets.xhtml b/src/main/webapp/dataset-widgets.xhtml index f635099dfdb..ed551a1676f 100644 --- a/src/main/webapp/dataset-widgets.xhtml +++ b/src/main/webapp/dataset-widgets.xhtml @@ -15,13 +15,15 @@ - + + + diff --git a/src/main/webapp/dataset.xhtml b/src/main/webapp/dataset.xhtml index 9426884d349..4efa598d04b 100644 --- a/src/main/webapp/dataset.xhtml +++ b/src/main/webapp/dataset.xhtml @@ -96,6 +96,29 @@ + + + + + + + + + + + + + + + + + + + + + + + @@ -103,27 +126,9 @@ - - - - - - - - - - - - - - - - - - - - - + + + @@ -627,23 +632,19 @@
    #{bundle['dataset.deaccession.reason']}
    -

    #{DatasetPage.workingVersion.versionNote}

    - -

    #{bundle['dataset.beAccessedAt']} #{DatasetPage.workingVersion.archiveNote}

    +

    #{DatasetPage.workingVersion.deaccessionNote}

    + +

    #{bundle['dataset.beAccessedAt']} + #{DatasetPage.workingVersion.deaccessionLink} + +

    -
    +
    @@ -858,6 +859,11 @@ $(this).ready(function () { refreshIfStillLocked(); }); + + var initialInterval = 5000; // 5 seconds + var maxInterval = 300000; // 5 minutes + var currentInterval = initialInterval; + var backoffFactor = 1.2; // Exponential factor function refreshIfStillLocked() { if ($('input[id$="datasetLockedForAnyReasonVariable"]').val() === 'true') { // if dataset is locked, instruct the page to @@ -882,18 +888,22 @@ $('button[id$="refreshButton"]').trigger('click'); //refreshAllCommand(); }, 1500); + } else { + // Reset the interval if the dataset is unlocked + currentInterval = initialInterval; } } } + function waitAndCheckLockAgain() { setTimeout(function () { // refresh the lock in the // backing bean; i.e., check, if the ingest has // already completed in the background: - //$('button[id$="refreshButton"]').trigger('click'); - //refreshLockCommand(); refreshAllLocksCommand(); - }, 10000); + // Increase the interval exponentially for the next check + currentInterval = Math.min((currentInterval * backoffFactor) + 2, maxInterval); + }, currentInterval); } //]]> @@ -981,6 +991,7 @@ + @@ -1177,14 +1188,14 @@

    -

    #{bundle['dataset.privateurl.introduction']}

    +

    #{bundle['dataset.privateurl.introduction']} + #{bundle['dataset.privateurl.onlyone']}

    -

    -

    #{bundle['dataset.privateurl.onlyone']}

    +

    @@ -1221,10 +1232,18 @@

    -

    + + +

    #{bundle['dataset.privateurl.anonymous.description']}

    -

    #{bundle['dataset.privateurl.anonymous.description.paragraph.two']} #{bundle['dataset.privateurl.createPrivateUrl.anonymized.unavailable']}.

    - +

    #{bundle['dataset.privateurl.anonymous.description.paragraph.two']}

    + + + + +

    #{bundle['dataset.privateurl.createPrivateUrl.anonymized.unavailable']}

    + + +
    + + +
    +

    #{bundle['dataset.publish.terms.help.tip']}

    @@ -2011,7 +2040,6 @@
    -

    #{bundle['dataset.rejectMessage']} #{disableReasonField ? '':bundle['dataset.rejectMessageReason']} @@ -2041,6 +2069,84 @@ + + +

    + + + + +

    +
    + + +
    +
    + + +
    + +     + + + + + + + + + + + + + + + + + + +
    + +
    +     +     @@ -2118,6 +2224,9 @@ function updateTemplate() { $('button[id$="updateTemplate"]').trigger('click'); } + function updateCSLCitation() { + $('button[id$="updateCSL"]').trigger('click'); + } function updateOwnerDataverse() { $('button[id$="updateOwnerDataverse"]').trigger('click'); } diff --git a/src/main/webapp/dataverse.xhtml b/src/main/webapp/dataverse.xhtml index d452f76c9cd..4ffac1c24b3 100644 --- a/src/main/webapp/dataverse.xhtml +++ b/src/main/webapp/dataverse.xhtml @@ -18,8 +18,7 @@ - - + @@ -49,6 +48,9 @@ + + + diff --git a/src/main/webapp/dataverse_homepage.xhtml b/src/main/webapp/dataverse_homepage.xhtml index bbf03ea22ad..f12fb40ae68 100644 --- a/src/main/webapp/dataverse_homepage.xhtml +++ b/src/main/webapp/dataverse_homepage.xhtml @@ -14,12 +14,14 @@ - + + + diff --git a/src/main/webapp/dataverse_template.xhtml b/src/main/webapp/dataverse_template.xhtml index a1f22a97642..719a59674a9 100644 --- a/src/main/webapp/dataverse_template.xhtml +++ b/src/main/webapp/dataverse_template.xhtml @@ -11,8 +11,9 @@ lang="#{dataverseSession.localeCode}" xml:lang="#{dataverseSession.localeCode}"> + - + <h:outputText value="#{pageTitle}"/> diff --git a/src/main/webapp/dataverseuser.xhtml b/src/main/webapp/dataverseuser.xhtml index bcb71733716..3cabca45a26 100644 --- a/src/main/webapp/dataverseuser.xhtml +++ b/src/main/webapp/dataverseuser.xhtml @@ -17,6 +17,17 @@ + + + + + + + + + + + -     + + + + + +
    - #{DataverseUserPage.userAuthProvider.persistentIdName} + #{bundle['user.orcid']} -   - - + +   + +
    + #{bundle['user.orcid']} + + + + +
    @@ -795,21 +819,6 @@ -
    - - - - -
    -

    -   - - - -

    -
    -
    diff --git a/src/main/webapp/editFilesFragment.xhtml b/src/main/webapp/editFilesFragment.xhtml index 6fab335c0f3..aec11eb1e54 100644 --- a/src/main/webapp/editFilesFragment.xhtml +++ b/src/main/webapp/editFilesFragment.xhtml @@ -9,6 +9,7 @@ xmlns:pt="http://java.sun.com/jsf/passthrough" xmlns:cc="http://java.sun.com/jsf/composite" xmlns:o="http://omnifaces.org/ui" + xmlns:fn="http://java.sun.com/jsp/jstl/functions" xmlns:iqbs="http://xmlns.jcp.org/jsf/composite/iqbs"> @@ -104,7 +105,7 @@ - +

    diff --git a/src/main/webapp/editdatafiles.xhtml b/src/main/webapp/editdatafiles.xhtml index be78359e02b..b2192089bf3 100644 --- a/src/main/webapp/editdatafiles.xhtml +++ b/src/main/webapp/editdatafiles.xhtml @@ -21,7 +21,7 @@ - + @@ -35,6 +35,8 @@ + +
    diff --git a/src/main/webapp/file-versions.xhtml b/src/main/webapp/file-versions.xhtml index f7f259ce2e0..fd457e137c2 100644 --- a/src/main/webapp/file-versions.xhtml +++ b/src/main/webapp/file-versions.xhtml @@ -126,7 +126,7 @@ + value="#{bundle['file.dataFilesTab.versions.description.deaccessionedReason']} #{versionTab.datasetVersion.deaccessionNote}" escape="false"/> diff --git a/src/main/webapp/file.xhtml b/src/main/webapp/file.xhtml index 835764d9cf5..b72c7dd4cb5 100644 --- a/src/main/webapp/file.xhtml +++ b/src/main/webapp/file.xhtml @@ -25,7 +25,7 @@ - + @@ -36,6 +36,8 @@ + +
    @@ -326,9 +328,13 @@
    #{bundle['dataset.deaccession.reason']}
    -

    #{FilePage.fileMetadata.datasetVersion.versionNote}

    - -

    #{bundle['dataset.beAccessedAt']} #{FilePage.fileMetadata.datasetVersion.archiveNote}

    +

    #{FilePage.fileMetadata.datasetVersion.deaccessionNote}

    + +

    #{bundle['dataset.beAccessedAt']} + #{FilePage.fileMetadata.datasetVersion.deaccessionLink} + + +

    diff --git a/src/main/webapp/filesFragment.xhtml b/src/main/webapp/filesFragment.xhtml index 154700f7cf4..ef6157e1e48 100644 --- a/src/main/webapp/filesFragment.xhtml +++ b/src/main/webapp/filesFragment.xhtml @@ -366,7 +366,7 @@

    -
    @@ -544,15 +544,16 @@ - - + #{bundle.preview} "#{empty(fileMetadata.directoryLabel) ? "":fileMetadata.directoryLabel.concat("/")}#{fileMetadata.label}" - - + #{bundle.preview} "#{empty(fileMetadata.directoryLabel) ? "":fileMetadata.directoryLabel.concat("/")}#{fileMetadata.label}" @@ -573,7 +574,7 @@ - + @@ -628,7 +629,7 @@ - + diff --git a/src/main/webapp/guestbook-terms-popup-fragment.xhtml b/src/main/webapp/guestbook-terms-popup-fragment.xhtml index 6eec4f100b1..7d157fcb72c 100644 --- a/src/main/webapp/guestbook-terms-popup-fragment.xhtml +++ b/src/main/webapp/guestbook-terms-popup-fragment.xhtml @@ -41,7 +41,7 @@ src="#{DatasetUtil:getLicenseIcon(workingVersion)}" title="#{DatasetUtil:getLicenseDescription(workingVersion)}" style="display: none" onload="this.style.display='inline'" /> - #{DatasetUtil:getLicenseName(workingVersion)} @@ -60,7 +60,7 @@
    + jsf:rendered="#{!empty workingVersion.termsOfUseAndAccess.termsOfAccess and (hasRestrictedFile or !empty fileDownloadHelper.filesForRequestAccess)}"> @@ -142,7 +142,7 @@ value="#{MarkupChecker:sanitizeBasicHTML(workingVersion.termsOfUseAndAccess.termsOfAccess)}" escape="false" />
    - +
    @@ -179,7 +179,7 @@ styleClass="form-control" value="#{guestbookResponse.email}" requiredMessage="#{bundle['requiredField']}" > - +
    diff --git a/src/main/webapp/guestbook.xhtml b/src/main/webapp/guestbook.xhtml index 3528ff5f032..04bf30614fe 100644 --- a/src/main/webapp/guestbook.xhtml +++ b/src/main/webapp/guestbook.xhtml @@ -15,7 +15,7 @@ - + @@ -24,7 +24,8 @@ - + + diff --git a/src/main/webapp/harvestclients.xhtml b/src/main/webapp/harvestclients.xhtml index 3c09ed4ecb0..4b5736566c2 100644 --- a/src/main/webapp/harvestclients.xhtml +++ b/src/main/webapp/harvestclients.xhtml @@ -15,7 +15,7 @@ - + @@ -203,6 +203,20 @@

    #{bundle['harvestclients.newClientDialog.nickname.helptext']}

    + +
    + +
    + + +

    #{bundle['harvestclients.newClientDialog.sourcename.helptext']}

    +
    +
    diff --git a/src/main/webapp/harvestsets.xhtml b/src/main/webapp/harvestsets.xhtml index 2ddcac76575..a9f4ab2fd71 100644 --- a/src/main/webapp/harvestsets.xhtml +++ b/src/main/webapp/harvestsets.xhtml @@ -15,7 +15,7 @@ - + diff --git a/src/main/webapp/loginpage.xhtml b/src/main/webapp/loginpage.xhtml index e1b77e9583b..d2b0ec5e434 100644 --- a/src/main/webapp/loginpage.xhtml +++ b/src/main/webapp/loginpage.xhtml @@ -185,9 +185,10 @@
    - - - + + + +

    diff --git a/src/main/webapp/metadataFragment.xhtml b/src/main/webapp/metadataFragment.xhtml index f8367ce01f8..f2560b70010 100755 --- a/src/main/webapp/metadataFragment.xhtml +++ b/src/main/webapp/metadataFragment.xhtml @@ -244,9 +244,7 @@

    + jsf:rendered="#{((editMode == 'METADATA' or dsf.datasetFieldType.shouldDisplayOnCreate() or !dsf.isEmpty() or dsf.required) and dsf.include) or (!datasetPage and dsf.include)}"> diff --git a/src/main/webapp/mydata_page.xhtml b/src/main/webapp/mydata_page.xhtml index b8c70f16079..084a482f0f7 100644 --- a/src/main/webapp/mydata_page.xhtml +++ b/src/main/webapp/mydata_page.xhtml @@ -15,11 +15,13 @@ - + + + diff --git a/src/main/webapp/oauth2/firstLogin.xhtml b/src/main/webapp/oauth2/firstLogin.xhtml index 6310a0b49da..07b1631d6cb 100644 --- a/src/main/webapp/oauth2/firstLogin.xhtml +++ b/src/main/webapp/oauth2/firstLogin.xhtml @@ -5,6 +5,7 @@ xmlns:ui="http://java.sun.com/jsf/facelets" xmlns:p="http://primefaces.org/ui" xmlns:jsf="http://xmlns.jcp.org/jsf" + xmlns:o="http://omnifaces.org/ui" xmlns:of="http://omnifaces.org/functions"> @@ -95,19 +96,19 @@
    -
    -
    + -
    +

    -   - - + +   + + -

    diff --git a/src/main/webapp/oauth2/orcidConfirm.xhtml b/src/main/webapp/oauth2/orcidConfirm.xhtml new file mode 100644 index 00000000000..6138f14ea29 --- /dev/null +++ b/src/main/webapp/oauth2/orcidConfirm.xhtml @@ -0,0 +1,72 @@ + + + + + + + + + + + + + + + + + + +

    + Http Return Code: + #{OAuth2Page.error.httpReturnCode} +

    + +

    Raw message body:

    + + 
    +                    
+                    #{OAuth2Page.error.messageBody}
+                
+
    +     + +     +     +     + diff --git a/src/main/webapp/passwordreset.xhtml b/src/main/webapp/passwordreset.xhtml index 4ffbdf21e08..e475f870394 100644 --- a/src/main/webapp/passwordreset.xhtml +++ b/src/main/webapp/passwordreset.xhtml @@ -14,13 +14,15 @@ - + + - + +
    diff --git a/src/main/webapp/previewurl.xhtml b/src/main/webapp/previewurl.xhtml index 980d775506b..c9e2ffeedfa 100644 --- a/src/main/webapp/previewurl.xhtml +++ b/src/main/webapp/previewurl.xhtml @@ -12,7 +12,7 @@ - + diff --git a/src/main/webapp/privateurl.xhtml b/src/main/webapp/privateurl.xhtml index 5ca90112a84..a1bac00534e 100644 --- a/src/main/webapp/privateurl.xhtml +++ b/src/main/webapp/privateurl.xhtml @@ -12,7 +12,7 @@ - + diff --git a/src/main/webapp/resources/css/structure.css b/src/main/webapp/resources/css/structure.css index b68acaf7488..e6634ff0684 100644 --- a/src/main/webapp/resources/css/structure.css +++ b/src/main/webapp/resources/css/structure.css @@ -600,10 +600,15 @@ div[id$='roleDisplay'] span.label, div[id$='roleDetails'] span.label {display:in #title {font-size:36px !important;line-height:1.1;font-weight:bold;margin-top:0px;margin-bottom:0px;} #author, #contact {display:block; margin-bottom:.5em;} +/* #11053 */ +#publication span { + margin: 0px 5px 0px 5px; +} + /* RESPONSIVE ORDER HACK BOOTSTRAP 3 https://stackoverflow.com/a/24834574 */ #dataset-colorder-block {height: 50px;} @media(min-width:992px){ - #dataset-summary-metadata, #deaccession-reason-block {margin-top: -50px;} + #dataset-colorder-block + div {margin-top: -50px;} } .bg-citation {background:#ECF6FB; border: 0;} @@ -671,15 +676,20 @@ div.edit-field div.ui-message {margin:6px 0;} white-space: pre-wrap !important; } -#metadata_author img { + +img[alt='ORCID logo'], img[alt='ROR logo'] { vertical-align: text-top; margin-right:1px; margin-left: 3px; } -#metadata_author img.ror { +img.ror { vertical-align: middle; } +.expanded.parenthetical[data-cvoc-protocol='ror'] span { + margin: 0px 5px 0px 5px; +} + /* TRUNCATION */ div.more-block {text-align:center; padding-top:250px; width:100%; position:absolute; bottom:0; background:linear-gradient(180deg,hsla(0,0%,100%,0),#fff 80%);} button.desc-more-link {margin:0; padding:0;} @@ -908,7 +918,16 @@ div.dvnDifferanceTable > div.ui-datatable-tablewrapper > table > tbody > tr, div div.dvnDifferanceTable > div.ui-datatable-tablewrapper > table > tbody > tr, div.dvnDifferanceTable > div.ui-datatable-tablewrapper > table > tbody > tr > td:last-child {border-right-width:0;} div.dvnDifferanceTable div.ui-datatable-tablewrapper table td.versionValue {width:30%;} div.dvnDifferanceTable div.ui-datatable-tablewrapper table td.versionDetails {width:35%;} -div.dvnDifferanceTable .diffDetailBlock {display:block;} +div.dvnDifferanceTable .diffDetailBlock { + display:block; + word-break:break-all; +} + +div.dvnDifferanceTable .versionValue { + word-break:break-all; +} +div[id$="versionsTable"] tbody {word-break:break-word;} + /* DATATABLE + DROPDOWN BUTTON + OVERFLOW VISIBLE */ thead.ui-datatable-scrollable-theadclone {display:none} @@ -1206,3 +1225,40 @@ span.label-default { background-color: #757575 } min-width: 130px; padding: 0px; } + + +/*CSL Citations*/ +#datasetForm .csl-entry { + border-style: outset; + border-radius: 5px; + border-color: #757575; + margin: 10px 0px 10px 0px; + padding: 10px; + display: inline; + } + +#datasetForm\:cslLabel { + margin-top:30px; + display:block; +} + +#datasetForm\:cslOutput, .csl-bib-body, .csl-right-inline { + display:inline; +} + +#datasetForm\:cslCitation .glyphicon-copy{ + font-size: x-large; + display: inline; + vertical-align: bottom; + margin-left: 5px; +} + +/* Suppress the number in some styles indication this references ordinal position in a larger bibliography (which is always 1 in our dialog)*/ +.csl-left-margin { + display:none; +} + +/*Align menu items*/ +.citation-download .btn { + margin-left: 8px; +} \ No newline at end of file diff --git a/src/main/webapp/resources/js/clipboard.min.js b/src/main/webapp/resources/js/clipboard.min.js index 98d4ccba698..1103f811ed8 100644 --- a/src/main/webapp/resources/js/clipboard.min.js +++ b/src/main/webapp/resources/js/clipboard.min.js @@ -1,7 +1,7 @@ /*! - * clipboard.js v2.0.6 + * clipboard.js v2.0.11 * https://clipboardjs.com/ * * Licensed MIT ยฉ Zeno Rocha */ -!function(t,e){"object"==typeof exports&&"object"==typeof module?module.exports=e():"function"==typeof define&&define.amd?define([],e):"object"==typeof exports?exports.ClipboardJS=e():t.ClipboardJS=e()}(this,function(){return n={134:(t,e,n)=>{"use strict";n.d(e,{default:()=>r});var e=n(817),o=n.n(e);function i(t){return(i="function"==typeof Symbol&&"symbol"==typeof Symbol.iterator?function(t){return typeof t}:function(t){return t&&"function"==typeof Symbol&&t.constructor===Symbol&&t!==Symbol.prototype?"symbol":typeof t})(t)}function a(t,e){for(var n=0;n{var e;"undefined"==typeof Element||Element.prototype.matches||((e=Element.prototype).matches=e.matchesSelector||e.mozMatchesSelector||e.msMatchesSelector||e.oMatchesSelector||e.webkitMatchesSelector),t.exports=function(t,e){for(;t&&9!==t.nodeType;){if("function"==typeof t.matches&&t.matches(e))return t;t=t.parentNode}}},438:(t,e,n)=>{var a=n(828);function i(t,e,n,r,o){var i=function(e,n,t,r){return function(t){t.delegateTarget=a(t.target,n),t.delegateTarget&&r.call(e,t)}}.apply(this,arguments);return t.addEventListener(n,i,o),{destroy:function(){t.removeEventListener(n,i,o)}}}t.exports=function(t,e,n,r,o){return"function"==typeof t.addEventListener?i.apply(null,arguments):"function"==typeof n?i.bind(null,document).apply(null,arguments):("string"==typeof t&&(t=document.querySelectorAll(t)),Array.prototype.map.call(t,function(t){return i(t,e,n,r,o)}))}},879:(t,n)=>{n.node=function(t){return void 0!==t&&t instanceof HTMLElement&&1===t.nodeType},n.nodeList=function(t){var e=Object.prototype.toString.call(t);return void 0!==t&&("[object NodeList]"===e||"[object HTMLCollection]"===e)&&"length"in t&&(0===t.length||n.node(t[0]))},n.string=function(t){return"string"==typeof t||t instanceof String},n.fn=function(t){return"[object Function]"===Object.prototype.toString.call(t)}},370:(t,e,n)=>{var u=n(879),s=n(438);t.exports=function(t,e,n){if(!t&&!e&&!n)throw new Error("Missing required arguments");if(!u.string(e))throw new TypeError("Second argument must be a String");if(!u.fn(n))throw new TypeError("Third argument must be a Function");if(u.node(t))return c=e,l=n,(a=t).addEventListener(c,l),{destroy:function(){a.removeEventListener(c,l)}};if(u.nodeList(t))return r=t,o=e,i=n,Array.prototype.forEach.call(r,function(t){t.addEventListener(o,i)}),{destroy:function(){Array.prototype.forEach.call(r,function(t){t.removeEventListener(o,i)})}};if(u.string(t))return t=t,e=e,n=n,s(document.body,t,e,n);throw new TypeError("First argument must be a String, HTMLElement, HTMLCollection, or NodeList");var r,o,i,a,c,l}},817:t=>{t.exports=function(t){var e,n="SELECT"===t.nodeName?(t.focus(),t.value):"INPUT"===t.nodeName||"TEXTAREA"===t.nodeName?((e=t.hasAttribute("readonly"))||t.setAttribute("readonly",""),t.select(),t.setSelectionRange(0,t.value.length),e||t.removeAttribute("readonly"),t.value):(t.hasAttribute("contenteditable")&&t.focus(),n=window.getSelection(),(e=document.createRange()).selectNodeContents(t),n.removeAllRanges(),n.addRange(e),n.toString());return n}},279:t=>{function e(){}e.prototype={on:function(t,e,n){var r=this.e||(this.e={});return(r[t]||(r[t]=[])).push({fn:e,ctx:n}),this},once:function(t,e,n){var r=this;function o(){r.off(t,o),e.apply(n,arguments)}return o._=e,this.on(t,o,n)},emit:function(t){for(var e=[].slice.call(arguments,1),n=((this.e||(this.e={}))[t]||[]).slice(),r=0,o=n.length;r{var e=t&&t.__esModule?()=>t.default:()=>t;return r.d(e,{a:e}),e},r.d=(t,e)=>{for(var n in e)r.o(e,n)&&!r.o(t,n)&&Object.defineProperty(t,n,{enumerable:!0,get:e[n]})},r.o=(t,e)=>Object.prototype.hasOwnProperty.call(t,e),r(134).default;function r(t){if(o[t])return o[t].exports;var e=o[t]={exports:{}};return n[t](e,e.exports,r),e.exports}var n,o}); \ No newline at end of file +!function(t,e){"object"==typeof exports&&"object"==typeof module?module.exports=e():"function"==typeof define&&define.amd?define([],e):"object"==typeof exports?exports.ClipboardJS=e():t.ClipboardJS=e()}(this,function(){return n={686:function(t,e,n){"use strict";n.d(e,{default:function(){return b}});var e=n(279),i=n.n(e),e=n(370),u=n.n(e),e=n(817),r=n.n(e);function c(t){try{return document.execCommand(t)}catch(t){return}}var a=function(t){t=r()(t);return c("cut"),t};function o(t,e){var n,o,t=(n=t,o="rtl"===document.documentElement.getAttribute("dir"),(t=document.createElement("textarea")).style.fontSize="12pt",t.style.border="0",t.style.padding="0",t.style.margin="0",t.style.position="absolute",t.style[o?"right":"left"]="-9999px",o=window.pageYOffset||document.documentElement.scrollTop,t.style.top="".concat(o,"px"),t.setAttribute("readonly",""),t.value=n,t);return e.container.appendChild(t),e=r()(t),c("copy"),t.remove(),e}var f=function(t){var e=1 - + @@ -24,6 +24,8 @@ + +

    Related roles for each DvObject type

    diff --git a/src/main/webapp/search-include-fragment.xhtml b/src/main/webapp/search-include-fragment.xhtml index fc224443a8e..b20c777140d 100644 --- a/src/main/webapp/search-include-fragment.xhtml +++ b/src/main/webapp/search-include-fragment.xhtml @@ -494,7 +494,7 @@
    - + diff --git a/src/main/webapp/search/advanced.xhtml b/src/main/webapp/search/advanced.xhtml index 24b78ba0dc9..c921fe930d5 100644 --- a/src/main/webapp/search/advanced.xhtml +++ b/src/main/webapp/search/advanced.xhtml @@ -15,13 +15,15 @@ - + + + diff --git a/src/main/webapp/superuser.xhtml b/src/main/webapp/superuser.xhtml index bc7426948b7..3c4212dcf92 100644 --- a/src/main/webapp/superuser.xhtml +++ b/src/main/webapp/superuser.xhtml @@ -13,11 +13,13 @@ - + + +