From 39967261b14eca5360692c670aa4f15d12af0f4a Mon Sep 17 00:00:00 2001 From: qqmye Date: Wed, 9 Oct 2019 11:36:02 -0400 Subject: [PATCH 1/8] #6137 make DataCite API URL for MDC configurable --- .../edu/harvard/iq/dataverse/api/MakeDataCountApi.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/MakeDataCountApi.java b/src/main/java/edu/harvard/iq/dataverse/api/MakeDataCountApi.java index 61c5b3039d5..02942cf60e3 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/MakeDataCountApi.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/MakeDataCountApi.java @@ -139,7 +139,12 @@ public Response updateCitationsForDataset(@PathParam("id") String id) throws Mal // DataCite wants "doi=", not "doi:". String authorityPlusIdentifier = persistentId.replaceFirst("doi:", ""); // curl https://api.datacite.org/events?doi=10.7910/dvn/hqzoob&source=crossref - URL url = new URL("https://api.datacite.org/events?doi=" + authorityPlusIdentifier + "&source=crossref"); + String baseUrl = System.getProperty("doi.mdcurlstring"); + if(null==baseUrl) { + //Backward compatible default to the production server + baseUrl="https://api.datacite.org"; + } + URL url = new URL(baseUrl + "/events?doi=" + authorityPlusIdentifier + "&source=crossref"); HttpURLConnection connection = (HttpURLConnection) url.openConnection(); connection.setRequestMethod("GET"); int status = connection.getResponseCode(); From 12799f788ae2d0458125a98b412aab34a976d151 Mon Sep 17 00:00:00 2001 From: qqmye Date: Wed, 9 Oct 2019 11:36:18 -0400 Subject: [PATCH 2/8] doc/install updates for #6137 --- conf/docker-aio/configure_doi.bash | 5 ++++ .../source/installation/config.rst | 23 +++++++++++++++++++ scripts/installer/glassfish-setup.sh | 5 ++++ scripts/installer/install | 6 ++++- 4 files changed, 38 insertions(+), 1 deletion(-) diff --git a/conf/docker-aio/configure_doi.bash b/conf/docker-aio/configure_doi.bash index 24ed6005b95..3aa29605037 100755 --- a/conf/docker-aio/configure_doi.bash +++ b/conf/docker-aio/configure_doi.bash @@ -17,3 +17,8 @@ if [ ! -z "${doi_baseurl}" ]; then doi_baseurl_esc=`echo ${doi_baseurl} | sed -e 's/:/\\:/'` bin/asadmin create-jvm-options "\"-Ddoi.baseurlstring=${doi_baseurl_esc}\"" fi +if [ ! -z "${doi_mdcbaseurl}" ]; then + bin/asadmin delete-jvm-options "-Ddoi.mdcbaseurlstring=https\://api.test.datacite.org" + doi_mdcbaseurl_esc=`echo ${doi_mdcbaseurl} | sed -e 's/:/\\:/'` + bin/asadmin create-jvm-options "\"-Ddoi.mdcbaseurlstring=${doi_mdcbaseurl_esc}\"" +fi diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index b1e90e7a1c8..9bbb7939f35 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -142,6 +142,7 @@ Here are the configuration options for DOIs: - :ref:`doi.baseurlstring` - :ref:`doi.username` - :ref:`doi.password` +- :ref:`doi.mdcbaseurlstring` **Database Settings:** @@ -956,6 +957,28 @@ See also these related database settings below: - :ref:`:Authority` - :ref:`:Shoulder` +.. _doi.baseurlstring: + +doi.mdcbaseurlstring ++++++++++++++++++ + +As of this writing, "https://api.datacite.org" (DataCite) and "https://api.test.datacite.org" (DataCite Testing) are the main valid values. + +Out of the box, Dataverse is configured to use a test DataCite MDC base URL string. You can delete it like this: + +``./asadmin delete-jvm-options '-Ddoi.mdcbaseurlstring=https\://api.test.datacite.org'`` + +Then, to switch to production DataCite, you can issue the following command: + +``./asadmin create-jvm-options '-Ddoi.mdcbaseurlstring=https\://api.datacite.org'`` + +For backward compatibility, if this option is not defined, the default used is "https\://api.datacite.org:. + +See also these related database settings below: + +- :ref:`:MDCLogPath` +- :ref:`:DisplayMDCMetrics` + .. _doi.username: doi.username diff --git a/scripts/installer/glassfish-setup.sh b/scripts/installer/glassfish-setup.sh index dfd77d91eaf..cde12cdedfc 100755 --- a/scripts/installer/glassfish-setup.sh +++ b/scripts/installer/glassfish-setup.sh @@ -28,6 +28,7 @@ # DOI_USERNAME # DOI_PASSWORD # DOI_BASEURL +# DOI_MDCBASEURL # # other local configuration: # HOST_ADDRESS @@ -88,6 +89,10 @@ function preliminary_setup() ./asadmin $ASADMIN_OPTS create-jvm-options '\-Ddoi.password=${ALIAS=doi_password_alias}' ./asadmin $ASADMIN_OPTS create-jvm-options "\-Ddoi.baseurlstring=$DOI_BASEURL_ESC" + # jvm-options use colons as separators, escape as literal + DOI_MDCBASEURL_ESC=`echo $DOI_MDCBASEURL | sed -e 's/:/\\\:/'` + ./asadmin $ASADMIN_OPTS create-jvm-options "\-Ddoi.basemdcurlstring=$DOI_MDCBASEURL_ESC" + ./asadmin $ASADMIN_OPTS create-jvm-options "-Ddataverse.timerServer=true" # enable comet support ./asadmin $ASADMIN_OPTS set server-config.network-config.protocols.protocol.http-listener-1.http.comet-support-enabled="true" diff --git a/scripts/installer/install b/scripts/installer/install index ef535f9c292..d40bb2ecd17 100755 --- a/scripts/installer/install +++ b/scripts/installer/install @@ -84,7 +84,8 @@ else 'DOI_USERNAME', 'DOI_PASSWORD', - 'DOI_BASEURL' + 'DOI_BASEURL', + 'DOI_MDCBASEURL' ); } @@ -114,6 +115,7 @@ my %CONFIG_DEFAULTS = ( 'DOI_USERNAME', 'dataciteuser', 'DOI_PASSWORD', 'datacitepassword', 'DOI_BASEURL', 'https://mds.test.datacite.org', + 'DOI_MDCBASEURL', 'https://api.test.datacite.org', ); @@ -141,6 +143,7 @@ my %CONFIG_PROMPTS = ( 'DOI_USERNAME', 'Datacite username', 'DOI_PASSWORD', 'Datacite password', 'DOI_BASEURL', 'Datacite URL' + 'DOI_MDCBASEURL', 'Datacite MakeDataCount URL' ); @@ -169,6 +172,7 @@ my %CONFIG_COMMENTS = ( 'DOI_USERNAME', 'DataCite or EZID username. Only necessary for publishing / minting DOIs.', 'DOI_PASSWORD', 'DataCite or EZID account password.', 'DOI_BASEURL', 'DataCite or EZID URL. Probably https://mds.datacite.org' + 'DOI_MDCBASEURL', 'DataCite Make Data Count API base URL. Probably https://api.datacite.org' ); From 346df71330f96146c4b9f71cb83ee954c3cd8e60 Mon Sep 17 00:00:00 2001 From: qqmye Date: Wed, 9 Oct 2019 20:24:35 -0400 Subject: [PATCH 3/8] cut/paste issue --- doc/sphinx-guides/source/installation/config.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index 9bbb7939f35..0466b72706f 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -957,10 +957,10 @@ See also these related database settings below: - :ref:`:Authority` - :ref:`:Shoulder` -.. _doi.baseurlstring: +.. _doi.mdcbaseurlstring: doi.mdcbaseurlstring -+++++++++++++++++ +++++++++++++++++++++ As of this writing, "https://api.datacite.org" (DataCite) and "https://api.test.datacite.org" (DataCite Testing) are the main valid values. From 40502201f42706f82a41ca1cdc0e7fe2a253bcd6 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Mon, 21 Oct 2019 17:55:54 -0400 Subject: [PATCH 4/8] two commas --- scripts/installer/install | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/installer/install b/scripts/installer/install index d40bb2ecd17..2f4da754cfc 100755 --- a/scripts/installer/install +++ b/scripts/installer/install @@ -142,7 +142,7 @@ my %CONFIG_PROMPTS = ( 'DOI_USERNAME', 'Datacite username', 'DOI_PASSWORD', 'Datacite password', - 'DOI_BASEURL', 'Datacite URL' + 'DOI_BASEURL', 'Datacite URL', 'DOI_MDCBASEURL', 'Datacite MakeDataCount URL' ); @@ -171,7 +171,7 @@ my %CONFIG_COMMENTS = ( 'DOI_USERNAME', 'DataCite or EZID username. Only necessary for publishing / minting DOIs.', 'DOI_PASSWORD', 'DataCite or EZID account password.', - 'DOI_BASEURL', 'DataCite or EZID URL. Probably https://mds.datacite.org' + 'DOI_BASEURL', 'DataCite or EZID URL. Probably https://mds.datacite.org', 'DOI_MDCBASEURL', 'DataCite Make Data Count API base URL. Probably https://api.datacite.org' ); From 09a5ff6e6e6eee1d320136642260ed62f46efb2e Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Mon, 21 Oct 2019 18:24:02 -0400 Subject: [PATCH 5/8] add note in admin guide --- doc/sphinx-guides/source/admin/make-data-count.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/sphinx-guides/source/admin/make-data-count.rst b/doc/sphinx-guides/source/admin/make-data-count.rst index deb0679b32d..d8fc51db002 100644 --- a/doc/sphinx-guides/source/admin/make-data-count.rst +++ b/doc/sphinx-guides/source/admin/make-data-count.rst @@ -63,6 +63,8 @@ Enable Logging for Make Data Count To make Dataverse log dataset usage (views and downloads) for Make Data Count, you must set the ``:MDCLogPath`` database setting. See :ref:`MDCLogPath` for details. +If you wish to start logging in advance of setting up other components, or wish to log without display MDC metrics for any other reason, you can set the optional ``:DisplayMDCMetrics`` database setting to false. See :ref:`DisplayMDCMetrics` for details. + After you have your first day of logs, you can process them the next day. Enable or Disable Display of Make Data Count Metrics From 2cf6ecb28074c6a2c79842b938d99dba5f1e383b Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Tue, 22 Oct 2019 14:56:44 -0400 Subject: [PATCH 6/8] consistent naming for jvm option --- scripts/installer/glassfish-setup.sh | 2 +- .../java/edu/harvard/iq/dataverse/api/MakeDataCountApi.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/installer/glassfish-setup.sh b/scripts/installer/glassfish-setup.sh index cde12cdedfc..7ab2473d622 100755 --- a/scripts/installer/glassfish-setup.sh +++ b/scripts/installer/glassfish-setup.sh @@ -91,7 +91,7 @@ function preliminary_setup() # jvm-options use colons as separators, escape as literal DOI_MDCBASEURL_ESC=`echo $DOI_MDCBASEURL | sed -e 's/:/\\\:/'` - ./asadmin $ASADMIN_OPTS create-jvm-options "\-Ddoi.basemdcurlstring=$DOI_MDCBASEURL_ESC" + ./asadmin $ASADMIN_OPTS create-jvm-options "\-Ddoi.mdcbaseurlstring=$DOI_MDCBASEURL_ESC" ./asadmin $ASADMIN_OPTS create-jvm-options "-Ddataverse.timerServer=true" # enable comet support diff --git a/src/main/java/edu/harvard/iq/dataverse/api/MakeDataCountApi.java b/src/main/java/edu/harvard/iq/dataverse/api/MakeDataCountApi.java index 02942cf60e3..5722acc11ec 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/MakeDataCountApi.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/MakeDataCountApi.java @@ -139,7 +139,7 @@ public Response updateCitationsForDataset(@PathParam("id") String id) throws Mal // DataCite wants "doi=", not "doi:". String authorityPlusIdentifier = persistentId.replaceFirst("doi:", ""); // curl https://api.datacite.org/events?doi=10.7910/dvn/hqzoob&source=crossref - String baseUrl = System.getProperty("doi.mdcurlstring"); + String baseUrl = System.getProperty("doi.mdcbaseurlstring"); if(null==baseUrl) { //Backward compatible default to the production server baseUrl="https://api.datacite.org"; From b2e3a1540c617de54f55f1d4766386c07728c712 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Tue, 22 Oct 2019 15:21:33 -0400 Subject: [PATCH 7/8] add logging of MDC citation get URL and tweak docs #6137 --- .../source/admin/make-data-count.rst | 6 ++++- .../source/admin/troubleshooting.rst | 25 +++++++++++++++++-- .../source/installation/config.rst | 2 ++ .../iq/dataverse/api/MakeDataCountApi.java | 1 + 4 files changed, 31 insertions(+), 3 deletions(-) diff --git a/doc/sphinx-guides/source/admin/make-data-count.rst b/doc/sphinx-guides/source/admin/make-data-count.rst index d8fc51db002..157b71d3e20 100644 --- a/doc/sphinx-guides/source/admin/make-data-count.rst +++ b/doc/sphinx-guides/source/admin/make-data-count.rst @@ -68,7 +68,7 @@ If you wish to start logging in advance of setting up other components, or wish After you have your first day of logs, you can process them the next day. Enable or Disable Display of Make Data Count Metrics -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ By default, when MDC logging is enabled (when ``:MDCLogPath`` is set), Dataverse will display MDC metrics instead of it's internal (legacy) metrics. You can avoid this (e.g. to collect MDC metrics for some period of time before starting to display them) by setting ``:DisplayMDCMetrics`` to false. @@ -135,11 +135,15 @@ Once you are satisfied with your testing, you should contact support@datacite.or - Views ("investigations" in COUNTER) - Downloads ("requests" in COUNTER) +.. _MDC-updateCitationsForDataset: + Configuring Dataverse for Make Data Count Citations --------------------------------------------------- Please note: as explained in the note above about limitations, this feature is not available to installations of Dataverse that use Handles. +To configure Dataverse to pull citations from the test vs. production DataCite server see :ref:`doi.mdcbaseurlstring` in the Installation Guide. + Please note that in the curl example, Bash environment variables are used with the idea that you can set a few environment variables and copy and paste the examples as is. For example, "$DOI" could become "doi:10.5072/FK2/BL2IBM" by issuing the following export command from Bash: ``export DOI="doi:10.5072/FK2/BL2IBM"`` diff --git a/doc/sphinx-guides/source/admin/troubleshooting.rst b/doc/sphinx-guides/source/admin/troubleshooting.rst index 1b22a58555b..eb7872bac20 100644 --- a/doc/sphinx-guides/source/admin/troubleshooting.rst +++ b/doc/sphinx-guides/source/admin/troubleshooting.rst @@ -32,12 +32,28 @@ A User Needs Their Account to Be Converted From Institutional (Shibboleth), ORCI See :ref:`converting-shibboleth-users-to-local` and :ref:`converting-oauth-users-to-local`. +.. _troubleshooting-glassfish: + Glassfish --------- -``server.log`` is the main place to look when you encounter problems. Hopefully an error message has been logged. If there's a stack trace, it may be of interest to developers, especially they can trace line numbers back to a tagged version. +.. _glassfish-log: + +Finding the Glassfish Log File +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +``/usr/local/glassfish4/glassfish/domains/domain1/logs/server.log`` is the main place to look when you encounter problems (assuming you installed Glassfish in the default directory). Hopefully an error message has been logged. If there's a stack trace, it may be of interest to developers, especially they can trace line numbers back to a tagged version or commit. Send more of the stack trace (the entire file if possible) to developers who can help (see "Getting Help", below) and be sure to say which version of Dataverse you are running. + +.. _increase-glassfish-logging: + +Increasing Glassfish Logging +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -For debugging purposes, you may find it helpful to increase logging levels as mentioned in the :doc:`/developers/debugging` section of the Developer Guide. +For debugging purposes, you may find it helpful to temporarily increase logging levels. Here's an example of increasing logging for the Java class behind the "datasets" API endpoints: + +``./asadmin set-log-levels edu.harvard.iq.dataverse.api.Datasets=FINE`` + +For more on setting log levels, see the :doc:`/developers/debugging` section of the Developer Guide. Our guides focus on using the command line to manage Glassfish but you might be interested in an admin GUI at http://localhost:4848 @@ -100,3 +116,8 @@ Many Files with a File Type of "Unknown", "Application", or "Binary" -------------------------------------------------------------------- From the home page of a Dataverse installation you can get a count of files by file type by clicking "Files" and then scrolling down to "File Type". If you see a lot of files that are "Unknown", "Application", or "Binary" you can have Dataverse attempt to redetect the file type by using the :ref:`Redetect File Type ` API endpoint. + +Getting Help +------------ + +If the troubleshooting advice above didn't help, contact any of the support channels mentioned in the :ref:`support` section of the Installation Guide. diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index 0466b72706f..f252ca310ba 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -962,6 +962,8 @@ See also these related database settings below: doi.mdcbaseurlstring ++++++++++++++++++++ +This configuration option affects the ``updateCitationsForDataset`` API endpoint documented under :ref:`MDC-updateCitationsForDataset` in the Admin Guide. + As of this writing, "https://api.datacite.org" (DataCite) and "https://api.test.datacite.org" (DataCite Testing) are the main valid values. Out of the box, Dataverse is configured to use a test DataCite MDC base URL string. You can delete it like this: diff --git a/src/main/java/edu/harvard/iq/dataverse/api/MakeDataCountApi.java b/src/main/java/edu/harvard/iq/dataverse/api/MakeDataCountApi.java index 5722acc11ec..da827de9c1f 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/MakeDataCountApi.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/MakeDataCountApi.java @@ -145,6 +145,7 @@ public Response updateCitationsForDataset(@PathParam("id") String id) throws Mal baseUrl="https://api.datacite.org"; } URL url = new URL(baseUrl + "/events?doi=" + authorityPlusIdentifier + "&source=crossref"); + logger.fine("Requesting citations from " + url); HttpURLConnection connection = (HttpURLConnection) url.openConnection(); connection.setRequestMethod("GET"); int status = connection.getResponseCode(); From 4bf2ad2c7a65d176e29c7ece21ccaa4159f4f91b Mon Sep 17 00:00:00 2001 From: qqmyers Date: Tue, 22 Oct 2019 15:31:44 -0400 Subject: [PATCH 8/8] per @landreev --- scripts/installer/install | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/installer/install b/scripts/installer/install index 2f4da754cfc..faac3452eba 100755 --- a/scripts/installer/install +++ b/scripts/installer/install @@ -1101,7 +1101,9 @@ sub run_asadmin_script { $ENV{'RSERVE_PORT'} = $CONFIG_DEFAULTS{'RSERVE_PORT'}; $ENV{'RSERVE_USER'} = $CONFIG_DEFAULTS{'RSERVE_USER'}; $ENV{'RSERVE_PASS'} = $CONFIG_DEFAULTS{'RSERVE_PASSWORD'}; - + $ENV{'DOI_BASEURL'} = $CONFIG_DEFAULTS{'DOI_BASEURL'}; + $ENV{'DOI_MDCBASEURL'} = $CONFIG_DEFAULTS{'DOI_MDCBASEURL'}; + $ENV{'HOST_ADDRESS'} = $CONFIG_DEFAULTS{'HOST_DNS_ADDRESS'}; my ($mail_server_host, $mail_server_port) = split (":", $CONFIG_DEFAULTS{'MAIL_SERVER'});