3583 dv cannot parse dois that don't have a separator in authority/prefix

doc/sphinx-guides/source/installation/config.rst

@@ -124,7 +124,7 @@ Out of the box, Dataverse is configured for DOIs. Here are the configuration opt
 - :ref:`:DoiProvider <:DoiProvider>`
 - :ref:`:Protocol <:Protocol>`
 - :ref:`:Authority <:Authority>`
-- :ref:`:DoiSeparator <:DoiSeparator>`
+- :ref:`:Shoulder <:Shoulder>`
 - :ref:`:IdentifierGenerationStyle <:IdentifierGenerationStyle>` (optional)
 - :ref:`:DataFilePIDFormat <:DataFilePIDFormat>` (optional)
 
@@ -552,7 +552,7 @@ See also these related database settings below:
 - :ref:`:DoiProvider`
 - :ref:`:Protocol`  
 - :ref:`:Authority`
-- :ref:`:DoiSeparator`
+- :ref:`:Shoulder`
 
 .. _doi.username:
 
@@ -717,7 +717,7 @@ As of this writing "EZID" and "DataCite" are the only valid options. DoiProvider
 
 ``curl -X PUT -d EZID http://localhost:8080/api/admin/settings/:DoiProvider``
 
-This setting relates to the ``:Protocol``, ``:Authority``, ``:DoiSeparator``, and ``:IdentifierGenerationStyle`` database settings below as well as the following JVM options:
+This setting relates to the ``:Protocol``, ``:Authority``, ``:Shoulder``, and ``:IdentifierGenerationStyle`` database settings below as well as the following JVM options:
 
 - :ref:`doi.baseurlstring`
 - :ref:`doi.username`
@@ -741,25 +741,23 @@ Use the authority assigned to you by your DoiProvider or HandleProvider.
 
 ``curl -X PUT -d 10.xxxx http://localhost:8080/api/admin/settings/:Authority``
 
-.. _:DoiSeparator:
+.. _:Shoulder:
 
-:DoiSeparator
-+++++++++++++
-
-It is recommended that you keep this as a slash ("/").
+:Shoulder
+++++++++++++
 
-``curl -X PUT -d "/" http://localhost:8080/api/admin/settings/:DoiSeparator``
+Out of the box, the DOI shoulder is set to "FK2/" but this is for testing only! When you apply for your DOI namespace, you may have requested a shoulder. The following is only an example and a trailing slash is optional.
 
-**Note:** The name DoiSeparator is a misnomer. This setting is used by some **handles**-specific code too. It *must* be set to '/' when using handles.
+``curl -X PUT -d "MyShoulder/" http://localhost:8080/api/admin/settings/:Shoulder``
 
 .. _:IdentifierGenerationStyle:
 
 :IdentifierGenerationStyle
 ++++++++++++++++++++++++++
 
-By default, Dataverse generates a random 6 character string to use as the identifier
+By default, Dataverse generates a random 6 character string, pre-pended by the Shoulder if set, to use as the identifier
 for a Dataset. Set this to ``sequentialNumber`` to use sequential numeric values 
-instead. (the assumed default setting is ``randomString``). 
+instead (again pre-pended by the Shoulder if set). (the assumed default setting is ``randomString``). 
 In addition to this setting, a database sequence must be created in the database. 
 We provide the script below (downloadable :download:`here </_static/util/createsequence.sql>`).
 You may need to make some changes to suit your system setup, see the comments for more information: 
@@ -785,7 +783,7 @@ This setting controls the way that the "identifier" component of a file's persis
 
 By default the identifier for a file is dependent on its parent dataset. For example, if the identifier of a dataset is "TJCLKP", the identifier for a file within that dataset will consist of the parent dataset's identifier followed by a slash ("/"), followed by a random 6 character string, yielding "TJCLKP/MLGWJO". Identifiers in this format are what you should expect if you leave ``:DataFilePIDFormat`` undefined or set it to ``DEPENDENT`` and have not changed the ``:IdentifierGenerationStyle`` setting from its default.
 
-Alternatively, the indentifier for File PIDs can be configured to be independent of Dataset PIDs using the setting "``INDEPENDENT``". In this case, file PIDs will not contain the PIDs of their parent datasets, and their PIDs will be generated the exact same way that datasets' PIDs are, based on the ``:IdentifierGenerationStyle`` setting described above (random 6 character strings or sequential numbers).
+Alternatively, the identifier for File PIDs can be configured to be independent of Dataset PIDs using the setting "``INDEPENDENT``". In this case, file PIDs will not contain the PIDs of their parent datasets, and their PIDs will be generated the exact same way that datasets' PIDs are, based on the ``:IdentifierGenerationStyle`` setting described above (random 6 character strings or sequential numbers, pre-pended by any shoulder).
 
 The chart below shows examples from each possible combination of parameters from the two settings. ``:IdentifierGenerationStyle`` can be either ``randomString`` (the default) or ``sequentialNumber`` and ``:DataFilePIDFormat`` can be either ``DEPENDENT`` (the default) or ``INDEPENDENT``. In the examples below the "identifier" for the dataset is "TJCLKP" for "randomString" and "100001" for "sequentialNumber".
 

scripts/api/setup-all.sh

@@ -49,9 +49,9 @@ curl -X PUT -d yes "$SERVER/admin/settings/:AllowSignUp"
 curl -X PUT -d /dataverseuser.xhtml?editMode=CREATE "$SERVER/admin/settings/:SignUpUrl"
 
 curl -X PUT -d doi "$SERVER/admin/settings/:Protocol"
-curl -X PUT -d 10.5072/FK2 "$SERVER/admin/settings/:Authority"
+curl -X PUT -d 10.5072 "$SERVER/admin/settings/:Authority"
+curl -X PUT -d "FK2/" "$SERVER/admin/settings/:Shoulder"
 curl -X PUT -d EZID "$SERVER/admin/settings/:DoiProvider"
-curl -X PUT -d / "$SERVER/admin/settings/:DoiSeparator"
 curl -X PUT -d burrito $SERVER/admin/settings/BuiltinUsers.KEY
 curl -X PUT -d localhost-only $SERVER/admin/settings/:BlockedApiPolicy
 echo

scripts/api/setup-optional-harvard.sh

@@ -2,6 +2,9 @@
 SERVER=http://localhost:8080/api
 
 echo "Setting up Harvard-specific settings"
+# :Authority and :Shoulder are commented out so this script can be used on test servers
+#curl -X PUT -d 10.7910 "$SERVER/admin/settings/:Authority"
+#curl -X PUT -d "DVN/" "$SERVER/admin/settings/:Shoulder"
 echo "- Application Status header"
 curl -s -X PUT -d 'Upgrade in progress...' $SERVER/admin/settings/:StatusMessageHeader
 echo "- Application Status message"

scripts/database/upgrades/upgrade_v4.8.6_to_v4.9.0.sql

@@ -1,56 +1,71 @@
-ALTER TABLE externaltool ADD COLUMN type character varying(255);
-ALTER TABLE externaltool ALTER COLUMN type SET NOT NULL;
--- Previously, the only explore tool was TwoRavens. We now persist the name of the tool.
-UPDATE guestbookresponse SET downloadtype = 'TwoRavens' WHERE downloadtype = 'Explore';
-ALTER TABLE filemetadata ADD COLUMN prov_freeform text;
--- ALTER TABLE datafile ADD COLUMN prov_cplid int;
-ALTER TABLE datafile ADD COLUMN prov_entityname text;
-
--- Moves DOI fields from Dataset to DVObject
--- so that Identifiers may be added to DataFiles
-
-ALTER TABLE dvobject ADD COLUMN
-  authority character varying(255),
-  ADD COLUMN doiseparator character varying(255),
-  ADD COLUMN globalidcreatetime timestamp without time zone,
-  ADD COLUMN identifierRegistered boolean,
-  ADD COLUMN identifier character varying(255),
-  ADD COLUMN protocol character varying(255);
-
-
-UPDATE dvobject
-SET authority=(SELECT dataset.authority
-FROM dataset
-WHERE dataset.id=dvobject.id AND dvobject.dtype='Dataset') where dvobject.dtype='Dataset';
-
-UPDATE dvobject
-SET doiseparator=(SELECT dataset.doiseparator
-FROM dataset
-WHERE dataset.id=dvobject.id AND dvobject.dtype='Dataset') where dvobject.dtype='Dataset';
-
-UPDATE dvobject
-SET globalidcreatetime=(SELECT dataset.globalidcreatetime
-FROM dataset
-WHERE dataset.id=dvobject.id AND dvobject.dtype='Dataset') where dvobject.dtype='Dataset';
-
-UPDATE dvobject
-SET identifierRegistered= true where globalidcreatetime is not null;
-
-UPDATE dvobject
-SET identifier=(SELECT dataset.identifier
-FROM dataset
-WHERE dataset.id=dvobject.id AND dvobject.dtype='Dataset') where dvobject.dtype='Dataset';
-
-UPDATE dvobject
-SET protocol=(SELECT dataset.protocol
-FROM dataset
-WHERE dataset.id=dvobject.id AND dvobject.dtype='Dataset') where dvobject.dtype='Dataset';
-
-ALTER TABLE dataset ALTER identifier DROP NOT NULL;
-
-ALTER TABLE dataset DROP COLUMN authority;
-ALTER TABLE dataset DROP COLUMN doiseparator;
-ALTER TABLE dataset DROP COLUMN globalidcreatetime;
-ALTER TABLE dataset DROP COLUMN identifier;
-ALTER TABLE dataset DROP COLUMN protocol;
-
+ALTER TABLE externaltool ADD COLUMN type character varying(255);
+ALTER TABLE externaltool ALTER COLUMN type SET NOT NULL;
+-- Previously, the only explore tool was TwoRavens. We now persist the name of the tool.
+UPDATE guestbookresponse SET downloadtype = 'TwoRavens' WHERE downloadtype = 'Explore';
+ALTER TABLE filemetadata ADD COLUMN prov_freeform text;
+-- ALTER TABLE datafile ADD COLUMN prov_cplid int;
+ALTER TABLE datafile ADD COLUMN prov_entityname text;
+
+-- Moves DOI fields from Dataset to DVObject
+-- so that Identifiers may be added to DataFiles
+
+ALTER TABLE dvobject ADD COLUMN
+ authority character varying(255),
+ ADD COLUMN globalidcreatetime timestamp without time zone,
+ ADD COLUMN doiseparator character varying(255),
+ ADD COLUMN identifierRegistered boolean,
+ ADD COLUMN identifier character varying(255),
+ ADD COLUMN protocol character varying(255);
+
+--Migrate data from Dataset to DvObject
+UPDATE dvobject
+SET authority=(SELECT dataset.authority
+FROM dataset
+WHERE dataset.id=dvobject.id AND dvobject.dtype='Dataset') where dvobject.dtype='Dataset';
+
+UPDATE dvobject
+SET globalidcreatetime=(SELECT dataset.globalidcreatetime
+FROM dataset
+WHERE dataset.id=dvobject.id AND dvobject.dtype='Dataset') where dvobject.dtype='Dataset';
+
+UPDATE dvobject
+SET doiseparator=(SELECT dataset.doiseparator
+FROM dataset
+WHERE dataset.id=dvobject.id AND dvobject.dtype='Dataset') where dvobject.dtype='Dataset';
+
+UPDATE dvobject
+SET identifierRegistered= true where globalidcreatetime is not null;
+
+UPDATE dvobject
+SET identifier=(SELECT dataset.identifier
+FROM dataset
+WHERE dataset.id=dvobject.id AND dvobject.dtype='Dataset') where dvobject.dtype='Dataset';
+
+UPDATE dvobject
+SET protocol=(SELECT dataset.protocol
+FROM dataset
+WHERE dataset.id=dvobject.id AND dvobject.dtype='Dataset' ) where dvobject.dtype='Dataset';
+
+--Once in DvObject re-parse identifier and authority
+UPDATE dvobject SET identifier=substring(authority, strpos(authority,'/')+1) || doiseparator || identifier WHERE strpos(authority,'/')>0 ;
+UPDATE dvobject SET authority=substring(authority from 0 for strpos(authority,'/')) WHERE strpos(authority,'/')>0;
+
+ALTER TABLE dataset ALTER identifier DROP NOT NULL;
+
+ALTER TABLE dataset DROP COLUMN authority;
+ALTER TABLE dataset DROP COLUMN doiseparator;
+ALTER TABLE dataset DROP COLUMN globalidcreatetime;
+ALTER TABLE dataset DROP COLUMN identifier;
+ALTER TABLE dataset DROP COLUMN protocol;
+
+ALTER TABLE dvobject DROP COLUMN doiseparator;
+
+--Add new setting into content for shoulder
+INSERT INTO setting(name, content)
+VALUES (':Shoulder', (SELECT substring(content, strpos(content,'/')+1) || '/' from setting  where name = ':Authority'));
+
+ --strip shoulder from authority setting
+ UPDATE setting
+ SET content=(SELECT substring(content from 0 for strpos(content,'/'))
+ FROM setting
+ WHERE name=':Authority' and strpos(content,'/')>0) where name=':Authority';

src/main/java/edu/harvard/iq/dataverse/AbstractIdServiceBean.java

@@ -26,9 +26,9 @@ public abstract class AbstractIdServiceBean implements IdServiceBean {
     SystemConfig systemConfig;
 
     @Override
-    public String getIdentifierForLookup(String protocol, String authority, String separator, String identifier) {
+    public String getIdentifierForLookup(String protocol, String authority, String identifier) {
         logger.log(Level.FINE,"getIdentifierForLookup");
-        return protocol + ":" + authority + separator + identifier;
+        return protocol + ":" + authority + "/" + identifier;
     }
 
 
@@ -96,7 +96,6 @@ public DvObject generateIdentifier(DvObject dvObject) {
 
         String protocol = dvObject.getProtocol() == null ? settingsService.getValueForKey(SettingsServiceBean.Key.Protocol) : dvObject.getProtocol();
         String authority = dvObject.getAuthority() == null ? settingsService.getValueForKey(SettingsServiceBean.Key.Authority) : dvObject.getAuthority();
-        String doiSeparator = dvObject.getDoiSeparator() == null ? settingsService.getValueForKey(SettingsServiceBean.Key.DoiSeparator) : dvObject.getDoiSeparator();
         IdServiceBean idServiceBean = IdServiceBean.getBean(protocol, commandEngine.getContext());
         if (dvObject.isInstanceofDataset()) {
             dvObject.setIdentifier(datasetService.generateDatasetIdentifier((Dataset) dvObject, idServiceBean));
@@ -109,9 +108,6 @@ public DvObject generateIdentifier(DvObject dvObject) {
         if (dvObject.getAuthority() == null) {
             dvObject.setAuthority(authority);
         }
-        if (dvObject.getDoiSeparator() == null) {
-            dvObject.setDoiSeparator(doiSeparator);
-        }
         return dvObject;
     }
 }

src/main/java/edu/harvard/iq/dataverse/DOIDataCiteServiceBean.java

@@ -95,15 +95,14 @@ public HashMap getIdentifierMetadata(DvObject dvObject) {
      * Looks up the metadata for a Global Identifier
      * @param protocol the identifier system, e.g. "doi"
      * @param authority the namespace that the authority manages in the identifier system
-     * @param separator the string that separates authority from local identifier part
      * @param identifier the local identifier part
      * @return a Map of metadata. It is empty when the lookup failed, e.g. when
      * the identifier does not exist.
      */
     @Override
-    public HashMap<String, String> lookupMetadataFromIdentifier(String protocol, String authority, String separator, String identifier) {
+    public HashMap<String, String> lookupMetadataFromIdentifier(String protocol, String authority, String identifier) {
         logger.log(Level.FINE,"lookupMetadataFromIdentifier");
-        String identifierOut = getIdentifierForLookup(protocol, authority, separator, identifier);
+        String identifierOut = getIdentifierForLookup(protocol, authority, identifier);
         HashMap<String, String> metadata = new HashMap<>();
         try {
             metadata = doiDataCiteRegisterService.getMetadata(identifierOut);

src/main/java/edu/harvard/iq/dataverse/DOIEZIdServiceBean.java

@@ -99,16 +99,15 @@ public HashMap<String, String> getIdentifierMetadata(DvObject dvObject) {
      * @param protocol the identifier system, e.g. "doi"
      * @param authority the namespace that the authority manages in the
      * identifier system
-     * @param separator the string that separates authority from local
      * identifier part
      * @param identifier the local identifier part
      * @return a Map of metadata. It is empty when the lookup failed, e.g. when
      * the identifier does not exist.
      */
     @Override
-    public HashMap<String, String> lookupMetadataFromIdentifier(String protocol, String authority, String separator, String identifier) {
+    public HashMap<String, String> lookupMetadataFromIdentifier(String protocol, String authority, String identifier) {
         logger.log(Level.FINE,"lookupMetadataFromIdentifier");
-        String identifierOut = getIdentifierForLookup(protocol, authority, separator, identifier);
+        String identifierOut = getIdentifierForLookup(protocol, authority, identifier);
         HashMap<String, String> metadata = new HashMap<>();
         try {
             metadata = ezidService.getMetadata(identifierOut);
@@ -181,15 +180,15 @@ public void deleteIdentifier(DvObject dvObject) throws Exception {
             updateIdentifierStatus(dvObject, "unavailable | withdrawn by author");
             HashMap<String, String> metadata = new HashMap<>();
             metadata.put("_target", "http://ezid.cdlib.org/id/" + dvObject.getProtocol() + ":" + dvObject.getAuthority()
-                    + dvObject.getDoiSeparator() + dvObject.getIdentifier());
+                    + "/" + dvObject.getIdentifier());
             try {
                 modifyIdentifierTargetURL(dvObject);
                 if (dvObject instanceof Dataset ) {
                     Dataset dataset = (Dataset) dvObject;
                     for (DataFile df : dataset.getFiles()) {
                         metadata = new HashMap<>();
                         metadata.put("_target", "http://ezid.cdlib.org/id/" + df.getProtocol() + ":" + df.getAuthority()
-                                + df.getDoiSeparator() + df.getIdentifier());
+                                + "/" + df.getIdentifier());
                                         modifyIdentifierTargetURL(df);
                     }
                 }