#2438 Add Asynchronous File registration to Publish Dataset Command

sekmiller · sekmiller · commit 77cd617290d9 · 2018-03-26T11:45:07.000-04:00
diff --git a/src/main/java/edu/harvard/iq/dataverse/AbstractIdServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/AbstractIdServiceBean.java
@@ -1,10 +1,7 @@
 package edu.harvard.iq.dataverse;
 
-import static edu.harvard.iq.dataverse.IdServiceBean.logger;
-import edu.harvard.iq.dataverse.engine.command.CommandContext;
 import edu.harvard.iq.dataverse.settings.SettingsServiceBean;
 import edu.harvard.iq.dataverse.util.SystemConfig;
-import java.text.SimpleDateFormat;
 
 import javax.ejb.EJB;
 import java.util.*;
@@ -20,6 +17,12 @@ public abstract class AbstractIdServiceBean implements IdServiceBean {
     @EJB
     SettingsServiceBean settingsService;
     @EJB
+    EjbDataverseEngine commandEngine;
+    @EJB
+    DatasetServiceBean datasetService;
+    @EJB
+    DataFileServiceBean datafileService;
+    @EJB
     SystemConfig systemConfig;
 
     @Override
@@ -87,5 +90,28 @@ public HashMap<String, String> getMetadataForTargetURL(DvObject dvObject) {
         metadata.put("_target", getTargetUrl(dvObject));
         return metadata;
     }
+    
+    @Override
+    public DvObject generateIdentifier(DvObject dvObject) {
 
+        String protocol = dvObject.getProtocol() == null ? settingsService.getValueForKey(SettingsServiceBean.Key.Protocol) : dvObject.getProtocol();
+        String authority = dvObject.getAuthority() == null ? settingsService.getValueForKey(SettingsServiceBean.Key.Authority) : dvObject.getAuthority();
+        String doiSeparator = dvObject.getDoiSeparator() == null ? settingsService.getValueForKey(SettingsServiceBean.Key.DoiSeparator) : dvObject.getDoiSeparator();
+        IdServiceBean idServiceBean = IdServiceBean.getBean(protocol, commandEngine.getContext());
+        if (dvObject.isInstanceofDataset()) {
+            dvObject.setIdentifier(datasetService.generateDatasetIdentifier((Dataset) dvObject, idServiceBean));
+        } else {
+            dvObject.setIdentifier(datafileService.generateDataFileIdentifier((DataFile) dvObject, idServiceBean));
+        }
+        if (dvObject.getProtocol() == null) {
+            dvObject.setProtocol(protocol);
+        }
+        if (dvObject.getAuthority() == null) {
+            dvObject.setAuthority(authority);
+        }
+        if (dvObject.getDoiSeparator() == null) {
+            dvObject.setDoiSeparator(doiSeparator);
+        }
+        return dvObject;
+    }
 }
diff --git a/src/main/java/edu/harvard/iq/dataverse/DOIDataCiteServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DOIDataCiteServiceBean.java
@@ -52,6 +52,9 @@ public boolean alreadyExists(DvObject dvObject) {
     @Override
     public String createIdentifier(DvObject dvObject) throws Exception {
         logger.log(Level.FINE,"createIdentifier");
+            if(dvObject.getIdentifier() == null || dvObject.getIdentifier().isEmpty() ){
+                dvObject = generateIdentifier(dvObject);
+            }
             String identifier = getIdentifier(dvObject);
             HashMap<String, String> metadata = getMetadataForCreateIndicator(dvObject);
         metadata.put("_status", "reserved");
diff --git a/src/main/java/edu/harvard/iq/dataverse/DOIEZIdServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DOIEZIdServiceBean.java
@@ -246,6 +246,9 @@ public List<String> getProviderInformation(){
     public String createIdentifier(DvObject dvObject) throws Throwable {
 
             logger.log(Level.FINE, "createIdentifier");
+            if(dvObject.getIdentifier() == null || dvObject.getIdentifier().isEmpty() ){
+                dvObject = generateIdentifier(dvObject);
+            }
             String identifier = getIdentifier(dvObject);
             HashMap<String, String> metadata = getMetadataForCreateIndicator(dvObject);
             metadata.put("datacite.resourcetype", "Dataset");
@@ -260,6 +263,7 @@ public String createIdentifier(DvObject dvObject) throws Throwable {
                 logger.log(Level.WARNING, "localized message {0}", e.getLocalizedMessage());
                 logger.log(Level.WARNING, "cause", e.getCause());
                 logger.log(Level.WARNING, "message {0}", e.getMessage());
+                logger.log(Level.WARNING, "identifier: ", identifier);
                 throw e;
             }
     }
diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java
@@ -1885,10 +1885,12 @@ private String releaseDataset(boolean minor) {
                 // the process. So it may be premature to show the "success" message at this point.                
                 if (dataset.isLockedFor(DatasetLock.Reason.Workflow)) {
                     JH.addMessage(FacesMessage.SEVERITY_WARN, BundleUtil.getStringFromBundle("dataset.locked.message"), BundleUtil.getStringFromBundle("dataset.publish.workflow.inprogress"));
-                } else if (dataset.isLockedFor(DatasetLock.Reason.pidRegister) || dataset.getFiles().size() > systemConfig.getPIDAsynchRegFileCount()){                    
+                }                
+                else if (dataset.isLockedFor(DatasetLock.Reason.pidRegister) || dataset.getFiles().size() > systemConfig.getPIDAsynchRegFileCount()){                    
                     JsfHelper.addWarningMessage(BundleUtil.getStringFromBundle("dataset.pidRegister.workflow.inprogress"));
                     JH.addMessage(FacesMessage.SEVERITY_WARN, BundleUtil.getStringFromBundle("dataset.locked.message"), BundleUtil.getStringFromBundle("dataset.pidRegister.workflow.inprogress"));
-                } else {
+                } 
+                else {
                     JsfHelper.addSuccessMessage(BundleUtil.getStringFromBundle("dataset.message.publishSuccess"));
                 }
             } catch (CommandException ex) {
diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DatasetServiceBean.java
@@ -6,6 +6,10 @@
 import edu.harvard.iq.dataverse.authorization.users.User;
 import edu.harvard.iq.dataverse.dataaccess.ImageThumbConverter;
 import edu.harvard.iq.dataverse.dataset.DatasetUtil;
+import edu.harvard.iq.dataverse.engine.command.CommandContext;
+import edu.harvard.iq.dataverse.engine.command.DataverseRequest;
+import edu.harvard.iq.dataverse.engine.command.exception.CommandException;
+import edu.harvard.iq.dataverse.engine.command.impl.FinalizeDatasetPublicationCommand;
 import edu.harvard.iq.dataverse.harvest.server.OAIRecordServiceBean;
 import edu.harvard.iq.dataverse.search.IndexServiceBean;
 import edu.harvard.iq.dataverse.settings.SettingsServiceBean;
@@ -904,6 +908,14 @@ public WorkflowComment addWorkflowComment(WorkflowComment workflowComment) {
         return workflowComment;
     }
     
+    @Asynchronous
+    public void callFinalizePublishCommandAsynchronously(Dataset theDataset, CommandContext ctxt, DataverseRequest request) throws CommandException {
+
+        String nonNullDefaultIfKeyNotFound = "";
+        String doiProvider = ctxt.settings().getValueForKey(SettingsServiceBean.Key.DoiProvider, nonNullDefaultIfKeyNotFound);
+        commandEngine.submit(new FinalizeDatasetPublicationCommand(theDataset, doiProvider, request));
+    }
+    
     /*
      Experimental asynchronous method for requesting persistent identifiers for 
      datafiles. We decided not to run this method on upload/create (so files 
diff --git a/src/main/java/edu/harvard/iq/dataverse/IdServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/IdServiceBean.java
@@ -71,6 +71,8 @@ static IdServiceBean getBean(CommandContext ctxt) {
         String    protocol = ctxt.settings().getValueForKey(SettingsServiceBean.Key.Protocol, nonNullDefaultIfKeyNotFound);
         return getBean(protocol, ctxt);
     }
+    
+     DvObject generateIdentifier(DvObject dvObject);
 
 
 }
diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
@@ -442,7 +442,6 @@ public Response publishDataset(@PathParam("id") String id, @QueryParam("type") S
     @Path("{id}/move/{targetDataverseAlias}")
     public Response moveDataset(@PathParam("id") String id, @PathParam("targetDataverseAlias") String targetDataverseAlias, @QueryParam("forceMove") Boolean force) {        
         try{
-            System.out.print("force: " + force);
             User u = findUserOrDie();            
             Dataset ds = findDatasetOrDie(id);
             Dataverse target = dataverseService.findByAlias(targetDataverseAlias);
diff --git a/src/main/java/edu/harvard/iq/dataverse/datasetutility/AddReplaceFileHelper.java b/src/main/java/edu/harvard/iq/dataverse/datasetutility/AddReplaceFileHelper.java
@@ -1427,6 +1427,7 @@ private boolean step_060_addFilesViaIngestService(){
                     maxIdentifier++;
                     dataFileIdentifier = datasetIdentifier + "/" + maxIdentifier.toString();
                 }
+                //commandEngine.submit(new CreateDataFileCommand(dataFile, workingVersion, dvRequest, dataFileIdentifier));
                 commandEngine.submit(new CreateDataFileCommand(dataFile, workingVersion, dvRequest, dataFileIdentifier, true));
             }
         } catch (CommandException cmdex) {
diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/FinalizeDatasetPublicationCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/FinalizeDatasetPublicationCommand.java
@@ -74,7 +74,6 @@ public Dataset execute(CommandContext ctxt) throws CommandException {
         // comes from there. There's a chance that the final merge, at the end of this
         // command, would be sufficient. -- L.A. Sep. 6 2017
         theDataset = ctxt.em().merge(theDataset);
-        
         //if the publisher hasn't contributed to this version
         DatasetVersionUser ddu = ctxt.datasets().getDatasetVersionUser(theDataset.getLatestVersion(), getUser());
         
@@ -91,7 +90,6 @@ public Dataset execute(CommandContext ctxt) throws CommandException {
         
         updateParentDataversesSubjectsField(theDataset, ctxt);
         publicizeExternalIdentifier(ctxt);
-        
         PrivateUrl privateUrl = ctxt.engine().submit(new GetPrivateUrlCommand(getRequest(), theDataset));
         if (privateUrl != null) {
             ctxt.engine().submit(new DeletePrivateUrlCommand(getRequest(), theDataset));
@@ -170,29 +168,19 @@ private void updateParentDataversesSubjectsField(Dataset savedDataset, CommandCo
     private void publicizeExternalIdentifier(CommandContext ctxt) throws CommandException {
         String protocol = theDataset.getProtocol();
         IdServiceBean idServiceBean = IdServiceBean.getBean(protocol, ctxt);
-        if (idServiceBean != null) {
-            try {
-                idServiceBean.publicizeIdentifier(theDataset);
-                theDataset.setGlobalIdCreateTime(new Date());
-                theDataset.setIdentifierRegistered(true);
-                //if the dataset is locked for PID registration then don't register here
-                //done asynch before calling
-                if (!theDataset.isLockedFor(DatasetLock.Reason.pidRegister)) {
-                    for (DataFile df : theDataset.getFiles()) {
-                        idServiceBean.publicizeIdentifier(df);
-                        df.setGlobalIdCreateTime(new Date());
-                        df.setIdentifierRegistered(true);
-                    }
-                } else {
-                    DatasetLock doomed = theDataset.getLockFor(DatasetLock.Reason.pidRegister);
-                    theDataset.getLocks().remove(doomed);
-                    ctxt.em().remove(doomed);
-                    ctxt.em().flush();
-                }
-
-            } catch (Throwable e) {
-                throw new CommandException(BundleUtil.getStringFromBundle("dataset.publish.error", idServiceBean.getProviderInformation()), this);
+        try {
+            idServiceBean.publicizeIdentifier(theDataset);
+            theDataset.setGlobalIdCreateTime(new Date());
+            theDataset.setIdentifierRegistered(true);
+            for (DataFile df : theDataset.getFiles()) {
+                idServiceBean.publicizeIdentifier(df);
+                df.setGlobalIdCreateTime(new Date());
+                df.setIdentifierRegistered(true);
+                DataFile merged = ctxt.em().merge(df);
+                merged = null;
             }
+        } catch (Throwable e) {
+            throw new CommandException(BundleUtil.getStringFromBundle("dataset.publish.error", idServiceBean.getProviderInformation()), this);
         }
     }
     
diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/PublishDatasetCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/PublishDatasetCommand.java
@@ -12,10 +12,10 @@
 import edu.harvard.iq.dataverse.settings.SettingsServiceBean;
 import edu.harvard.iq.dataverse.workflow.Workflow;
 import edu.harvard.iq.dataverse.workflow.WorkflowContext.TriggerType;
-import edu.harvard.iq.dataverse.workflow.step.WorkflowStep;
 import java.util.Date;
 import java.util.Optional;
 import static java.util.stream.Collectors.joining;
+import javax.ejb.Asynchronous;
 
 /**
  * Kick-off a dataset publication process. The process may complete immediately, 
@@ -75,24 +75,34 @@ public PublishDatasetResult execute(CommandContext ctxt) throws CommandException
             return new PublishDatasetResult(theDataset, false);
             
         } else {
-            //if there are more than required size files  register them asychronously (default is 10)
-            if (theDataset.getFiles().size() >  ctxt.systemConfig().getPIDAsynchRegFileCount()){                 
-                Workflow dummy = new Workflow();
-                ctxt.workflows().startPIDRegister(dummy, buildContext(doiProvider, TriggerType.PrePublishDataset), ctxt );
-                AuthenticatedUser requestor = request.getAuthenticatedUser();
-                DatasetLock lock = new DatasetLock(DatasetLock.Reason.pidRegister, requestor);
+            //if there are more than required size files  then call Finalize asychronously (default is 10)
+            if (theDataset.getFiles().size() > ctxt.systemConfig().getPIDAsynchRegFileCount()) {                
+                String info = "Adding File PIDs asynchronously";
+                AuthenticatedUser user = request.getAuthenticatedUser() ;
+                DatasetLock lock = new DatasetLock(DatasetLock.Reason.pidRegister, user);
                 lock.setDataset(theDataset);
+                lock.setInfo(info);
                 lock.setStartTime(new Date());
-                ctxt.em().merge(lock);          
-                return new PublishDatasetResult(ctxt.em().merge(theDataset), false);                                
+                theDataset.getLocks().add(lock);
+                callFinalizeAsync(ctxt);
+                return new PublishDatasetResult(theDataset, false);
             }
             // Synchronous publishing (no workflow involved)
-            theDataset = ctxt.engine().submit( new FinalizeDatasetPublicationCommand(theDataset, doiProvider, getRequest()) );
+            theDataset = ctxt.engine().submit(new FinalizeDatasetPublicationCommand(theDataset, doiProvider, getRequest()));
             return new PublishDatasetResult(ctxt.em().merge(theDataset), true);
         }
     }
     
     
+    @Asynchronous
+    private void callFinalizeAsync(CommandContext ctxt) throws CommandException {
+        try {
+            ctxt.datasets().callFinalizePublishCommandAsynchronously(theDataset, ctxt, request);
+        } catch (CommandException ce){
+            throw new CommandException("Publish Dataset failed", this);
+        }
+    }    
+    
     /**
      * See that publishing the dataset in the requested manner makes sense, at
      * the given state of the dataset.
diff --git a/src/main/java/edu/harvard/iq/dataverse/workflow/WorkflowServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/workflow/WorkflowServiceBean.java
@@ -97,46 +97,8 @@ public void start(Workflow wf, WorkflowContext ctxt) throws CommandException {
         forward(wf, ctxt);
     }
     
-    /**
-     * Starts executing workflow {@code wf} to register File PIDs
-     *
-     * @param wf the workflow to execute.
-     * @param ctxt the context in which the workflow is executed.
-     * @param cctxt context for calling command
-     * @throws CommandException If the dataset could not be locked.
-     */
-    @Asynchronous
-    public void startPIDRegister(Workflow wf, WorkflowContext ctxt, CommandContext cctxt) throws CommandException {
-        ctxt = refresh(ctxt);
-        lockDatasetForPIDRegistration(ctxt);
-        runPIDRegistration(wf, ctxt, cctxt);
-    }
-    
-    @Asynchronous
-    private void runPIDRegistration(Workflow wf, WorkflowContext ctxt, CommandContext cctxt) throws CommandException{
-        
-        String protocol = ctxt.getDataset().getProtocol();
-        Dataset theDataset = ctxt.getDataset();
-        IdServiceBean idServiceBean = IdServiceBean.getBean(protocol, cctxt);
-        if (idServiceBean!= null )
-        try {
-            for (DataFile df: theDataset.getFiles()){
-                idServiceBean.publicizeIdentifier(df);
-                df.setGlobalIdCreateTime(new Date());
-                df.setIdentifierRegistered(true);
-            }
-            
-        } catch (Throwable e) {
-            throw new CommandException(BundleUtil.getStringFromBundle("dataset.publish.error", idServiceBean.getProviderInformation()),null); 
-        }
-        
-        try {
-             engine.submit( new FinalizeDatasetPublicationCommand(ctxt.getDataset(), ctxt.getDoiProvider(), ctxt.getRequest()) );                                
-            } catch (CommandException ex) {
-                logger.log(Level.SEVERE, "Exception finalizing workflow " + ctxt.getInvocationId() +": " + ex.getMessage(), ex);
-                rollbackPIDRegistration(wf, ctxt);
-        }       
-    }
+
+
 
     /**
      * Starting the resume process for a pending workflow. We first delete the
@@ -177,26 +139,6 @@ private void doResume(PendingWorkflowInvocation pending, String body) {
             executeSteps(wf, ctxt, pending.getPendingStepIdx() + 1);
         }
     }
-    
-    @Asynchronous
-    private void rollbackPIDRegistration(Workflow wf, WorkflowContext ctxt) {
-        ctxt = refresh(ctxt);
-       
-        logger.log( Level.INFO, "Removing registration lock");
-        try {
-            logger.log(Level.INFO, "PID Registration Failed");
-            engine.submit( new RemoveLockCommand(ctxt.getRequest(), ctxt.getDataset(), DatasetLock.Reason.pidRegister) );
-            
-            // Corner case - delete locks generated within this same transaction.
-            Query deleteQuery = em.createQuery("DELETE from DatasetLock l WHERE l.dataset.id=:id AND l.reason=:reason");
-            deleteQuery.setParameter("id", ctxt.getDataset().getId() );
-            deleteQuery.setParameter("reason", DatasetLock.Reason.Workflow );
-            deleteQuery.executeUpdate();
-            
-        } catch (CommandException ex) {
-            logger.log(Level.SEVERE, "Error restoring dataset locks state after rollback: " + ex.getMessage(), ex);
-        }
-    }
 
     @Asynchronous
     private void rollback(Workflow wf, WorkflowContext ctxt, Failure failure, int lastCompletedStepIdx) {
@@ -301,15 +243,6 @@ void lockDataset( WorkflowContext ctxt ) throws CommandException {
         em.flush();
     }
     
-    @TransactionAttribute(TransactionAttributeType.REQUIRES_NEW)
-    void lockDatasetForPIDRegistration( WorkflowContext ctxt ) throws CommandException {
-        final DatasetLock datasetLock = new DatasetLock(DatasetLock.Reason.pidRegister, ctxt.getRequest().getAuthenticatedUser());
-        datasetLock.setDataset(ctxt.getDataset());
-        ctxt.getDataset().getLocks().add(datasetLock);
-        em.persist(datasetLock);
-        em.flush();
-    }
-    
     //
     //
     //////////////////////////////////////////////////////////////
diff --git a/src/main/webapp/file.xhtml b/src/main/webapp/file.xhtml
@@ -362,7 +362,8 @@
                                                     #{FilePage.file.getGlobalId()}
                                                 </div>
                                             </div>
-                                            <div class="form-group" jsf:rendered="#{FilePage.publiclyDownloadable}">
+                                            <!--Only show Public download url if identifier is available -->
+                                            <div class="form-group" jsf:rendered="#{FilePage.publiclyDownloadable and FilePage.file.identifier != null}">
                                                 <label class="col-sm-3 control-label">                                                   
                                                    #{bundle['file.metadataTab.fileMetadata.downloadUrl.label']}
                                                 </label>

Original file line number	Diff line number	Diff line change
`@@ -71,6 +71,8 @@ static IdServiceBean getBean(CommandContext ctxt) {`
`71`	`71`	`String protocol = ctxt.settings().getValueForKey(SettingsServiceBean.Key.Protocol, nonNullDefaultIfKeyNotFound);`
`72`	`72`	`return getBean(protocol, ctxt);`
`73`	`73`	`}`
	`74`	`+`
	`75`	`+ DvObject generateIdentifier(DvObject dvObject);`
`74`	`76`
`75`	`77`
`76`	`78`	`}`
Original file line number	Diff line number	Diff line change
`@@ -1427,6 +1427,7 @@ private boolean step_060_addFilesViaIngestService(){`
`1427`	`1427`	`maxIdentifier++;`
`1428`	`1428`	`dataFileIdentifier = datasetIdentifier + "/" + maxIdentifier.toString();`
`1429`	`1429`	`}`
	`1430`	`+ //commandEngine.submit(new CreateDataFileCommand(dataFile, workingVersion, dvRequest, dataFileIdentifier));`
`1430`	`1431`	`commandEngine.submit(new CreateDataFileCommand(dataFile, workingVersion, dvRequest, dataFileIdentifier, true));`
`1431`	`1432`	`}`
`1432`	`1433`	`} catch (CommandException cmdex) {`