7940 stop harvest in progress

doc/release-notes/7940-stop-harvest-in-progress

@@ -0,0 +1,4 @@
+## Mechanism added for stopping a harvest in progress
+
+It is now possible for an admin to stop a long-running harvesting job. See [Harvesting Clients](https://guides.dataverse.org/en/latest/admin/harvestclients.html) guide for more information. 
+

doc/sphinx-guides/source/admin/harvestclients.rst

@@ -21,6 +21,21 @@ Clients are managed on the "Harvesting Clients" page accessible via the :doc:`da
 
 The process of creating a new, or editing an existing client, is largely self-explanatory. It is split into logical steps, in a way that allows the user to go back and correct the entries made earlier. The process is interactive and guidance text is provided. For example, the user is required to enter the URL of the remote OAI server. When they click *Next*, the application will try to establish a connection to the server in order to verify that it is working, and to obtain the information about the sets of metadata records and the metadata formats it supports. The choices offered to the user on the next page will be based on this extra information. If the application fails to establish a connection to the remote archive at the address specified, or if an invalid response is received, the user is given an opportunity to check and correct the URL they entered.
 
+How to Stop a Harvesting Run in Progress
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Some harvesting jobs, especially the initial full harvest of a very large set - such as the default set of public datasets at IQSS - can take many hours. In case it is necessary to terminate such a long-running job, the following mechanism is provided (note that it is only available to a sysadmin with shell access to the application server): Create an empty file in the domain logs directory with the following name: ``stopharvest_<name>.<pid>``, where ``<name>`` is the nickname of the harvesting client and ``<pid>`` is the process id of the Application Server (Payara). This flag file needs to be owned by the same user that's running Payara, so that the application can remove it after stopping the job in progress.
+
+For example:
+
+.. code-block:: bash
+
+  sudo touch /usr/local/payara5/glassfish/domains/domain1/logs/stopharvest_bigarchive.70916
+  sudo chown dataverse /usr/local/payara5/glassfish/domains/domain1/logs/stopharvest_bigarchive.70916
+
+Note: If the application server is stopped and restarted, any running harvesting jobs will be killed but may remain marked as in progress in the database. We thus recommend using the mechanism here to stop ongoing harvests prior to a server restart.
+
+
 What if a Run Fails?
 ~~~~~~~~~~~~~~~~~~~~
 

src/main/java/edu/harvard/iq/dataverse/DashboardPage.java

@@ -97,12 +97,8 @@ public int getNumberOfConfiguredHarvestClients() {
     }
 
     public long getNumberOfHarvestedDatasets() {
-        List<HarvestingClient> configuredHarvestingClients = harvestingClientService.getAllHarvestingClients();
-        if (configuredHarvestingClients == null || configuredHarvestingClients.isEmpty()) {
-            return 0L;
-        }
 
-        Long numOfDatasets = harvestingClientService.getNumberOfHarvestedDatasetByClients(configuredHarvestingClients);
+        Long numOfDatasets = harvestingClientService.getNumberOfHarvestedDatasetsByAllClients();
 
         if (numOfDatasets != null && numOfDatasets > 0L) {
             return numOfDatasets;
@@ -142,7 +138,7 @@ public String getHarvestClientsInfoLabel() {
             infoLabel = configuredHarvestingClients.size() + " harvesting clients configured; ";
         }        
 
-        Long numOfDatasets = harvestingClientService.getNumberOfHarvestedDatasetByClients(configuredHarvestingClients);
+        Long numOfDatasets = harvestingClientService.getNumberOfHarvestedDatasetsByAllClients();
 
         if (numOfDatasets != null && numOfDatasets > 0L) {
             return infoLabel + numOfDatasets + " harvested datasets";

src/main/java/edu/harvard/iq/dataverse/harvest/client/ClientHarvestRun.java

@@ -40,12 +40,13 @@ public void setId(Long id) {
         this.id = id;
     }
 
-    public enum RunResultType { SUCCESS, FAILURE, INPROGRESS };
+    public enum RunResultType { SUCCESS, FAILURE, INPROGRESS, INTERRUPTED };
 
     private static String RESULT_LABEL_SUCCESS = "SUCCESS";
     private static String RESULT_LABEL_FAILURE = "FAILED";
     private static String RESULT_LABEL_INPROGRESS = "IN PROGRESS";
     private static String RESULT_DELETE_IN_PROGRESS = "DELETE IN PROGRESS";
+    private static String RESULT_LABEL_INTERRUPTED = "INTERRUPTED";
 
     @ManyToOne
     @JoinColumn(nullable = false)
@@ -76,6 +77,8 @@ public String getResultLabel() {
             return RESULT_LABEL_FAILURE;
         } else if (isInProgress()) {
             return RESULT_LABEL_INPROGRESS;
+        } else if (isInterrupted()) {
+            return RESULT_LABEL_INTERRUPTED;
         }
         return null;
     }
@@ -84,8 +87,8 @@ public String getDetailedResultLabel() {
         if (harvestingClient != null && harvestingClient.isDeleteInProgress()) {
             return RESULT_DELETE_IN_PROGRESS;
         }
-        if (isSuccess()) {
-            String resultLabel = RESULT_LABEL_SUCCESS;
+        if (isSuccess() || isInterrupted()) {
+            String resultLabel = getResultLabel();
 
             resultLabel = resultLabel.concat("; "+harvestedDatasetCount+" harvested, ");
             resultLabel = resultLabel.concat(deletedDatasetCount+" deleted, ");
@@ -128,6 +131,14 @@ public void setInProgress() {
         harvestResult = RunResultType.INPROGRESS;
     }
 
+    public boolean isInterrupted() {
+        return RunResultType.INTERRUPTED == harvestResult;
+    }
+
+    public void setInterrupted() {
+        harvestResult = RunResultType.INTERRUPTED;
+    }
+
     // Time of this harvest attempt:
     @Temporal(value = TemporalType.TIMESTAMP)
     private Date startTime;

src/main/java/edu/harvard/iq/dataverse/harvest/client/HarvesterServiceBean.java

@@ -48,6 +48,9 @@
 import java.net.http.HttpClient;
 import java.net.http.HttpRequest;
 import java.net.http.HttpResponse;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.nio.file.Path;
 import javax.persistence.EntityManager;
 import javax.persistence.PersistenceContext;
 
@@ -85,6 +88,7 @@ public class HarvesterServiceBean {
     public static final String HARVEST_RESULT_FAILED="failed";
     public static final String DATAVERSE_PROPRIETARY_METADATA_FORMAT="dataverse_json";
     public static final String DATAVERSE_PROPRIETARY_METADATA_API="/api/datasets/export?exporter="+DATAVERSE_PROPRIETARY_METADATA_FORMAT+"&persistentId=";
+    public static final String DATAVERSE_HARVEST_STOP_FILE="../logs/stopharvest_";
 
     public HarvesterServiceBean() {
 
@@ -130,7 +134,7 @@ public List<HarvestTimerInfo> getHarvestTimers() {
     }
 
     /**
-     * Run a harvest for an individual harvesting Dataverse
+     * Run a harvest for an individual harvesting client
      * @param dataverseRequest
      * @param harvestingClientId
      * @throws IOException
@@ -141,12 +145,9 @@ public void doHarvest(DataverseRequest dataverseRequest, Long harvestingClientId
         if (harvestingClientConfig == null) {
             throw new IOException("No such harvesting client: id="+harvestingClientId);
         }
-
-        Dataverse harvestingDataverse = harvestingClientConfig.getDataverse();
-
-        MutableBoolean harvestErrorOccurred = new MutableBoolean(false);
+
         String logTimestamp = logFormatter.format(new Date());
-        Logger hdLogger = Logger.getLogger("edu.harvard.iq.dataverse.harvest.client.HarvesterServiceBean." + harvestingDataverse.getAlias() + logTimestamp);
+        Logger hdLogger = Logger.getLogger("edu.harvard.iq.dataverse.harvest.client.HarvesterServiceBean." + harvestingClientConfig.getName() + logTimestamp);
         String logFileName = "../logs" + File.separator + "harvest_" + harvestingClientConfig.getName() + "_" + logTimestamp + ".log";
         FileHandler fileHandler = new FileHandler(logFileName);
         hdLogger.setUseParentHandlers(false);
@@ -155,29 +156,23 @@ public void doHarvest(DataverseRequest dataverseRequest, Long harvestingClientId
         PrintWriter importCleanupLog = new PrintWriter(new FileWriter( "../logs/harvest_cleanup_" + harvestingClientConfig.getName() + "_" + logTimestamp+".txt"));
 
 
-        List<Long> harvestedDatasetIds = null;
-
-        List<Long> harvestedDatasetIdsThisBatch = new ArrayList<Long>();
-
-        List<String> failedIdentifiers = new ArrayList<String>();
-        List<String> deletedIdentifiers = new ArrayList<String>();
+        List<Long> harvestedDatasetIds = new ArrayList<>();
+        List<String> failedIdentifiers = new ArrayList<>();
+        List<String> deletedIdentifiers = new ArrayList<>();
 
         Date harvestStartTime = new Date();
 
         try {
-            boolean harvestingNow = harvestingClientConfig.isHarvestingNow();
-
-            if (harvestingNow) {
-                harvestErrorOccurred.setValue(true);
-                hdLogger.log(Level.SEVERE, "Cannot begin harvesting, Dataverse " + harvestingDataverse.getName() + " is currently being harvested.");
+            if (harvestingClientConfig.isHarvestingNow()) {
+                hdLogger.log(Level.SEVERE, "Cannot start harvest, client " + harvestingClientConfig.getName() + " is already harvesting.");
 
             } else {
                 harvestingClientService.resetHarvestInProgress(harvestingClientId);
                 harvestingClientService.setHarvestInProgress(harvestingClientId, harvestStartTime);
 
 
                 if (harvestingClientConfig.isOai()) {
-                    harvestedDatasetIds = harvestOAI(dataverseRequest, harvestingClientConfig, hdLogger, importCleanupLog, harvestErrorOccurred, failedIdentifiers, deletedIdentifiers, harvestedDatasetIdsThisBatch);
+                    harvestOAI(dataverseRequest, harvestingClientConfig, hdLogger, importCleanupLog, failedIdentifiers, deletedIdentifiers, harvestedDatasetIds);
 
                 } else {
                     throw new IOException("Unsupported harvest type");
@@ -187,18 +182,17 @@ public void doHarvest(DataverseRequest dataverseRequest, Long harvestingClientId
                hdLogger.log(Level.INFO, "Datasets created/updated: " + harvestedDatasetIds.size() + ", datasets deleted: " + deletedIdentifiers.size() + ", datasets failed: " + failedIdentifiers.size());
 
             }
+        } catch (StopHarvestException she) {
+            hdLogger.log(Level.INFO, "HARVEST INTERRUPTED BY EXTERNAL REQUEST");
+            harvestingClientService.setPartiallyCompleted(harvestingClientId, new Date(), harvestedDatasetIds.size(), failedIdentifiers.size(), deletedIdentifiers.size());
         } catch (Throwable e) {
-            harvestErrorOccurred.setValue(true);
+            // Any other exception should be treated as a complete failure
             String message = "Exception processing harvest, server= " + harvestingClientConfig.getHarvestingUrl() + ",format=" + harvestingClientConfig.getMetadataPrefix() + " " + e.getClass().getName() + " " + e.getMessage();
             hdLogger.log(Level.SEVERE, message);
             logException(e, hdLogger);
             hdLogger.log(Level.INFO, "HARVEST NOT COMPLETED DUE TO UNEXPECTED ERROR.");
-            // TODO: 
-            // even though this harvesting run failed, we may have had successfully 
-            // processed some number of datasets, by the time the exception was thrown. 
-            // We should record that number too. And the number of the datasets that
-            // had failed, that we may have counted.  -- L.A. 4.4
-            harvestingClientService.setHarvestFailure(harvestingClientId, new Date());
+
+            harvestingClientService.setHarvestFailure(harvestingClientId, new Date(), harvestedDatasetIds.size(), failedIdentifiers.size(), deletedIdentifiers.size());
 
         } finally {
             harvestingClientService.resetHarvestInProgress(harvestingClientId);
@@ -215,12 +209,11 @@ public void doHarvest(DataverseRequest dataverseRequest, Long harvestingClientId
      * @param harvestErrorOccurred  have we encountered any errors during harvest?
      * @param failedIdentifiers     Study Identifiers for failed "GetRecord" requests
      */
-    private List<Long> harvestOAI(DataverseRequest dataverseRequest, HarvestingClient harvestingClient, Logger hdLogger, PrintWriter importCleanupLog, MutableBoolean harvestErrorOccurred, List<String> failedIdentifiers, List<String> deletedIdentifiers, List<Long> harvestedDatasetIdsThisBatch)
-            throws IOException, ParserConfigurationException, SAXException, TransformerException {
+    private void harvestOAI(DataverseRequest dataverseRequest, HarvestingClient harvestingClient, Logger hdLogger, PrintWriter importCleanupLog, List<String> failedIdentifiers, List<String> deletedIdentifiers, List<Long> harvestedDatasetIds)
+            throws IOException, ParserConfigurationException, SAXException, TransformerException, StopHarvestException {
 
         logBeginOaiHarvest(hdLogger, harvestingClient);
 
-        List<Long> harvestedDatasetIds = new ArrayList<Long>();
         OaiHandler oaiHandler;
         HttpClient httpClient = null; 
 
@@ -243,6 +236,10 @@ private List<Long> harvestOAI(DataverseRequest dataverseRequest, HarvestingClien
 
         try {
             for (Iterator<Header> idIter = oaiHandler.runListIdentifiers(); idIter.hasNext();) {
+                // Before each iteration, check if this harvesting job needs to be aborted:
+                if (checkIfStoppingJob(harvestingClient)) {
+                    throw new StopHarvestException("Harvesting stopped by external request");
+                }
 
                 Header h = idIter.next();
                 String identifier = h.getIdentifier();
@@ -265,18 +262,11 @@ private List<Long> harvestOAI(DataverseRequest dataverseRequest, HarvestingClien
 
                 if (datasetId != null) {
                     harvestedDatasetIds.add(datasetId);
-
-                    if ( harvestedDatasetIdsThisBatch == null ) {
-                        harvestedDatasetIdsThisBatch = new ArrayList<Long>();
-                    }
-                    harvestedDatasetIdsThisBatch.add(datasetId);
-
                 }
 
                 if (getRecordErrorOccurred.booleanValue() == true) {
                     failedIdentifiers.add(identifier);
-                    harvestErrorOccurred.setValue(true);
-                    //temporary:
+                    //can be uncommented out for testing failure handling:
                     //throw new IOException("Exception occured, stopping harvest");
                 }
             }
@@ -286,8 +276,6 @@ private List<Long> harvestOAI(DataverseRequest dataverseRequest, HarvestingClien
 
         logCompletedOaiHarvest(hdLogger, harvestingClient);
 
-        return harvestedDatasetIds;
-
     }    
 
     private Long processRecord(DataverseRequest dataverseRequest, Logger hdLogger, PrintWriter importCleanupLog, OaiHandler oaiHandler, String identifier, MutableBoolean recordErrorOccurred, List<String> deletedIdentifiers, Date dateStamp, HttpClient httpClient) {
@@ -303,7 +291,7 @@ private Long processRecord(DataverseRequest dataverseRequest, Logger hdLogger, P
                 // Make direct call to obtain the proprietary Dataverse metadata
                 // in JSON from the remote Dataverse server:
                 String metadataApiUrl = oaiHandler.getProprietaryDataverseMetadataURL(identifier);
-                logger.info("calling "+metadataApiUrl);
+                logger.fine("calling "+metadataApiUrl);
                 tempFile = retrieveProprietaryDataverseMetadata(httpClient, metadataApiUrl);
 
             } else {
@@ -410,6 +398,26 @@ private void deleteHarvestedDatasetIfExists(String persistentIdentifier, Dataver
         }
         hdLogger.info("No dataset found for " + persistentIdentifier + ", skipping delete. ");
     }
+
+    private boolean checkIfStoppingJob(HarvestingClient harvestingClient) {
+        Long pid = ProcessHandle.current().pid();
+        String stopFileName = DATAVERSE_HARVEST_STOP_FILE + harvestingClient.getName() + "." + pid; 
+        Path stopFilePath = Paths.get(stopFileName);
+
+        if (Files.exists(stopFilePath)) {
+            // Now that we know that the file is there, let's (try to) delete it,
+            // so that the harvest can be re-run. 
+            try {
+                Files.delete(stopFilePath);
+            } catch (IOException ioex) {
+                // No need to treat this is a big deal (could be a permission, etc.)
+                logger.warning("Failed to delete the flag file "+stopFileName + "; check permissions and delete manually.");
+            }
+            return true;
+        }
+
+        return false;                 
+    }
 
     private void logBeginOaiHarvest(Logger hdLogger, HarvestingClient harvestingClient) {
         hdLogger.log(Level.INFO, "BEGIN HARVEST, oaiUrl="