diff --git a/doc/release-notes/7940-stop-harvest-in-progress b/doc/release-notes/7940-stop-harvest-in-progress new file mode 100644 index 00000000000..cb27a900f15 --- /dev/null +++ b/doc/release-notes/7940-stop-harvest-in-progress @@ -0,0 +1,4 @@ +## Mechanism added for stopping a harvest in progress + +It is now possible for an admin to stop a long-running harvesting job. See [Harvesting Clients](https://guides.dataverse.org/en/latest/admin/harvestclients.html) guide for more information. + diff --git a/doc/sphinx-guides/source/admin/harvestclients.rst b/doc/sphinx-guides/source/admin/harvestclients.rst index c655d5af763..e94a6aa1730 100644 --- a/doc/sphinx-guides/source/admin/harvestclients.rst +++ b/doc/sphinx-guides/source/admin/harvestclients.rst @@ -21,6 +21,21 @@ Clients are managed on the "Harvesting Clients" page accessible via the :doc:`da The process of creating a new, or editing an existing client, is largely self-explanatory. It is split into logical steps, in a way that allows the user to go back and correct the entries made earlier. The process is interactive and guidance text is provided. For example, the user is required to enter the URL of the remote OAI server. When they click *Next*, the application will try to establish a connection to the server in order to verify that it is working, and to obtain the information about the sets of metadata records and the metadata formats it supports. The choices offered to the user on the next page will be based on this extra information. If the application fails to establish a connection to the remote archive at the address specified, or if an invalid response is received, the user is given an opportunity to check and correct the URL they entered. +How to Stop a Harvesting Run in Progress +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Some harvesting jobs, especially the initial full harvest of a very large set - such as the default set of public datasets at IQSS - can take many hours. In case it is necessary to terminate such a long-running job, the following mechanism is provided (note that it is only available to a sysadmin with shell access to the application server): Create an empty file in the domain logs directory with the following name: ``stopharvest_.``, where ```` is the nickname of the harvesting client and ```` is the process id of the Application Server (Payara). This flag file needs to be owned by the same user that's running Payara, so that the application can remove it after stopping the job in progress. + +For example: + +.. code-block:: bash + + sudo touch /usr/local/payara5/glassfish/domains/domain1/logs/stopharvest_bigarchive.70916 + sudo chown dataverse /usr/local/payara5/glassfish/domains/domain1/logs/stopharvest_bigarchive.70916 + +Note: If the application server is stopped and restarted, any running harvesting jobs will be killed but may remain marked as in progress in the database. We thus recommend using the mechanism here to stop ongoing harvests prior to a server restart. + + What if a Run Fails? ~~~~~~~~~~~~~~~~~~~~ diff --git a/src/main/java/edu/harvard/iq/dataverse/DashboardPage.java b/src/main/java/edu/harvard/iq/dataverse/DashboardPage.java index 5b6cdd23775..99c7951c96e 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DashboardPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/DashboardPage.java @@ -97,12 +97,8 @@ public int getNumberOfConfiguredHarvestClients() { } public long getNumberOfHarvestedDatasets() { - List configuredHarvestingClients = harvestingClientService.getAllHarvestingClients(); - if (configuredHarvestingClients == null || configuredHarvestingClients.isEmpty()) { - return 0L; - } - Long numOfDatasets = harvestingClientService.getNumberOfHarvestedDatasetByClients(configuredHarvestingClients); + Long numOfDatasets = harvestingClientService.getNumberOfHarvestedDatasetsByAllClients(); if (numOfDatasets != null && numOfDatasets > 0L) { return numOfDatasets; @@ -142,7 +138,7 @@ public String getHarvestClientsInfoLabel() { infoLabel = configuredHarvestingClients.size() + " harvesting clients configured; "; } - Long numOfDatasets = harvestingClientService.getNumberOfHarvestedDatasetByClients(configuredHarvestingClients); + Long numOfDatasets = harvestingClientService.getNumberOfHarvestedDatasetsByAllClients(); if (numOfDatasets != null && numOfDatasets > 0L) { return infoLabel + numOfDatasets + " harvested datasets"; diff --git a/src/main/java/edu/harvard/iq/dataverse/harvest/client/ClientHarvestRun.java b/src/main/java/edu/harvard/iq/dataverse/harvest/client/ClientHarvestRun.java index 0dc94f835e9..50d06807a13 100644 --- a/src/main/java/edu/harvard/iq/dataverse/harvest/client/ClientHarvestRun.java +++ b/src/main/java/edu/harvard/iq/dataverse/harvest/client/ClientHarvestRun.java @@ -40,12 +40,13 @@ public void setId(Long id) { this.id = id; } - public enum RunResultType { SUCCESS, FAILURE, INPROGRESS }; + public enum RunResultType { SUCCESS, FAILURE, INPROGRESS, INTERRUPTED }; private static String RESULT_LABEL_SUCCESS = "SUCCESS"; private static String RESULT_LABEL_FAILURE = "FAILED"; private static String RESULT_LABEL_INPROGRESS = "IN PROGRESS"; private static String RESULT_DELETE_IN_PROGRESS = "DELETE IN PROGRESS"; + private static String RESULT_LABEL_INTERRUPTED = "INTERRUPTED"; @ManyToOne @JoinColumn(nullable = false) @@ -76,6 +77,8 @@ public String getResultLabel() { return RESULT_LABEL_FAILURE; } else if (isInProgress()) { return RESULT_LABEL_INPROGRESS; + } else if (isInterrupted()) { + return RESULT_LABEL_INTERRUPTED; } return null; } @@ -84,8 +87,8 @@ public String getDetailedResultLabel() { if (harvestingClient != null && harvestingClient.isDeleteInProgress()) { return RESULT_DELETE_IN_PROGRESS; } - if (isSuccess()) { - String resultLabel = RESULT_LABEL_SUCCESS; + if (isSuccess() || isInterrupted()) { + String resultLabel = getResultLabel(); resultLabel = resultLabel.concat("; "+harvestedDatasetCount+" harvested, "); resultLabel = resultLabel.concat(deletedDatasetCount+" deleted, "); @@ -128,6 +131,14 @@ public void setInProgress() { harvestResult = RunResultType.INPROGRESS; } + public boolean isInterrupted() { + return RunResultType.INTERRUPTED == harvestResult; + } + + public void setInterrupted() { + harvestResult = RunResultType.INTERRUPTED; + } + // Time of this harvest attempt: @Temporal(value = TemporalType.TIMESTAMP) private Date startTime; diff --git a/src/main/java/edu/harvard/iq/dataverse/harvest/client/HarvesterServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/harvest/client/HarvesterServiceBean.java index e7156dfe9aa..058a20451d6 100644 --- a/src/main/java/edu/harvard/iq/dataverse/harvest/client/HarvesterServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/harvest/client/HarvesterServiceBean.java @@ -48,6 +48,9 @@ import java.net.http.HttpClient; import java.net.http.HttpRequest; import java.net.http.HttpResponse; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.nio.file.Path; import javax.persistence.EntityManager; import javax.persistence.PersistenceContext; @@ -85,6 +88,7 @@ public class HarvesterServiceBean { public static final String HARVEST_RESULT_FAILED="failed"; public static final String DATAVERSE_PROPRIETARY_METADATA_FORMAT="dataverse_json"; public static final String DATAVERSE_PROPRIETARY_METADATA_API="/api/datasets/export?exporter="+DATAVERSE_PROPRIETARY_METADATA_FORMAT+"&persistentId="; + public static final String DATAVERSE_HARVEST_STOP_FILE="../logs/stopharvest_"; public HarvesterServiceBean() { @@ -130,7 +134,7 @@ public List getHarvestTimers() { } /** - * Run a harvest for an individual harvesting Dataverse + * Run a harvest for an individual harvesting client * @param dataverseRequest * @param harvestingClientId * @throws IOException @@ -141,12 +145,9 @@ public void doHarvest(DataverseRequest dataverseRequest, Long harvestingClientId if (harvestingClientConfig == null) { throw new IOException("No such harvesting client: id="+harvestingClientId); } - - Dataverse harvestingDataverse = harvestingClientConfig.getDataverse(); - - MutableBoolean harvestErrorOccurred = new MutableBoolean(false); + String logTimestamp = logFormatter.format(new Date()); - Logger hdLogger = Logger.getLogger("edu.harvard.iq.dataverse.harvest.client.HarvesterServiceBean." + harvestingDataverse.getAlias() + logTimestamp); + Logger hdLogger = Logger.getLogger("edu.harvard.iq.dataverse.harvest.client.HarvesterServiceBean." + harvestingClientConfig.getName() + logTimestamp); String logFileName = "../logs" + File.separator + "harvest_" + harvestingClientConfig.getName() + "_" + logTimestamp + ".log"; FileHandler fileHandler = new FileHandler(logFileName); hdLogger.setUseParentHandlers(false); @@ -155,21 +156,15 @@ public void doHarvest(DataverseRequest dataverseRequest, Long harvestingClientId PrintWriter importCleanupLog = new PrintWriter(new FileWriter( "../logs/harvest_cleanup_" + harvestingClientConfig.getName() + "_" + logTimestamp+".txt")); - List harvestedDatasetIds = null; - - List harvestedDatasetIdsThisBatch = new ArrayList(); - - List failedIdentifiers = new ArrayList(); - List deletedIdentifiers = new ArrayList(); + List harvestedDatasetIds = new ArrayList<>(); + List failedIdentifiers = new ArrayList<>(); + List deletedIdentifiers = new ArrayList<>(); Date harvestStartTime = new Date(); try { - boolean harvestingNow = harvestingClientConfig.isHarvestingNow(); - - if (harvestingNow) { - harvestErrorOccurred.setValue(true); - hdLogger.log(Level.SEVERE, "Cannot begin harvesting, Dataverse " + harvestingDataverse.getName() + " is currently being harvested."); + if (harvestingClientConfig.isHarvestingNow()) { + hdLogger.log(Level.SEVERE, "Cannot start harvest, client " + harvestingClientConfig.getName() + " is already harvesting."); } else { harvestingClientService.resetHarvestInProgress(harvestingClientId); @@ -177,7 +172,7 @@ public void doHarvest(DataverseRequest dataverseRequest, Long harvestingClientId if (harvestingClientConfig.isOai()) { - harvestedDatasetIds = harvestOAI(dataverseRequest, harvestingClientConfig, hdLogger, importCleanupLog, harvestErrorOccurred, failedIdentifiers, deletedIdentifiers, harvestedDatasetIdsThisBatch); + harvestOAI(dataverseRequest, harvestingClientConfig, hdLogger, importCleanupLog, failedIdentifiers, deletedIdentifiers, harvestedDatasetIds); } else { throw new IOException("Unsupported harvest type"); @@ -187,18 +182,17 @@ public void doHarvest(DataverseRequest dataverseRequest, Long harvestingClientId hdLogger.log(Level.INFO, "Datasets created/updated: " + harvestedDatasetIds.size() + ", datasets deleted: " + deletedIdentifiers.size() + ", datasets failed: " + failedIdentifiers.size()); } + } catch (StopHarvestException she) { + hdLogger.log(Level.INFO, "HARVEST INTERRUPTED BY EXTERNAL REQUEST"); + harvestingClientService.setPartiallyCompleted(harvestingClientId, new Date(), harvestedDatasetIds.size(), failedIdentifiers.size(), deletedIdentifiers.size()); } catch (Throwable e) { - harvestErrorOccurred.setValue(true); + // Any other exception should be treated as a complete failure String message = "Exception processing harvest, server= " + harvestingClientConfig.getHarvestingUrl() + ",format=" + harvestingClientConfig.getMetadataPrefix() + " " + e.getClass().getName() + " " + e.getMessage(); hdLogger.log(Level.SEVERE, message); logException(e, hdLogger); hdLogger.log(Level.INFO, "HARVEST NOT COMPLETED DUE TO UNEXPECTED ERROR."); - // TODO: - // even though this harvesting run failed, we may have had successfully - // processed some number of datasets, by the time the exception was thrown. - // We should record that number too. And the number of the datasets that - // had failed, that we may have counted. -- L.A. 4.4 - harvestingClientService.setHarvestFailure(harvestingClientId, new Date()); + + harvestingClientService.setHarvestFailure(harvestingClientId, new Date(), harvestedDatasetIds.size(), failedIdentifiers.size(), deletedIdentifiers.size()); } finally { harvestingClientService.resetHarvestInProgress(harvestingClientId); @@ -215,12 +209,11 @@ public void doHarvest(DataverseRequest dataverseRequest, Long harvestingClientId * @param harvestErrorOccurred have we encountered any errors during harvest? * @param failedIdentifiers Study Identifiers for failed "GetRecord" requests */ - private List harvestOAI(DataverseRequest dataverseRequest, HarvestingClient harvestingClient, Logger hdLogger, PrintWriter importCleanupLog, MutableBoolean harvestErrorOccurred, List failedIdentifiers, List deletedIdentifiers, List harvestedDatasetIdsThisBatch) - throws IOException, ParserConfigurationException, SAXException, TransformerException { + private void harvestOAI(DataverseRequest dataverseRequest, HarvestingClient harvestingClient, Logger hdLogger, PrintWriter importCleanupLog, List failedIdentifiers, List deletedIdentifiers, List harvestedDatasetIds) + throws IOException, ParserConfigurationException, SAXException, TransformerException, StopHarvestException { logBeginOaiHarvest(hdLogger, harvestingClient); - List harvestedDatasetIds = new ArrayList(); OaiHandler oaiHandler; HttpClient httpClient = null; @@ -243,6 +236,10 @@ private List harvestOAI(DataverseRequest dataverseRequest, HarvestingClien try { for (Iterator
idIter = oaiHandler.runListIdentifiers(); idIter.hasNext();) { + // Before each iteration, check if this harvesting job needs to be aborted: + if (checkIfStoppingJob(harvestingClient)) { + throw new StopHarvestException("Harvesting stopped by external request"); + } Header h = idIter.next(); String identifier = h.getIdentifier(); @@ -265,18 +262,11 @@ private List harvestOAI(DataverseRequest dataverseRequest, HarvestingClien if (datasetId != null) { harvestedDatasetIds.add(datasetId); - - if ( harvestedDatasetIdsThisBatch == null ) { - harvestedDatasetIdsThisBatch = new ArrayList(); - } - harvestedDatasetIdsThisBatch.add(datasetId); - } if (getRecordErrorOccurred.booleanValue() == true) { failedIdentifiers.add(identifier); - harvestErrorOccurred.setValue(true); - //temporary: + //can be uncommented out for testing failure handling: //throw new IOException("Exception occured, stopping harvest"); } } @@ -286,8 +276,6 @@ private List harvestOAI(DataverseRequest dataverseRequest, HarvestingClien logCompletedOaiHarvest(hdLogger, harvestingClient); - return harvestedDatasetIds; - } private Long processRecord(DataverseRequest dataverseRequest, Logger hdLogger, PrintWriter importCleanupLog, OaiHandler oaiHandler, String identifier, MutableBoolean recordErrorOccurred, List deletedIdentifiers, Date dateStamp, HttpClient httpClient) { @@ -303,7 +291,7 @@ private Long processRecord(DataverseRequest dataverseRequest, Logger hdLogger, P // Make direct call to obtain the proprietary Dataverse metadata // in JSON from the remote Dataverse server: String metadataApiUrl = oaiHandler.getProprietaryDataverseMetadataURL(identifier); - logger.info("calling "+metadataApiUrl); + logger.fine("calling "+metadataApiUrl); tempFile = retrieveProprietaryDataverseMetadata(httpClient, metadataApiUrl); } else { @@ -410,6 +398,26 @@ private void deleteHarvestedDatasetIfExists(String persistentIdentifier, Dataver } hdLogger.info("No dataset found for " + persistentIdentifier + ", skipping delete. "); } + + private boolean checkIfStoppingJob(HarvestingClient harvestingClient) { + Long pid = ProcessHandle.current().pid(); + String stopFileName = DATAVERSE_HARVEST_STOP_FILE + harvestingClient.getName() + "." + pid; + Path stopFilePath = Paths.get(stopFileName); + + if (Files.exists(stopFilePath)) { + // Now that we know that the file is there, let's (try to) delete it, + // so that the harvest can be re-run. + try { + Files.delete(stopFilePath); + } catch (IOException ioex) { + // No need to treat this is a big deal (could be a permission, etc.) + logger.warning("Failed to delete the flag file "+stopFileName + "; check permissions and delete manually."); + } + return true; + } + + return false; + } private void logBeginOaiHarvest(Logger hdLogger, HarvestingClient harvestingClient) { hdLogger.log(Level.INFO, "BEGIN HARVEST, oaiUrl=" diff --git a/src/main/java/edu/harvard/iq/dataverse/harvest/client/HarvestingClientServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/harvest/client/HarvestingClientServiceBean.java index 0af73550190..13cc44ce919 100644 --- a/src/main/java/edu/harvard/iq/dataverse/harvest/client/HarvestingClientServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/harvest/client/HarvestingClientServiceBean.java @@ -167,28 +167,20 @@ public void deleteClient(Long clientId) { @TransactionAttribute(TransactionAttributeType.REQUIRES_NEW) public void setHarvestSuccess(Long hcId, Date currentTime, int harvestedCount, int failedCount, int deletedCount) { - HarvestingClient harvestingClient = em.find(HarvestingClient.class, hcId); - if (harvestingClient == null) { - return; - } - em.refresh(harvestingClient); - - ClientHarvestRun currentRun = harvestingClient.getLastRun(); - - if (currentRun != null && currentRun.isInProgress()) { - // TODO: what if there's no current run in progress? should we just - // give up quietly, or should we make a noise of some kind? -- L.A. 4.4 - - currentRun.setSuccess(); - currentRun.setFinishTime(currentTime); - currentRun.setHarvestedDatasetCount(new Long(harvestedCount)); - currentRun.setFailedDatasetCount(new Long(failedCount)); - currentRun.setDeletedDatasetCount(new Long(deletedCount)); - } + recordHarvestJobStatus(hcId, currentTime, harvestedCount, failedCount, deletedCount, ClientHarvestRun.RunResultType.SUCCESS); } @TransactionAttribute(TransactionAttributeType.REQUIRES_NEW) - public void setHarvestFailure(Long hcId, Date currentTime) { + public void setHarvestFailure(Long hcId, Date currentTime, int harvestedCount, int failedCount, int deletedCount) { + recordHarvestJobStatus(hcId, currentTime, harvestedCount, failedCount, deletedCount, ClientHarvestRun.RunResultType.FAILURE); + } + + @TransactionAttribute(TransactionAttributeType.REQUIRES_NEW) + public void setPartiallyCompleted(Long hcId, Date finishTime, int harvestedCount, int failedCount, int deletedCount) { + recordHarvestJobStatus(hcId, finishTime, harvestedCount, failedCount, deletedCount, ClientHarvestRun.RunResultType.INTERRUPTED); + } + + public void recordHarvestJobStatus(Long hcId, Date finishTime, int harvestedCount, int failedCount, int deletedCount, ClientHarvestRun.RunResultType result) { HarvestingClient harvestingClient = em.find(HarvestingClient.class, hcId); if (harvestingClient == null) { return; @@ -198,28 +190,40 @@ public void setHarvestFailure(Long hcId, Date currentTime) { ClientHarvestRun currentRun = harvestingClient.getLastRun(); if (currentRun != null && currentRun.isInProgress()) { - // TODO: what if there's no current run in progress? should we just - // give up quietly, or should we make a noise of some kind? -- L.A. 4.4 - currentRun.setFailed(); - currentRun.setFinishTime(currentTime); + currentRun.setResult(result); + currentRun.setFinishTime(finishTime); + currentRun.setHarvestedDatasetCount(Long.valueOf(harvestedCount)); + currentRun.setFailedDatasetCount(Long.valueOf(failedCount)); + currentRun.setDeletedDatasetCount(Long.valueOf(deletedCount)); } - } + } + + public Long getNumberOfHarvestedDatasetsByAllClients() { + try { + return (Long) em.createNativeQuery("SELECT count(d.id) FROM dataset d " + + " WHERE d.harvestingclient_id IS NOT NULL").getSingleResult(); + + } catch (Exception ex) { + logger.info("Warning: exception looking up the total number of harvested datasets: " + ex.getMessage()); + return 0L; + } + } public Long getNumberOfHarvestedDatasetByClients(List clients) { - String dvs = null; + String clientIds = null; for (HarvestingClient client: clients) { - if (dvs == null) { - dvs = client.getDataverse().getId().toString(); + if (clientIds == null) { + clientIds = client.getId().toString(); } else { - dvs = dvs.concat(","+client.getDataverse().getId().toString()); + clientIds = clientIds.concat(","+client.getId().toString()); } } try { - return (Long) em.createNativeQuery("SELECT count(d.id) FROM dataset d, " - + " dvobject o WHERE d.id = o.id AND o.owner_id in (" - + dvs + ")").getSingleResult(); + return (Long) em.createNativeQuery("SELECT count(d.id) FROM dataset d " + + " WHERE d.harvestingclient_id in (" + + clientIds + ")").getSingleResult(); } catch (Exception ex) { logger.info("Warning: exception trying to count harvested datasets by clients: " + ex.getMessage()); diff --git a/src/main/java/edu/harvard/iq/dataverse/harvest/client/StopHarvestException.java b/src/main/java/edu/harvard/iq/dataverse/harvest/client/StopHarvestException.java new file mode 100644 index 00000000000..dffa2dd0385 --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/harvest/client/StopHarvestException.java @@ -0,0 +1,17 @@ +package edu.harvard.iq.dataverse.harvest.client; + +/** + * + * @author landreev + */ + +public class StopHarvestException extends Exception { + public StopHarvestException(String message) { + super(message); + } + + public StopHarvestException(String message, Throwable cause) { + super(message, cause); + } + +} diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties index b19e80020ba..f7b46c308f5 100644 --- a/src/main/java/propertyFiles/Bundle.properties +++ b/src/main/java/propertyFiles/Bundle.properties @@ -520,7 +520,7 @@ harvestclients.btn.add=Add Client harvestclients.tab.header.name=Nickname harvestclients.tab.header.url=URL harvestclients.tab.header.lastrun=Last Run -harvestclients.tab.header.lastresults=Last Results +harvestclients.tab.header.lastresults=Last Result harvestclients.tab.header.action=Actions harvestclients.tab.header.action.btn.run=Run Harvesting harvestclients.tab.header.action.btn.edit=Edit diff --git a/src/main/webapp/dashboard.xhtml b/src/main/webapp/dashboard.xhtml index c5b6a507a92..5a72b52937b 100644 --- a/src/main/webapp/dashboard.xhtml +++ b/src/main/webapp/dashboard.xhtml @@ -42,7 +42,7 @@ #{dashboardPage.numberOfHarvestedDatasets}

- +