diff --git a/bitrepository-audit-trail-service/src/main/java/org/bitrepository/audittrails/AuditTrailService.java b/bitrepository-audit-trail-service/src/main/java/org/bitrepository/audittrails/AuditTrailService.java index a4878e94e..0f4fe3057 100644 --- a/bitrepository-audit-trail-service/src/main/java/org/bitrepository/audittrails/AuditTrailService.java +++ b/bitrepository-audit-trail-service/src/main/java/org/bitrepository/audittrails/AuditTrailService.java @@ -25,9 +25,11 @@ package org.bitrepository.audittrails; import org.bitrepository.audittrails.collector.AuditTrailCollector; +import org.bitrepository.audittrails.preserver.AuditTrailPreserver; import org.bitrepository.audittrails.store.AuditEventIterator; import org.bitrepository.audittrails.store.AuditTrailStore; import org.bitrepository.audittrails.webservice.CollectorInfo; +import org.bitrepository.audittrails.webservice.PreservationInfo; import org.bitrepository.bitrepositoryelements.FileAction; import org.bitrepository.common.ArgumentValidator; import org.bitrepository.common.settings.Settings; @@ -51,20 +53,22 @@ public class AuditTrailService implements LifeCycledService { private final Logger log = LoggerFactory.getLogger(getClass()); private final AuditTrailStore store; private final AuditTrailCollector collector; + private final AuditTrailPreserver preserver; private final ContributorMediator mediator; private final Settings settings; /** + * Constructor for audit trail service. + * * @param store The store for the audit trail data. * @param collector The collector of new audit trail data. + * @param preserver The preserver for saving collected audit trail data to a collection. + * Provide non-null preserver for enabling preservation. * @param mediator The mediator for the communication of this contributor. * @param settings The AuditTrailService settings. */ - public AuditTrailService( - AuditTrailStore store, - AuditTrailCollector collector, - ContributorMediator mediator, - Settings settings) { + public AuditTrailService(AuditTrailStore store, AuditTrailCollector collector, AuditTrailPreserver preserver, + ContributorMediator mediator, Settings settings) { ArgumentValidator.checkNotNull(collector, "AuditTrailCollector collector"); ArgumentValidator.checkNotNull(store, "AuditTrailStore store"); ArgumentValidator.checkNotNull(mediator, "ContributorMediator mediator"); @@ -72,10 +76,23 @@ public AuditTrailService( this.store = store; this.collector = collector; + this.preserver = preserver; this.mediator = mediator; this.settings = settings; + } - mediator.start(); + /** + * Constructor for audit trail service with disabled preservation. + * + * See {@link #AuditTrailService(AuditTrailStore, AuditTrailCollector, AuditTrailPreserver, ContributorMediator, + * Settings)} for param descriptions. + */ + public AuditTrailService( + AuditTrailStore store, + AuditTrailCollector collector, + ContributorMediator mediator, + Settings settings) { + this(store, collector, null, mediator, settings); } /** @@ -89,7 +106,7 @@ public AuditTrailService( * @param reportingComponent Restrict the results to only be reported by this component * @param actor Restrict the results to only be events caused by this actor * @param action Restrict the results to only be about this type of action - * @param fingerprint the fingerprint + * @param fingerprint The fingerprint * @param operationID Restrict the results to only this operationID * @return an iterator to all AuditTrailEvents matching the criteria from the parameters */ @@ -128,6 +145,18 @@ public List getCollectorInfos() { return infos; } + /** + * Get preservation info if preservation of audit trails is enabled. + * + * @return PreservationInfo or null if not enabled. + */ + public PreservationInfo getPreservationInfo() { + if (preserver == null ) { + return null; + } + return preserver.getPreservationInfo(); + } + /** * Get the list of known contributors from the backend. * @@ -139,14 +168,21 @@ public List getContributors() { @Override public void start() { + if (preserver != null) { + preserver.start(); + } mediator.start(); } @Override public void shutdown() { collector.close(); + if (preserver != null) { + preserver.close(); + } store.close(); mediator.close(); + MessageBus messageBus = MessageBusManager.getMessageBus(); if (messageBus != null) { try { diff --git a/bitrepository-audit-trail-service/src/main/java/org/bitrepository/audittrails/AuditTrailServiceFactory.java b/bitrepository-audit-trail-service/src/main/java/org/bitrepository/audittrails/AuditTrailServiceFactory.java index fe9ad3db8..c2ef50221 100644 --- a/bitrepository-audit-trail-service/src/main/java/org/bitrepository/audittrails/AuditTrailServiceFactory.java +++ b/bitrepository-audit-trail-service/src/main/java/org/bitrepository/audittrails/AuditTrailServiceFactory.java @@ -116,16 +116,17 @@ public static synchronized AuditTrailService getAuditTrailService() { securityManager, serviceSettings.getID()); AuditTrailCollector collector = new AuditTrailCollector(settings, client, store, alarmDispatcher); - AuditTrailPreserver preserver; + if (serviceSettings.isSetAuditTrailPreservation()) { - preserver = new LocalAuditTrailPreserver( + log.info("Audit trail preservation enabled in configuration. Audit trail service will preserve trails."); + AuditTrailPreserver preserver = new LocalAuditTrailPreserver( settings, store, putClient, ProtocolComponentFactory.getInstance().getFileExchange(settings)); - preserver.start(); + auditTrailService = new AuditTrailService(store, collector, preserver, mediator, settings); } else { log.info("Audit trail preservation disabled, no configuration defined."); + auditTrailService = new AuditTrailService(store, collector, mediator, settings); } - - auditTrailService = new AuditTrailService(store, collector, mediator, settings); + auditTrailService.start(); } return auditTrailService; diff --git a/bitrepository-audit-trail-service/src/main/java/org/bitrepository/audittrails/collector/AuditTrailCollectionTimerTask.java b/bitrepository-audit-trail-service/src/main/java/org/bitrepository/audittrails/collector/AuditTrailCollectionTimerTask.java index 0d8494a7f..8800e3aad 100644 --- a/bitrepository-audit-trail-service/src/main/java/org/bitrepository/audittrails/collector/AuditTrailCollectionTimerTask.java +++ b/bitrepository-audit-trail-service/src/main/java/org/bitrepository/audittrails/collector/AuditTrailCollectionTimerTask.java @@ -21,6 +21,7 @@ */ package org.bitrepository.audittrails.collector; +import org.bitrepository.common.TimerTaskSchedule; import org.bitrepository.common.utils.SettingsUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -32,7 +33,7 @@ public class AuditTrailCollectionTimerTask extends TimerTask { private final Logger log = LoggerFactory.getLogger(getClass()); private final IncrementalCollector collector; - private final CollectionSchedule schedule; + private final TimerTaskSchedule schedule; /** * @param collector The collector doing the actual work. @@ -40,7 +41,7 @@ public class AuditTrailCollectionTimerTask extends TimerTask { * @param gracePeriod The period that should pass before the first scheduled collection */ public AuditTrailCollectionTimerTask(IncrementalCollector collector, long interval, int gracePeriod) { - this.schedule = new CollectionSchedule(interval, gracePeriod); + this.schedule = new TimerTaskSchedule(interval, gracePeriod); this.collector = collector; log.info("Scheduled next collection of audit trails for {}", schedule.getNextRun()); } diff --git a/bitrepository-audit-trail-service/src/main/java/org/bitrepository/audittrails/collector/AuditTrailCollector.java b/bitrepository-audit-trail-service/src/main/java/org/bitrepository/audittrails/collector/AuditTrailCollector.java index 76dc694e9..860ce34a9 100644 --- a/bitrepository-audit-trail-service/src/main/java/org/bitrepository/audittrails/collector/AuditTrailCollector.java +++ b/bitrepository-audit-trail-service/src/main/java/org/bitrepository/audittrails/collector/AuditTrailCollector.java @@ -103,7 +103,7 @@ public CollectorInfo getCollectorInfo(String collectionID) { info.setLastDuration("Collection has not finished yet"); } } else { - info.setLastStart("Audit trail collection have not started"); + info.setLastStart("Audit trail collection has not started"); info.setLastDuration("Not available"); } info.setNextStart(TimeUtils.shortDate(nextRun)); diff --git a/bitrepository-audit-trail-service/src/main/java/org/bitrepository/audittrails/collector/IncrementalCollector.java b/bitrepository-audit-trail-service/src/main/java/org/bitrepository/audittrails/collector/IncrementalCollector.java index e36a9115d..ef25d8c0e 100644 --- a/bitrepository-audit-trail-service/src/main/java/org/bitrepository/audittrails/collector/IncrementalCollector.java +++ b/bitrepository-audit-trail-service/src/main/java/org/bitrepository/audittrails/collector/IncrementalCollector.java @@ -66,12 +66,12 @@ public class IncrementalCollector { private long collectedAudits = 0; /** - * @param collectionID the collection ID + * @param collectionID The collection ID * @param clientID The clientID to use for the requests. * @param client The client to use for the operations. * @param store Where to persist the received results. - * @param maxNumberOfResults A optional limit on the number of audit trail events to request. If not set, {} - * @param alarmDispatcher the alarm dispatcher + * @param maxNumberOfResults An optional limit on the number of audit trail events to request. If not set, {} + * @param alarmDispatcher The alarm dispatcher */ public IncrementalCollector(String collectionID, String clientID, AuditTrailClient client, AuditTrailStore store, int maxNumberOfResults, AlarmDispatcher alarmDispatcher) { @@ -103,7 +103,6 @@ public long getNumberOfCollectedAudits() { /** * Setup and initiates the collection of audit trails through the client. - * Adds one to the sequence number to request only newer audit trails. * * @param contributors the collection of IDs of contributor */ @@ -121,7 +120,8 @@ public void performCollection(Collection contributors) { } /** - * Collect a page of audit trails from the active contributors + * Collect a page of audit trails from the active contributors. + * Adds 1 to the sequence number to only collect newer audit trails. * * @param contributors The contributors to collect from * @return Collection the contributors that have more audits to collect @@ -129,9 +129,9 @@ public void performCollection(Collection contributors) { private Collection collect(Collection contributors) { List queries = new ArrayList<>(); - for (String contributorId : contributors) { - long seq = store.largestSequenceNumber(contributorId, collectionID); - queries.add(new AuditTrailQuery(contributorId, seq + 1, null, maxNumberOfResults)); + for (String contributorID : contributors) { + long seqNum = store.largestSequenceNumber(contributorID, collectionID); + queries.add(new AuditTrailQuery(contributorID, seqNum + 1, null, maxNumberOfResults)); } log.debug("Collecting of AuditTrails for '{}' with ContributorQueries: {}", collectionID, queries); diff --git a/bitrepository-audit-trail-service/src/main/java/org/bitrepository/audittrails/preserver/AuditPacker.java b/bitrepository-audit-trail-service/src/main/java/org/bitrepository/audittrails/preserver/AuditPacker.java index 01edfa8a2..179f1dd4b 100644 --- a/bitrepository-audit-trail-service/src/main/java/org/bitrepository/audittrails/preserver/AuditPacker.java +++ b/bitrepository-audit-trail-service/src/main/java/org/bitrepository/audittrails/preserver/AuditPacker.java @@ -58,12 +58,14 @@ public class AuditPacker { /** * Map between the contributor id and the reached preservation sequence number. */ - private final Map seqReached = new HashMap<>(); + private final Map seqNumsReached = new HashMap<>(); /** * Whether the output stream should be appended to the file. */ private static final boolean APPEND = true; + private long packedAuditCount = 0; + /** * Constructor. * @@ -77,7 +79,7 @@ public AuditPacker(AuditTrailStore store, AuditTrailPreservation settings, Strin this.directory = FileUtils.retrieveDirectory(settings.getAuditTrailPreservationTemporaryDirectory()); this.contributors.addAll(SettingsUtils.getAuditContributorsForCollection(collectionID)); - initialiseReachedSequenceNumbers(); + initializeReachedSequenceNumbers(); } /** @@ -86,16 +88,16 @@ public AuditPacker(AuditTrailStore store, AuditTrailPreservation settings, Strin * @return A mapping between the contributor ids and their preservation sequence numbers. */ public Map getSequenceNumbersReached() { - return new HashMap<>(seqReached); + return new HashMap<>(seqNumsReached); } /** * Retrieves the preservation sequence number for each contributor and inserts it into the map. */ - private void initialiseReachedSequenceNumbers() { + private void initializeReachedSequenceNumbers() { for (String contributor : contributors) { - Long seq = store.getPreservationSequenceNumber(contributor, collectionID); - seqReached.put(contributor, seq); + Long seqNum = store.getPreservationSequenceNumber(contributor, collectionID); + seqNumsReached.put(contributor, seqNum); } } @@ -107,6 +109,7 @@ private void initialiseReachedSequenceNumbers() { * @return A compressed file with all the audit trails. */ public synchronized File createNewPackage() { + resetPackedAuditCount(); File container = new File(directory, collectionID + "-audit-trails-" + System.currentTimeMillis()); try { if (container.createNewFile()) { @@ -124,6 +127,14 @@ public synchronized File createNewPackage() { return null; } + /** + * Resets {@link #packedAuditCount}. + * Done before creating a new package to ensure only new packed audits are counted. + */ + private void resetPackedAuditCount() { + packedAuditCount = 0; + } + /** * Packs all newest audit trails from every contributor into the given file. * @@ -148,15 +159,16 @@ private void packContributors(File container) throws IOException { /** * Writes all the newest audit trails for a single contributor to the PrintWriter. * - * @param contributorId The id of the contributor to write the files for. + * @param contributorID The id of the contributor to write the files for. * @param writer The PrinterWriter where the output will be written. */ - private void packContributor(String contributorId, PrintWriter writer) { - long nextSeqNumber = store.getPreservationSequenceNumber(contributorId, collectionID); + private void packContributor(String contributorID, PrintWriter writer) { + long nextSeqNumber = store.getPreservationSequenceNumber(contributorID, collectionID) + 1; long largestSeqNumber = -1; long numPackedAudits = 0; - log.debug("Starting to pack AuditTrails for contributor: " + contributorId + " for collection: " + collectionID); - AuditEventIterator iterator = store.getAuditTrailsByIterator(null, collectionID, contributorId, nextSeqNumber, null, + log.debug("Starting to pack AuditTrails at seq-number {} for contributor: {} for collection: {}", + nextSeqNumber, contributorID, collectionID); + AuditEventIterator iterator = store.getAuditTrailsByIterator(null, collectionID, contributorID, nextSeqNumber, null, null, null, null, null, null, null); long timeStart = System.currentTimeMillis(); long logInterval = 1000; @@ -171,12 +183,15 @@ private void packContributor(String contributorId, PrintWriter writer) { writer.println(event); if ((numPackedAudits % logInterval) == 0) { - log.debug("Packed " + numPackedAudits + " AuditTrails in: " + (System.currentTimeMillis() - timeStart) + " ms"); + log.debug("Packed {} AuditTrails in: {} ms", numPackedAudits, System.currentTimeMillis() - timeStart); } } - log.debug("Packed a total of: " + numPackedAudits + " AuditTrails in: " + (System.currentTimeMillis() - timeStart) + " ms"); + log.debug("Packed a total of: {} AuditTrails in: {} ms", + numPackedAudits, System.currentTimeMillis() - timeStart); + if (numPackedAudits > 0) { - seqReached.put(contributorId, largestSeqNumber); + packedAuditCount += numPackedAudits; + seqNumsReached.put(contributorID, largestSeqNumber); } } @@ -194,4 +209,13 @@ private File createCompressedFile(File fileToCompress) throws IOException { FileUtils.zipFile(fileToCompress, zippedFile); return zippedFile; } + + /** + * Get the last count of packed audits i.e. count of new audits from all contributors for the collection. + * + * @return Count of packed audits. + */ + public long getPackedAuditCount() { + return packedAuditCount; + } } diff --git a/bitrepository-audit-trail-service/src/main/java/org/bitrepository/audittrails/preserver/AuditPreservationEventHandler.java b/bitrepository-audit-trail-service/src/main/java/org/bitrepository/audittrails/preserver/AuditPreservationEventHandler.java index 54b1faa77..bd0f4e733 100644 --- a/bitrepository-audit-trail-service/src/main/java/org/bitrepository/audittrails/preserver/AuditPreservationEventHandler.java +++ b/bitrepository-audit-trail-service/src/main/java/org/bitrepository/audittrails/preserver/AuditPreservationEventHandler.java @@ -44,12 +44,13 @@ public class AuditPreservationEventHandler implements EventHandler { private final String collectionID; /** - * @param preservationSequenceNumber The map between the contributor ids and their respective sequence number. + * @param contributorSequenceNumbers The map between the contributor ids and their respective sequence number. * @param store The store which should be updated with these sequence numbers. * @param collectionID The ID of the collection that needs to have its sequence number updated. */ - public AuditPreservationEventHandler(Map preservationSequenceNumber, AuditTrailStore store, String collectionID) { - this.seqNumbers = preservationSequenceNumber; + public AuditPreservationEventHandler(Map contributorSequenceNumbers, AuditTrailStore store, + String collectionID) { + this.seqNumbers = contributorSequenceNumbers; this.store = store; this.collectionID = collectionID; } @@ -58,8 +59,9 @@ public AuditPreservationEventHandler(Map preservationSequenceNumbe public void handleEvent(OperationEvent event) { if (event.getEventType() == OperationEventType.COMPLETE) { updateStoreWithResults(); + } else { - log.debug("Event for preservation of audit trails: " + event.toString()); + log.debug("Event for preservation of audit trails: {}", event); } } @@ -68,11 +70,12 @@ public void handleEvent(OperationEvent event) { */ private void updateStoreWithResults() { for (Map.Entry entry : seqNumbers.entrySet()) { - if (store.havePreservationKey(entry.getKey(), collectionID)) { - store.setPreservationSequenceNumber(entry.getKey(), collectionID, entry.getValue()); + String contributorID = entry.getKey(); + if (store.hasPreservationKey(contributorID, collectionID)) { + store.setPreservationSequenceNumber(contributorID, collectionID, entry.getValue()); } else { - log.debug("Preservation key for contributor: " + entry.getKey() + " in collection: " + collectionID + - " is not known by the database."); + log.debug("Preservation key for contributor: {} in collection: {} is not known by the database.", + contributorID, collectionID); } } } diff --git a/bitrepository-audit-trail-service/src/main/java/org/bitrepository/audittrails/preserver/AuditTrailPreserver.java b/bitrepository-audit-trail-service/src/main/java/org/bitrepository/audittrails/preserver/AuditTrailPreserver.java index b3d8ae992..9f44f9f90 100644 --- a/bitrepository-audit-trail-service/src/main/java/org/bitrepository/audittrails/preserver/AuditTrailPreserver.java +++ b/bitrepository-audit-trail-service/src/main/java/org/bitrepository/audittrails/preserver/AuditTrailPreserver.java @@ -21,6 +21,8 @@ */ package org.bitrepository.audittrails.preserver; +import org.bitrepository.audittrails.webservice.PreservationInfo; + /** * Interface for the preservation of audit trails. * This will automatically preserve the audit trails with a given interval. @@ -40,4 +42,10 @@ public interface AuditTrailPreserver { * Stop the preservation of audit trails. */ void close(); + + /** + * Gets the latest preservation info. + * @return Object containing info about the latest preservation. + */ + PreservationInfo getPreservationInfo(); } diff --git a/bitrepository-audit-trail-service/src/main/java/org/bitrepository/audittrails/preserver/LocalAuditTrailPreserver.java b/bitrepository-audit-trail-service/src/main/java/org/bitrepository/audittrails/preserver/LocalAuditTrailPreserver.java index b83ab1204..00b774cfc 100644 --- a/bitrepository-audit-trail-service/src/main/java/org/bitrepository/audittrails/preserver/LocalAuditTrailPreserver.java +++ b/bitrepository-audit-trail-service/src/main/java/org/bitrepository/audittrails/preserver/LocalAuditTrailPreserver.java @@ -21,7 +21,9 @@ */ package org.bitrepository.audittrails.preserver; +import org.bitrepository.common.TimerTaskSchedule; import org.bitrepository.audittrails.store.AuditTrailStore; +import org.bitrepository.audittrails.webservice.PreservationInfo; import org.bitrepository.bitrepositoryelements.ChecksumDataForFileTYPE; import org.bitrepository.bitrepositoryelements.ChecksumSpecTYPE; import org.bitrepository.client.eventhandler.EventHandler; @@ -48,6 +50,7 @@ import java.net.URL; import java.util.Date; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.Timer; import java.util.TimerTask; @@ -61,11 +64,12 @@ public class LocalAuditTrailPreserver implements AuditTrailPreserver { private final AuditTrailStore store; private final BlockingPutFileClient client; private Timer timer; - private AuditPreservationTimerTask auditTask = null; + private AuditPreservationTimerTask preservationTask = null; private final Map auditPackers = new HashMap<>(); private final AuditTrailPreservation preservationSettings; private final Settings settings; private final FileExchange exchange; + private long preservedAuditCount = 0; /** * @param settings The preservationSettings for the audit trail service. @@ -85,6 +89,26 @@ public LocalAuditTrailPreserver(Settings settings, AuditTrailStore store, PutFil for (String collectionID : SettingsUtils.getAllCollectionsIDs()) { this.auditPackers.put(collectionID, new AuditPacker(store, preservationSettings, collectionID)); } + initializeDatabaseEntries(); + } + + /** + * Ensures that the collection and contributor entries exist in the database and adds a preservation entry + * for each pairing. + */ + private void initializeDatabaseEntries() { + log.debug("Initializing collections and contributors in db."); + List collections = SettingsUtils.getAllCollectionsIDs(); + for (String collectionID : collections) { + store.addCollection(collectionID); + for (String contributorID : SettingsUtils.getAuditContributorsForCollection(collectionID)) { + store.addContributor(contributorID); + + if (!store.hasPreservationKey(contributorID, collectionID)) { + store.setPreservationSequenceNumber(contributorID, collectionID, 0L); + } + } + } } @Override @@ -95,10 +119,11 @@ public void start() { } long preservationInterval = preservationSettings.getAuditTrailPreservationInterval(); long timerCheckInterval = preservationInterval / 10; - log.info("Instantiating the preservation of audit trails every " + TimeUtils.millisecondsToHuman(preservationInterval)); - timer = new Timer(); - auditTask = new AuditPreservationTimerTask(preservationInterval); - timer.scheduleAtFixedRate(auditTask, timerCheckInterval, timerCheckInterval); + log.info("Instantiating the preservation of audit trails every {}", + TimeUtils.millisecondsToHuman(preservationInterval)); + timer = new Timer(true); + preservationTask = new AuditPreservationTimerTask(preservationInterval); + timer.scheduleAtFixedRate(preservationTask, timerCheckInterval, timerCheckInterval); } @Override @@ -110,16 +135,19 @@ public void close() { @Override public void preserveRepositoryAuditTrails() { - if (auditTask == null) { - log.info("preserving the audit trails "); - } else { - auditTask.resetTime(); - } + resetPreservedAuditCount(); for (String collectionID : SettingsUtils.getAllCollectionsIDs()) { performAuditTrailPreservation(collectionID); } } + /** + * Resets the preserved audit trail count so newly preserved audits can be counted. + */ + private void resetPreservedAuditCount() { + preservedAuditCount = 0; + } + /** * Performs the audit trails preservation. * Uses the AuditPacker to pack the audit trails in a file, then uploads the file to the default file-server, and @@ -131,18 +159,27 @@ public void preserveRepositoryAuditTrails() { */ private synchronized void performAuditTrailPreservation(String collectionID) { try { - File auditPackage = auditPackers.get(collectionID).createNewPackage(); - URL url = uploadFile(auditPackage); - log.info("Uploaded the file '" + auditPackage + "' to '" + url.toExternalForm() + "'"); + AuditPacker auditPacker = auditPackers.get(collectionID); + File auditPackage = auditPacker.createNewPackage(); + + if (auditPacker.getPackedAuditCount() > 0) { + URL url = uploadFile(auditPackage); + log.info("Uploaded the file '{}' to '{}'", auditPackage, url.toExternalForm()); + + ChecksumDataForFileTYPE checksumData = getValidationChecksumDataForFile(auditPackage); - ChecksumDataForFileTYPE checksumData = getValidationChecksumDataForFile(auditPackage); + EventHandler eventHandler = new AuditPreservationEventHandler(auditPacker.getSequenceNumbersReached(), + store, collectionID); + client.putFile(preservationSettings.getAuditTrailPreservationCollection(), url, auditPackage.getName(), + auditPackage.length(), checksumData, null, eventHandler, + "Preservation of audit trails from the AuditTrail service."); - EventHandler eventHandler = new AuditPreservationEventHandler(auditPackers.get(collectionID).getSequenceNumbersReached(), store, - collectionID); - client.putFile(preservationSettings.getAuditTrailPreservationCollection(), url, auditPackage.getName(), auditPackage.length(), - checksumData, null, eventHandler, "Preservation of audit trails from the AuditTrail service."); + preservedAuditCount += auditPacker.getPackedAuditCount(); + } else { + log.info("No new audit trails to preserve. No preservation file uploaded."); + } - log.debug("Cleanup of the uploaded audit trail package."); + log.debug("Cleanup of the audit trail package."); FileUtils.delete(auditPackage); } catch (IOException e) { throw new CoordinationLayerException("Cannot perform the preservation of audit trails.", e); @@ -179,38 +216,67 @@ private URL uploadFile(File file) throws IOException { return uploadedFileURL; } + @Override + public PreservationInfo getPreservationInfo() { + PreservationInfo info = new PreservationInfo(); + info.setCollectionID(preservationSettings.getAuditTrailPreservationCollection()); + + Date lastStart = preservationTask.getLastPreservationStart(); + Date lastFinish = preservationTask.getLastPreservationFinish(); + Date nextStart = preservationTask.getNextScheduledRun(); + + // Need to handle the case where preservation has not started/finished yet. + if (lastStart != null) { + info.setLastStart(TimeUtils.shortDate(lastStart)); + if (lastFinish != null) { + long lastDurationMS = lastFinish.getTime() - lastStart.getTime(); + info.setLastDuration(TimeUtils.millisecondsToHuman(lastDurationMS)); + } else { + info.setLastDuration("Preservation has not finished yet"); + } + } else { + info.setLastStart("Audit trail preservation has not started yet"); + info.setLastDuration("Not available"); + } + info.setNextStart(TimeUtils.shortDate(nextStart)); + info.setPreservedAuditCount(preservedAuditCount); + return info; + } + /** * Timer task for keeping track of the automated collecting of audit trails. */ private class AuditPreservationTimerTask extends TimerTask { - private final long interval; - private Date nextRun; + private final Logger log = LoggerFactory.getLogger(getClass()); + private final TimerTaskSchedule schedule; /** * @param interval The interval between running this timer task. */ private AuditPreservationTimerTask(long interval) { - this.interval = interval; - resetTime(); + this.schedule = new TimerTaskSchedule(interval, 0); } - /** - * Resets the date for next run. - */ - private void resetTime() { - nextRun = new Date(System.currentTimeMillis() + interval); + public Date getNextScheduledRun() { + return schedule.getNextRun(); + } + + public Date getLastPreservationStart() { + return schedule.getLastStart(); + } + + public Date getLastPreservationFinish() { + return schedule.getLastFinish(); } @Override public void run() { - if (nextRun.getTime() < System.currentTimeMillis()) { - try { - log.debug("Time to preserve the audit trails."); - preserveRepositoryAuditTrails(); - } catch (Exception e) { - log.error("Caught exception while attempting to preserve AuditTrails", e); - } - resetTime(); + if (getNextScheduledRun().getTime() < System.currentTimeMillis()) { + log.info("Starting preservation of audit trails."); + schedule.start(); + preserveRepositoryAuditTrails(); + schedule.finish(); + log.info("Finished preservation. Scheduled new preservation task to start {}", getNextScheduledRun()); } } } diff --git a/bitrepository-audit-trail-service/src/main/java/org/bitrepository/audittrails/store/AuditDatabaseExtractor.java b/bitrepository-audit-trail-service/src/main/java/org/bitrepository/audittrails/store/AuditDatabaseExtractor.java index 2d800da0c..cc85d6238 100644 --- a/bitrepository-audit-trail-service/src/main/java/org/bitrepository/audittrails/store/AuditDatabaseExtractor.java +++ b/bitrepository-audit-trail-service/src/main/java/org/bitrepository/audittrails/store/AuditDatabaseExtractor.java @@ -221,18 +221,18 @@ private String joinWithContributorTable() { private String createRestriction() { StringBuilder res = new StringBuilder(); - if (model.getFileId() != null) { + if (model.getFileID() != null) { nextArgument(res); res.append(FILE_TABLE + "." + FILE_FILE_ID + " = ? "); } - if (model.getCollectionId() != null) { + if (model.getCollectionID() != null) { nextArgument(res); res.append(FILE_TABLE + "." + FILE_COLLECTION_KEY + " = ( SELECT " + COLLECTION_KEY + " FROM " + COLLECTION_TABLE + " WHERE " + COLLECTION_ID + " = ? )"); } - if (model.getContributorId() != null) { + if (model.getContributorID() != null) { nextArgument(res); res.append(AUDIT_TRAIL_TABLE + "." + AUDIT_TRAIL_CONTRIBUTOR_KEY + " = ( SELECT " + CONTRIBUTOR_KEY + " FROM " + CONTRIBUTOR_TABLE + " WHERE " + CONTRIBUTOR_ID + " = ? )"); @@ -301,16 +301,16 @@ private void nextArgument(StringBuilder res) { private Object[] extractArgumentsFromModel() { List res = new ArrayList<>(); - if (model.getFileId() != null) { - res.add(model.getFileId()); + if (model.getFileID() != null) { + res.add(model.getFileID()); } - if (model.getCollectionId() != null) { - res.add(model.getCollectionId()); + if (model.getCollectionID() != null) { + res.add(model.getCollectionID()); } - if (model.getContributorId() != null) { - res.add(model.getContributorId()); + if (model.getContributorID() != null) { + res.add(model.getContributorID()); } if (model.getMinSeqNumber() != null) { diff --git a/bitrepository-audit-trail-service/src/main/java/org/bitrepository/audittrails/store/AuditEventIterator.java b/bitrepository-audit-trail-service/src/main/java/org/bitrepository/audittrails/store/AuditEventIterator.java index 71de265db..53e2b1503 100644 --- a/bitrepository-audit-trail-service/src/main/java/org/bitrepository/audittrails/store/AuditEventIterator.java +++ b/bitrepository-audit-trail-service/src/main/java/org/bitrepository/audittrails/store/AuditEventIterator.java @@ -98,7 +98,8 @@ public AuditTrailEvent getNextAuditTrailEvent() { long tStart = System.currentTimeMillis(); log.debug("Executing query to get AuditTrailEvents ResultSet"); auditResultSet = ps.executeQuery(); - log.debug("Finished executing AuditTrailEvents query, it took: " + (System.currentTimeMillis() - tStart) + "ms"); + log.debug("Finished executing AuditTrailEvents query, it took: {} ms", + System.currentTimeMillis() - tStart); } if (auditResultSet.next()) { event = new AuditTrailEvent(); diff --git a/bitrepository-audit-trail-service/src/main/java/org/bitrepository/audittrails/store/AuditTrailAdder.java b/bitrepository-audit-trail-service/src/main/java/org/bitrepository/audittrails/store/AuditTrailAdder.java index 8debe7e02..fdd163887 100644 --- a/bitrepository-audit-trail-service/src/main/java/org/bitrepository/audittrails/store/AuditTrailAdder.java +++ b/bitrepository-audit-trail-service/src/main/java/org/bitrepository/audittrails/store/AuditTrailAdder.java @@ -103,7 +103,7 @@ public class AuditTrailAdder { + " WHERE actor_name = ?)," + " ?, ?, ?, ?, ?, ? )"; - private final String addLatestSequencesSql = "INSERT INTO collection_progress " + private final String addLatestSequenceSql = "INSERT INTO collection_progress " + "(collectionID, contributorID, latest_sequence_number)" + " ( SELECT collectionID, ?, ? FROM collection" + " WHERE collectionID = ?" @@ -144,7 +144,7 @@ private void init() throws SQLException { addFileIDPs = conn.prepareStatement(addFileIDSql); addAuditTrailPs = conn.prepareStatement(addAuditTrailSql); updateLatestSeqPs = conn.prepareStatement(updateLatestSequenceSql); - addLatestSeqPs = conn.prepareStatement(addLatestSequencesSql); + addLatestSeqPs = conn.prepareStatement(addLatestSequenceSql); } /** diff --git a/bitrepository-audit-trail-service/src/main/java/org/bitrepository/audittrails/store/AuditTrailServiceDAO.java b/bitrepository-audit-trail-service/src/main/java/org/bitrepository/audittrails/store/AuditTrailServiceDAO.java index 5c8c3f9f6..570f478df 100644 --- a/bitrepository-audit-trail-service/src/main/java/org/bitrepository/audittrails/store/AuditTrailServiceDAO.java +++ b/bitrepository-audit-trail-service/src/main/java/org/bitrepository/audittrails/store/AuditTrailServiceDAO.java @@ -49,14 +49,14 @@ public AuditTrailServiceDAO(DatabaseManager databaseManager) { } @Override - public AuditEventIterator getAuditTrailsByIterator(String fileID, String collectionID, String contributorId, + public AuditEventIterator getAuditTrailsByIterator(String fileID, String collectionID, String contributorID, Long minSeqNumber, Long maxSeqNumber, String actorName, FileAction operation, Date startDate, Date endDate, String fingerprint, String operationID) { ExtractModel model = new ExtractModel(); - model.setFileId(fileID); - model.setCollectionId(collectionID); - model.setContributorId(contributorId); + model.setFileID(fileID); + model.setCollectionID(collectionID); + model.setContributorID(contributorID); model.setMinSeqNumber(minSeqNumber); model.setMaxSeqNumber(maxSeqNumber); model.setActorName(actorName); @@ -88,7 +88,7 @@ public List getKnownContributors() { @Override public long largestSequenceNumber(String contributorID, String collectionID) { - ArgumentValidator.checkNotNullOrEmpty(contributorID, "String contributorId"); + ArgumentValidator.checkNotNullOrEmpty(contributorID, "String contributorID"); ArgumentValidator.checkNotNullOrEmpty(collectionID, "String collectionID"); String sql = "SELECT latest_sequence_number FROM collection_progress" @@ -101,8 +101,8 @@ public long largestSequenceNumber(String contributorID, String collectionID) { } @Override - public long getPreservationSequenceNumber(String contributorId, String collectionID) { - ArgumentValidator.checkNotNullOrEmpty(contributorId, "String contributorId"); + public long getPreservationSequenceNumber(String contributorID, String collectionID) { + ArgumentValidator.checkNotNullOrEmpty(contributorID, "String contributorID"); ArgumentValidator.checkNotNullOrEmpty(collectionID, "String collectionID"); String sql = "SELECT preserved_seq_number FROM preservation" @@ -113,19 +113,18 @@ public long getPreservationSequenceNumber(String contributorId, String collectio + " SELECT collection_key FROM collection" + " WHERE collectionid = ? )"; - Long seq = DatabaseUtils.selectLongValue(dbConnector, sql, contributorId, collectionID); - if (seq != null) { - return seq.intValue(); - } - return 0; + Long seq = DatabaseUtils.selectLongValue(dbConnector, sql, contributorID, collectionID); + log.debug("Looked up latest preservation seq number for {} in collection '{}', it was: {}", + contributorID, collectionID, seq); + return (seq != null ? seq : 0L); } @Override - public void setPreservationSequenceNumber(String contributorId, String collectionID, long seqNumber) { - ArgumentValidator.checkNotNullOrEmpty(contributorId, "String contributorId"); + public void setPreservationSequenceNumber(String contributorID, String collectionID, long seqNumber) { + ArgumentValidator.checkNotNullOrEmpty(contributorID, "String contributorID"); ArgumentValidator.checkNotNegative(seqNumber, "int seqNumber"); - long preservationKey = retrievePreservationKey(contributorId, collectionID); - log.debug("Updating preservation sequence number for contributor: " + contributorId + long preservationKey = retrievePreservationKey(contributorID, collectionID); + log.debug("Updating preservation sequence number for contributor: " + contributorID + " in collection: " + collectionID + " to seq: " + seqNumber); String sqlUpdate = "UPDATE preservation SET preserved_seq_number = ?" @@ -134,7 +133,7 @@ public void setPreservationSequenceNumber(String contributorId, String collectio } @Override - public boolean havePreservationKey(String contributorID, String collectionID) { + public boolean hasPreservationKey(String contributorID, String collectionID) { String sql = "SELECT preservation_key FROM preservation" + " WHERE contributor_key = (" + " SELECT contributor_key FROM contributor" @@ -150,11 +149,11 @@ public boolean havePreservationKey(String contributorID, String collectionID) { /** * Retrieves the key of the preservation table entry for the given collection and contributor. * - * @param contributorId The contributor of the preservation table entry. + * @param contributorID The contributor of the preservation table entry. * @param collectionID The collection of the preservation table entry. * @return The key of the entry in the preservation table. */ - private Long retrievePreservationKey(String contributorId, String collectionID) { + private Long retrievePreservationKey(String contributorID, String collectionID) { String sqlRetrieve = "SELECT preservation_key FROM preservation" + " WHERE contributor_key = (" + " SELECT contributor_key FROM contributor" @@ -162,32 +161,52 @@ private Long retrievePreservationKey(String contributorId, String collectionID) + " AND collection_key = (" + " SELECT collection_key FROM collection" + " WHERE collectionid = ? )"; - Long guid = DatabaseUtils.selectLongValue(dbConnector, sqlRetrieve, contributorId, collectionID); + Long guid = DatabaseUtils.selectLongValue(dbConnector, sqlRetrieve, contributorID, collectionID); if (guid == null) { - log.debug("Inserting preservation entry for contributor '" + contributorId + "' and collection '" - + collectionID + "' into the preservation table."); + log.debug("Inserting preservation entry for contributor '{}' and collection '{}'" + + " into the preservation table.", contributorID, collectionID); String sqlInsert = "INSERT INTO preservation ( contributor_key, collection_key)" + " VALUES ( " + "(SELECT contributor_key FROM contributor" + " WHERE contributor_id = ?)" + ", " + "( SELECT collection_key FROM collection" - + " WHERE collectionid" + " = ? )" + + " WHERE collectionid = ? )" + ")"; - DatabaseUtils.executeStatement(dbConnector, sqlInsert, contributorId, collectionID); + DatabaseUtils.executeStatement(dbConnector, sqlInsert, contributorID, collectionID); - guid = DatabaseUtils.selectLongValue(dbConnector, sqlRetrieve, contributorId, collectionID); + guid = DatabaseUtils.selectLongValue(dbConnector, sqlRetrieve, contributorID, collectionID); } if (guid == null) { - throw new IllegalStateException("PreservationKey cannot be obtained for contributor: " + contributorId + + throw new IllegalStateException("PreservationKey cannot be obtained for contributor: " + contributorID + " in collection: " + collectionID); } return guid; } + @Override + public void addCollection(String collectionID) { + log.debug("Inserting collection with ID '{}' into db.", collectionID); + String insertQuery = "INSERT INTO collection ( collectionid )" + + " ( SELECT ? FROM collection" + + " WHERE collectionid = ?" + + " HAVING count(*) = 0 )"; + DatabaseUtils.executeStatement(dbConnector, insertQuery, collectionID, collectionID); + } + + @Override + public void addContributor(String contributorID) { + log.debug("Inserting contributor with ID '{}' into db.", contributorID); + String insertQuery = "INSERT INTO contributor ( contributor_id )" + + " ( SELECT ? FROM contributor" + + " WHERE contributor_id = ?" + + " HAVING count(*) = 0 )"; + DatabaseUtils.executeStatement(dbConnector, insertQuery, contributorID, contributorID); + } + @Override public void close() { try { diff --git a/bitrepository-audit-trail-service/src/main/java/org/bitrepository/audittrails/store/AuditTrailStore.java b/bitrepository-audit-trail-service/src/main/java/org/bitrepository/audittrails/store/AuditTrailStore.java index 62f457837..ac852ec76 100644 --- a/bitrepository-audit-trail-service/src/main/java/org/bitrepository/audittrails/store/AuditTrailStore.java +++ b/bitrepository-audit-trail-service/src/main/java/org/bitrepository/audittrails/store/AuditTrailStore.java @@ -40,7 +40,7 @@ public interface AuditTrailStore { * * @param fileID [OPTIONAL] The id of the file for restricting the extraction. * @param collectionID [OPTIONAL] The id of the collection from which to retrieve audit trails. - * @param contributorId [OPTIONAL] The id of the contributor for restricting the extraction. + * @param contributorID [OPTIONAL] The id of the contributor for restricting the extraction. * @param minSeqNumber [OPTIONAL] The minimum sequence number for restricting the extraction. * @param maxSeqNumber [OPTIONAL] The maximum sequence number for restricting the extraction. * @param actorName [OPTIONAL] The name of the actor for restricting the extraction. @@ -51,7 +51,7 @@ public interface AuditTrailStore { * @param operationID [OPTIONAL] The ID of the operation (conversationID) for the audits * @return The requested audit trails from the store. */ - AuditEventIterator getAuditTrailsByIterator(String fileID, String collectionID, String contributorId, + AuditEventIterator getAuditTrailsByIterator(String fileID, String collectionID, String contributorID, Long minSeqNumber, Long maxSeqNumber, String actorName, FileAction operation, Date startDate, Date endDate, String fingerprint, String operationID); @@ -65,33 +65,45 @@ AuditEventIterator getAuditTrailsByIterator(String fileID, String collectionID, */ void addAuditTrails(AuditTrailEvents auditTrailsEvents, String collectionID, String contributorID); + /** + * Add a collection to the store. + * @param collectionID The collection ID to add. + */ + void addCollection(String collectionID); + + /** + * Add a contributor to the store. + * @param contributorID The contributor ID to add. + */ + void addContributor(String contributorID); + /** * Retrieves the largest sequence number for a given contributor. * - * @param contributorId The id of the contributor to retrieve the largest sequence number from. + * @param contributorID The id of the contributor to retrieve the largest sequence number from. * @param collectionID The id of the collection for the sequence number of the contributor. * @return The largest sequence number. */ - long largestSequenceNumber(String contributorId, String collectionID); + long largestSequenceNumber(String contributorID, String collectionID); /** * Retrieves the preservation sequence number for the given contributor, which tells how far the preservation * of the audit trails has gotten. * - * @param contributorId The id of the contributor. + * @param contributorID The id of the contributor. * @param collectionID The id of the collection for the sequence number of the contributor. * @return The preservation sequence number for the given contributor. */ - long getPreservationSequenceNumber(String contributorId, String collectionID); + long getPreservationSequenceNumber(String contributorID, String collectionID); /** * Set the preservation sequence number for the given contributor. * - * @param contributorId The id of the contributor. + * @param contributorID The id of the contributor. * @param collectionID The id of the collection for the sequence number of the contributor. * @param seqNumber The new preservation sequence number for the given contributor. */ - void setPreservationSequenceNumber(String contributorId, String collectionID, long seqNumber); + void setPreservationSequenceNumber(String contributorID, String collectionID, long seqNumber); /** * Check to see if the database knows a contributor @@ -100,7 +112,7 @@ AuditEventIterator getAuditTrailsByIterator(String fileID, String collectionID, * @param collectionID The ID of the collection; * @return boolean true, if the contributor is known by the database, false otherwise. */ - boolean havePreservationKey(String contributorID, String collectionID); + boolean hasPreservationKey(String contributorID, String collectionID); /** * Get the list of known audit trail contributors. I.e. those contributors which have delivered diff --git a/bitrepository-audit-trail-service/src/main/java/org/bitrepository/audittrails/store/ExtractModel.java b/bitrepository-audit-trail-service/src/main/java/org/bitrepository/audittrails/store/ExtractModel.java index eb9e9ac9b..ac051866c 100644 --- a/bitrepository-audit-trail-service/src/main/java/org/bitrepository/audittrails/store/ExtractModel.java +++ b/bitrepository-audit-trail-service/src/main/java/org/bitrepository/audittrails/store/ExtractModel.java @@ -30,15 +30,15 @@ */ class ExtractModel { /** - * @see #getFileId(). + * @see #getFileID(). */ private String fileID; /** - * @see #getCollectionId(). + * @see #getCollectionID(). */ private String collectionID; /** - * @see #getContributorId(). + * @see #getContributorID(). */ private String contributorId; /** @@ -80,42 +80,42 @@ public ExtractModel() { /** * @return The fileID; */ - public String getFileId() { + public String getFileID() { return fileID; } /** * @param fileID The new file id. */ - public void setFileId(String fileID) { + public void setFileID(String fileID) { this.fileID = fileID; } /** * @return The collectionID; */ - public String getCollectionId() { + public String getCollectionID() { return collectionID; } /** * @param collectionID The new collection id. */ - public void setCollectionId(String collectionID) { + public void setCollectionID(String collectionID) { this.collectionID = collectionID; } /** * @return The contributorId; */ - public String getContributorId() { + public String getContributorID() { return contributorId; } /** * @param contributorId The new id of the contributor. */ - public void setContributorId(String contributorId) { + public void setContributorID(String contributorId) { this.contributorId = contributorId; } diff --git a/bitrepository-audit-trail-service/src/main/java/org/bitrepository/audittrails/webservice/CollectorInfo.java b/bitrepository-audit-trail-service/src/main/java/org/bitrepository/audittrails/webservice/CollectorInfo.java index 4da5e144e..63c50fb29 100644 --- a/bitrepository-audit-trail-service/src/main/java/org/bitrepository/audittrails/webservice/CollectorInfo.java +++ b/bitrepository-audit-trail-service/src/main/java/org/bitrepository/audittrails/webservice/CollectorInfo.java @@ -26,11 +26,11 @@ @XmlRootElement public class CollectorInfo { - String collectionID; - String lastStart; - String lastDuration; - String nextStart; - long collectedAudits; + private String collectionID; + private String lastStart; + private String lastDuration; + private String nextStart; + private long collectedAudits; public String getCollectionID() { return collectionID; diff --git a/bitrepository-audit-trail-service/src/main/java/org/bitrepository/audittrails/webservice/PreservationInfo.java b/bitrepository-audit-trail-service/src/main/java/org/bitrepository/audittrails/webservice/PreservationInfo.java new file mode 100644 index 000000000..cda437c92 --- /dev/null +++ b/bitrepository-audit-trail-service/src/main/java/org/bitrepository/audittrails/webservice/PreservationInfo.java @@ -0,0 +1,52 @@ +package org.bitrepository.audittrails.webservice; + +import javax.xml.bind.annotation.XmlRootElement; + +@XmlRootElement +public class PreservationInfo { + private String collectionID; + private String lastStart; + private String lastDuration; + private String nextStart; + private long preservedAuditCount; + + public String getCollectionID() { + return collectionID; + } + + public void setCollectionID(String collectionID) { + this.collectionID = collectionID; + } + + public String getLastStart() { + return lastStart; + } + + public void setLastStart(String lastStart) { + this.lastStart = lastStart; + } + + public String getLastDuration() { + return lastDuration; + } + + public void setLastDuration(String lastDuration) { + this.lastDuration = lastDuration; + } + + public String getNextStart() { + return nextStart; + } + + public void setNextStart(String nextStart) { + this.nextStart = nextStart; + } + + public long getPreservedAuditCount() { + return preservedAuditCount; + } + + public void setPreservedAuditCount(long preservedAuditCount) { + this.preservedAuditCount = preservedAuditCount; + } +} diff --git a/bitrepository-audit-trail-service/src/main/java/org/bitrepository/audittrails/webservice/RestAuditTrailService.java b/bitrepository-audit-trail-service/src/main/java/org/bitrepository/audittrails/webservice/RestAuditTrailService.java index aae6baab9..9fe47349f 100644 --- a/bitrepository-audit-trail-service/src/main/java/org/bitrepository/audittrails/webservice/RestAuditTrailService.java +++ b/bitrepository-audit-trail-service/src/main/java/org/bitrepository/audittrails/webservice/RestAuditTrailService.java @@ -55,11 +55,7 @@ import java.util.TimeZone; @Path("/AuditTrailService") - public class RestAuditTrailService { - /** - * The log. - */ private final Logger log = LoggerFactory.getLogger(getClass()); private final AuditTrailService service; private final CalendarUtils calendarUtils = CalendarUtils.getInstance(TimeZone.getDefault()); @@ -73,27 +69,16 @@ public RestAuditTrailService() { @Consumes("application/x-www-form-urlencoded") @Produces("application/json") public StreamingOutput queryAuditTrailEvents( - @FormParam("fromDate") - String fromDate, - @FormParam("toDate") - String toDate, - @FormParam("fileID") - String fileID, - @FormParam("reportingComponent") - String reportingComponent, - @FormParam("actor") - String actor, - @FormParam("action") - String action, - @FormParam("collectionID") - String collectionID, - @FormParam("fingerprint") - String fingerprint, - @FormParam("operationID") - String operationID, - @DefaultValue("1000") - @FormParam("maxAudittrails") - Integer maxResults) { + @FormParam("fromDate") String fromDate, + @FormParam("toDate") String toDate, + @FormParam("fileID") String fileID, + @FormParam("reportingComponent") String reportingComponent, + @FormParam("actor") String actor, + @FormParam("action") String action, + @FormParam("collectionID") String collectionID, + @FormParam("fingerprint") String fingerprint, + @FormParam("operationID") String operationID, + @DefaultValue("1000") @FormParam("maxAuditTrails") Integer maxResults) { Date from = calendarUtils.makeStartDateObject(fromDate); Date to = calendarUtils.makeEndDateObject(toDate); @@ -149,6 +134,20 @@ public List getCollectionSchedule() { return service.getCollectorInfos(); } + @GET + @Path("/preservationSchedule") + @Produces(MediaType.APPLICATION_JSON) + public PreservationInfo getPreservationSchedule() { + PreservationInfo preservationInfo = service.getPreservationInfo(); + if (preservationInfo != null) { + return preservationInfo; + } else { + throw new WebApplicationException(Response.status(Response.Status.NOT_FOUND) + .entity("404: Preservation must be enabled in settings to use this endpoint") + .type(MediaType.TEXT_PLAIN).build()); + } + } + private void writeAuditResult(AuditTrailEvent event, JsonGenerator jg) throws IOException { jg.writeStartObject(); jg.writeObjectField("fileID", event.getFileID()); @@ -162,7 +161,6 @@ private void writeAuditResult(AuditTrailEvent event, JsonGenerator jg) throws IO jg.writeObjectField("operationID", contentOrEmptyString(event.getOperationID())); jg.writeEndObject(); jg.flush(); - } @GET diff --git a/bitrepository-audit-trail-service/src/test/java/org/bitrepository/audittrails/preserver/AuditPackerTest.java b/bitrepository-audit-trail-service/src/test/java/org/bitrepository/audittrails/preserver/AuditPackerTest.java new file mode 100644 index 000000000..6dfd5efe4 --- /dev/null +++ b/bitrepository-audit-trail-service/src/test/java/org/bitrepository/audittrails/preserver/AuditPackerTest.java @@ -0,0 +1,62 @@ +package org.bitrepository.audittrails.preserver; + +import org.bitrepository.audittrails.store.AuditTrailStore; +import org.bitrepository.common.settings.Settings; +import org.bitrepository.common.settings.TestSettingsProvider; +import org.bitrepository.common.utils.SettingsUtils; +import org.bitrepository.settings.referencesettings.AuditTrailPreservation; +import org.jaccept.structure.ExtendedTestCase; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +import java.util.List; +import java.util.Map; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +import static org.testng.Assert.assertEquals; + +public class AuditPackerTest extends ExtendedTestCase { + private String collectionID; + private AuditTrailPreservation preservationSettings; + private AuditTrailStore store; + + @BeforeClass(alwaysRun = true) + public void setup() { + Settings settings = TestSettingsProvider.reloadSettings("LocalAuditPreservationUnderTest"); + preservationSettings = settings.getReferenceSettings().getAuditTrailServiceSettings().getAuditTrailPreservation(); + collectionID = settings.getCollections().get(0).getID(); + SettingsUtils.initialize(settings); + store = mock(AuditTrailStore.class); + } + + @Test + public void testCreateNewPackage() { + AuditPacker packer = new AuditPacker(store, preservationSettings, collectionID); + Map seqNumsReached = packer.getSequenceNumbersReached(); + assertEquals(seqNumsReached.entrySet().size(), 3); + assertEquals(packer.getPackedAuditCount(), 0); + + // Create a stubbed event iterator for each expected contributor containing only one event. + List iterators = List.of( + new StubAuditEventIterator(), new StubAuditEventIterator(), new StubAuditEventIterator()); + + when(store.getAuditTrailsByIterator( + any(), eq(collectionID), anyString(), any(Long.class), any(), any(), any(), any(), any(), any(), any()) + ).thenReturn(iterators.get(0)).thenReturn(iterators.get(1)).thenReturn(iterators.get(2)); + + // Do the actual call to createNewPackage - this will fetch first event from the iterators. + packer.createNewPackage(); + Long[] expectedSeqNums = {1L, 1L, 1L}; + assertEquals(packer.getPackedAuditCount(), 3); + assertEquals(packer.getSequenceNumbersReached().values().toArray(new Long[0]), expectedSeqNums); + + // As the iterators have no new audits there should be no newly packed audits on a new call. + packer.createNewPackage(); + assertEquals(packer.getPackedAuditCount(), 0); + assertEquals(packer.getSequenceNumbersReached().values().toArray(new Long[0]), expectedSeqNums); + } +} diff --git a/bitrepository-audit-trail-service/src/test/java/org/bitrepository/audittrails/preserver/AuditPreservationEventHandlerTest.java b/bitrepository-audit-trail-service/src/test/java/org/bitrepository/audittrails/preserver/AuditPreservationEventHandlerTest.java index 8dfff35f3..611e83e11 100644 --- a/bitrepository-audit-trail-service/src/test/java/org/bitrepository/audittrails/preserver/AuditPreservationEventHandlerTest.java +++ b/bitrepository-audit-trail-service/src/test/java/org/bitrepository/audittrails/preserver/AuditPreservationEventHandlerTest.java @@ -47,7 +47,7 @@ public void auditPreservationEventHandlerTest() throws Exception { Map map = new HashMap<>(); map.put(PILLARID, 1L); AuditTrailStore store = mock(AuditTrailStore.class); - when(store.havePreservationKey(eq(PILLARID), eq(TEST_COLLECTION))).thenReturn(true); + when(store.hasPreservationKey(eq(PILLARID), eq(TEST_COLLECTION))).thenReturn(true); AuditPreservationEventHandler eventHandler = new AuditPreservationEventHandler(map, store, TEST_COLLECTION); diff --git a/bitrepository-audit-trail-service/src/test/java/org/bitrepository/audittrails/preserver/LocalAuditPreservationTest.java b/bitrepository-audit-trail-service/src/test/java/org/bitrepository/audittrails/preserver/LocalAuditPreservationTest.java index ee0e2df7d..cbc79cb77 100644 --- a/bitrepository-audit-trail-service/src/test/java/org/bitrepository/audittrails/preserver/LocalAuditPreservationTest.java +++ b/bitrepository-audit-trail-service/src/test/java/org/bitrepository/audittrails/preserver/LocalAuditPreservationTest.java @@ -23,7 +23,6 @@ import org.bitrepository.audittrails.store.AuditEventIterator; import org.bitrepository.audittrails.store.AuditTrailStore; -import org.bitrepository.bitrepositoryelements.AuditTrailEvent; import org.bitrepository.bitrepositoryelements.ChecksumDataForFileTYPE; import org.bitrepository.bitrepositoryelements.ChecksumSpecTYPE; import org.bitrepository.bitrepositoryelements.FileAction; @@ -32,7 +31,6 @@ import org.bitrepository.common.DefaultThreadFactory; import org.bitrepository.common.settings.Settings; import org.bitrepository.common.settings.TestSettingsProvider; -import org.bitrepository.common.utils.CalendarUtils; import org.bitrepository.common.utils.SettingsUtils; import org.bitrepository.modify.putfile.PutFileClient; import org.bitrepository.protocol.FileExchange; @@ -44,7 +42,6 @@ import org.testng.annotations.Test; import java.io.FileInputStream; -import java.math.BigInteger; import java.net.URL; import java.sql.Date; @@ -62,8 +59,7 @@ public class LocalAuditPreservationTest extends ExtendedTestCase { /** The settings for the tests. Should be instantiated in the setup.*/ Settings settings; - String PILLARID = "pillarID"; - String ACTOR = "actor"; + String PILLAR_ID = "pillarID"; String collectionID; private URL testUploadUrl; private DefaultThreadFactory threadFactory; @@ -96,7 +92,7 @@ public void auditPreservationSchedulingTest() throws Exception { settings.getReferenceSettings().getAuditTrailServiceSettings().setTimerTaskCheckInterval(100); settings.getReferenceSettings().getAuditTrailServiceSettings().getAuditTrailPreservation().setAuditTrailPreservationInterval(300); settings.getRepositorySettings().getGetAuditTrailSettings().getNonPillarContributorIDs().clear(); - settings.getRepositorySettings().getGetAuditTrailSettings().getNonPillarContributorIDs().add(PILLARID); + settings.getRepositorySettings().getGetAuditTrailSettings().getNonPillarContributorIDs().add(PILLAR_ID); addStep("Create the preserver", "No calls to store or client"); FileExchange fileExchangeMock = mock(FileExchange.class); @@ -112,7 +108,7 @@ public void auditPreservationSchedulingTest() throws Exception { Assert.assertEquals(store.getCallsToSetPreservationSequenceNumber(), 0); Assert.assertEquals(client.getCallsToPutFile(), 0);*/ - verify(store).getPreservationSequenceNumber(PILLARID, collectionID); + verify(store).getPreservationSequenceNumber(PILLAR_ID, collectionID); verifyNoMoreInteractions(store); addStep("Start the preservation scheduling and wait for more than one interval", ""); @@ -134,8 +130,8 @@ public AuditEventIterator answer(InvocationOnMock invocation) { preserver.close(); // getPreservationSequenceNumber should be called twice, first to 'initialize' auditpacker, and second to // run the preserver/packer... - verify(store, times(2)).getPreservationSequenceNumber(PILLARID, collectionID); - verify(store).getAuditTrailsByIterator(null, null, PILLARID, 0L, + verify(store, times(2)).getPreservationSequenceNumber(PILLAR_ID, collectionID); + verify(store).getAuditTrailsByIterator(null, null, PILLAR_ID, 0L, null, null, null, null, null, null, null); verify(iterator).getNextAuditTrailEvent(); //Assert.assertEquals(store.getCallsToGetAuditTrails(), settings.getRepositorySettings().getGetAuditTrailSettings().getNonPillarContributorIDs().size()); @@ -152,9 +148,9 @@ public void auditPreservationIngestTest() throws Exception { MockPutClient client = new MockPutClient(); settings.getRepositorySettings().getCollections().getCollection().get(0).getPillarIDs().getPillarID().clear(); - settings.getRepositorySettings().getCollections().getCollection().get(0).getPillarIDs().getPillarID().add(PILLARID); + settings.getRepositorySettings().getCollections().getCollection().get(0).getPillarIDs().getPillarID().add(PILLAR_ID); settings.getRepositorySettings().getGetAuditTrailSettings().getNonPillarContributorIDs().clear(); - settings.getRepositorySettings().getGetAuditTrailSettings().getNonPillarContributorIDs().add(PILLARID); + settings.getRepositorySettings().getGetAuditTrailSettings().getNonPillarContributorIDs().add(PILLAR_ID); SettingsUtils.initialize(settings); AuditTrailStore store = mock(AuditTrailStore.class); @@ -164,10 +160,14 @@ public void auditPreservationIngestTest() throws Exception { FileExchange fileExchange = mock(FileExchange.class); LocalAuditTrailPreserver preserver = new LocalAuditTrailPreserver(settings, store, client, fileExchange); - - verify(store).getPreservationSequenceNumber(PILLARID, collectionID); + + verify(store).addCollection(collectionID); + verify(store).addContributor(PILLAR_ID); + verify(store).getPreservationSequenceNumber(PILLAR_ID, collectionID); + verify(store).hasPreservationKey(PILLAR_ID, collectionID); + verify(store).setPreservationSequenceNumber(PILLAR_ID, collectionID, 0); verifyNoMoreInteractions(store); - + addStep("Call the preservation of audit trails now.", "Should make calls to the store, upload the file and call the client"); @@ -183,8 +183,8 @@ public AuditEventIterator answer(InvocationOnMock invocation) { preserver.preserveRepositoryAuditTrails(); // getPreservationSequenceNumber should be called twice, first to 'initialize' auditpacker, and second to // run the preserver/packer... - verify(store, times(2)).getPreservationSequenceNumber(PILLARID, collectionID); - verify(store).getAuditTrailsByIterator(null, collectionID, PILLARID, 0L, + verify(store, times(2)).getPreservationSequenceNumber(PILLAR_ID, collectionID); + verify(store).getAuditTrailsByIterator(null, collectionID, PILLAR_ID, 1L, null, null, null, null, null, null, null); assertEquals(client.getCallsToPutFile(), 1); @@ -214,27 +214,4 @@ public int getCallsToPutFile() { return callsToPutFile; } } - - private class StubAuditEventIterator extends AuditEventIterator { - boolean called = false; - public StubAuditEventIterator() { - super(null); - } - - @Override - public AuditTrailEvent getNextAuditTrailEvent() { - if(called) { - return null; - } else { - called = true; - AuditTrailEvent e1 = new AuditTrailEvent(); - e1.setActionDateTime(CalendarUtils.getNow()); - e1.setActionOnFile(FileAction.FAILURE); - e1.setActorOnFile(ACTOR); - e1.setSequenceNumber(BigInteger.ONE); - e1.setReportingComponent(PILLARID); - return e1; - } - } - } } diff --git a/bitrepository-audit-trail-service/src/test/java/org/bitrepository/audittrails/preserver/StubAuditEventIterator.java b/bitrepository-audit-trail-service/src/test/java/org/bitrepository/audittrails/preserver/StubAuditEventIterator.java new file mode 100644 index 000000000..f4c118949 --- /dev/null +++ b/bitrepository-audit-trail-service/src/test/java/org/bitrepository/audittrails/preserver/StubAuditEventIterator.java @@ -0,0 +1,37 @@ +package org.bitrepository.audittrails.preserver; + +import org.bitrepository.audittrails.store.AuditEventIterator; +import org.bitrepository.bitrepositoryelements.AuditTrailEvent; +import org.bitrepository.bitrepositoryelements.FileAction; +import org.bitrepository.common.utils.CalendarUtils; + +import java.math.BigInteger; + +/** + * AuditEventIterator that will only return an event on first iteration. + */ +public class StubAuditEventIterator extends AuditEventIterator { + boolean called = false; + public StubAuditEventIterator() { + super(null); + } + + @Override + public AuditTrailEvent getNextAuditTrailEvent() { + String pillarID = "pillarID"; + String actor = "actor"; + + if(called) { + return null; + } else { + called = true; + AuditTrailEvent e1 = new AuditTrailEvent(); + e1.setActionDateTime(CalendarUtils.getNow()); + e1.setActionOnFile(FileAction.FAILURE); + e1.setActorOnFile(actor); + e1.setSequenceNumber(BigInteger.ONE); + e1.setReportingComponent(pillarID); + return e1; + } + } +} \ No newline at end of file diff --git a/bitrepository-audit-trail-service/src/main/java/org/bitrepository/audittrails/collector/CollectionSchedule.java b/bitrepository-core/src/main/java/org/bitrepository/common/TimerTaskSchedule.java similarity index 67% rename from bitrepository-audit-trail-service/src/main/java/org/bitrepository/audittrails/collector/CollectionSchedule.java rename to bitrepository-core/src/main/java/org/bitrepository/common/TimerTaskSchedule.java index 5af2b6505..2f1dc21be 100644 --- a/bitrepository-audit-trail-service/src/main/java/org/bitrepository/audittrails/collector/CollectionSchedule.java +++ b/bitrepository-core/src/main/java/org/bitrepository/common/TimerTaskSchedule.java @@ -19,18 +19,17 @@ * . * #L% */ -package org.bitrepository.audittrails.collector; +package org.bitrepository.common; import java.util.Date; /** * Class to handle the information about previous, current and future - * timing and schedule of AuditTrail collection - * Next run is at first scheduled as the interval after the current collection. - * When a collection has been finished, the next run is updated so the - * interval is after a collection has finished. + * timing and schedule of a timed task - e.g. audit-trail-service's AuditTrailCollectionTimerTask. + * Next run is at first scheduled as the interval after the current run of the task. + * When a task has been finished, the next run is updated so the interval is after a task has finished. */ -public class CollectionSchedule { +public class TimerTaskSchedule { private Date nextRun; private Date lastStart = null; private Date lastFinish = null; @@ -38,48 +37,48 @@ public class CollectionSchedule { private final long schedulingInterval; /** - * Constructor to create the CollectionSchedule. + * Constructor to create the TaskSchedule. * - * @param schedulingInterval The interval at which to schedule collection - * @param gracePeriod The grace period to wait before the first scheduling the first collection + * @param schedulingInterval The interval at which to schedule a new run of the task. + * @param gracePeriod The grace period to wait before the first scheduling. */ - public CollectionSchedule(long schedulingInterval, int gracePeriod) { + public TimerTaskSchedule(long schedulingInterval, int gracePeriod) { this.schedulingInterval = schedulingInterval; nextRun = new Date(System.currentTimeMillis() + gracePeriod); } /** - * @return the date of the next scheduled collection + * @return The date of the next scheduled task. */ public Date getNextRun() { return nextRun; } /** - * @return the date of the last finished collection, or the current collection if non has finished yet. - * May return null, if the first collection has not yet been started. + * @return The date of the last finished task, or the current run if none have finished yet. + * May return null, if the first run has not yet been started. */ public Date getLastStart() { return lastStart; } /** - * @return the date of the last finished collection. Returns null if no collection has finished yet. + * @return The date of the last finished task. Returns null if no run has finished yet. */ public Date getLastFinish() { return lastFinish; } /** - * @return the date of the currently running collection. Returns null, if no collection is currently running. + * @return The date of the currently running task. Returns null, if no task is currently running. */ public Date getCurrentStart() { return currentStart; } /** - * Indicate that a collection has been started. - * Updates the next scheduled collection. + * Indicate that a task has been started. + * Updates the next scheduled run of the task. */ public void start() { currentStart = new Date(System.currentTimeMillis()); @@ -90,8 +89,8 @@ public void start() { } /** - * Indicate that a collection has finished. - * Updates the next scheduled collection. + * Indicate that a task has finished. + * Updates the next scheduled run. */ public void finish() { lastFinish = new Date(System.currentTimeMillis()); diff --git a/bitrepository-core/src/test/resources/settings/xml/bitrepository-devel/RepositorySettings.xml b/bitrepository-core/src/test/resources/settings/xml/bitrepository-devel/RepositorySettings.xml index 1902c9ecb..95e978f64 100644 --- a/bitrepository-core/src/test/resources/settings/xml/bitrepository-devel/RepositorySettings.xml +++ b/bitrepository-core/src/test/resources/settings/xml/bitrepository-devel/RepositorySettings.xml @@ -27,11 +27,18 @@ Development - TwoPillarCollection - Pillar1Pillar2 + + TwoPillarCollection + + Pillar1 + Pillar2 + - SinglePillarCollection - Pillar1 + + SinglePillarCollection + + Pillar1 + diff --git a/bitrepository-webclient/src/main/webapp/audit-trail-service.html b/bitrepository-webclient/src/main/webapp/audit-trail-service.html index a39d1f34f..33901a461 100644 --- a/bitrepository-webclient/src/main/webapp/audit-trail-service.html +++ b/bitrepository-webclient/src/main/webapp/audit-trail-service.html @@ -20,319 +20,384 @@ #L% --> - - - Bitrepository audit-trail service - - - - - - - -
-
+ + + Bitrepository audit-trail service + + + + + + + +
+
+
+
-
-
-
-
-

Audit-trail service

-
-
-

- -

-

-
+
+
+

Audit-trail service

+
+
+

+ +

+
+
-
-
-
-
- - Show collection schedule - -
-
-
- - - - - - - - - - - -
Collection IDLast startLast durationLast collected number of auditsNext start
-
-
+
+
+
+
+ + Show collection schedule + +
+
+
+ + + + + + + + + + + +
Collection IDLast startLast durationLast collected number of auditsNext start
+
- -
-
- Audit trails display filters - - - - - - - - - - -
+ +
+
+ Audit trails display filters + + + + + + + + + + + +
+
+
+ Audit trails +
No request sent yet
+
+
- - - - + +
+ + + + + + - + +