diff --git a/bitrepository-audit-trail-service/src/main/java/org/bitrepository/audittrails/AuditTrailService.java b/bitrepository-audit-trail-service/src/main/java/org/bitrepository/audittrails/AuditTrailService.java
index 0f4fe3057..df018b467 100644
--- a/bitrepository-audit-trail-service/src/main/java/org/bitrepository/audittrails/AuditTrailService.java
+++ b/bitrepository-audit-trail-service/src/main/java/org/bitrepository/audittrails/AuditTrailService.java
@@ -83,7 +83,7 @@ public AuditTrailService(AuditTrailStore store, AuditTrailCollector collector, A
/**
* Constructor for audit trail service with disabled preservation.
- *
+ *
* See {@link #AuditTrailService(AuditTrailStore, AuditTrailCollector, AuditTrailPreserver, ContributorMediator,
* Settings)} for param descriptions.
*/
@@ -119,9 +119,7 @@ public AuditEventIterator queryAuditTrailEventsByIterator(Date fromDate, Date to
}
/**
- * Collects all the newest audit trails from the given collection.
- * TODO this currently calls all collections. It should only call a specified collection, which should be given
- * as argument.
+ * Collects all the newest audit trails from all collections.
*/
public void collectAuditTrails() {
for (org.bitrepository.settings.repositorysettings.Collection c
@@ -130,6 +128,13 @@ public void collectAuditTrails() {
}
}
+ /**
+ * Collects all the newest audit trails from a specific collection.
+ */
+ public void collectAuditTrails(String collectionID) {
+ collector.collectNewestAudits(collectionID);
+ }
+
/**
* Get the list of {@link CollectorInfo}
*
@@ -151,7 +156,7 @@ public List getCollectorInfos() {
* @return PreservationInfo or null if not enabled.
*/
public PreservationInfo getPreservationInfo() {
- if (preserver == null ) {
+ if (preserver == null) {
return null;
}
return preserver.getPreservationInfo();
diff --git a/bitrepository-audit-trail-service/src/main/java/org/bitrepository/audittrails/webservice/RestAuditTrailService.java b/bitrepository-audit-trail-service/src/main/java/org/bitrepository/audittrails/webservice/RestAuditTrailService.java
index 9fe47349f..348c3f417 100644
--- a/bitrepository-audit-trail-service/src/main/java/org/bitrepository/audittrails/webservice/RestAuditTrailService.java
+++ b/bitrepository-audit-trail-service/src/main/java/org/bitrepository/audittrails/webservice/RestAuditTrailService.java
@@ -44,6 +44,7 @@
import javax.ws.rs.POST;
import javax.ws.rs.Path;
import javax.ws.rs.Produces;
+import javax.ws.rs.QueryParam;
import javax.ws.rs.WebApplicationException;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
@@ -52,6 +53,7 @@
import java.sql.SQLException;
import java.util.Date;
import java.util.List;
+import java.util.Locale;
import java.util.TimeZone;
@Path("/AuditTrailService")
@@ -68,8 +70,7 @@ public RestAuditTrailService() {
@Path("/queryAuditTrailEvents/")
@Consumes("application/x-www-form-urlencoded")
@Produces("application/json")
- public StreamingOutput queryAuditTrailEvents(
- @FormParam("fromDate") String fromDate,
+ public StreamingOutput queryAuditTrailEvents(@FormParam("fromDate") String fromDate,
@FormParam("toDate") String toDate,
@FormParam("fileID") String fileID,
@FormParam("reportingComponent") String reportingComponent,
@@ -83,9 +84,9 @@ public StreamingOutput queryAuditTrailEvents(
Date to = calendarUtils.makeEndDateObject(toDate);
final int maxAudits = maxResults;
- final AuditEventIterator it = service.queryAuditTrailEventsByIterator(from, to, contentOrNull(fileID), collectionID,
- contentOrNull(reportingComponent), contentOrNull(actor), filterAction(action), contentOrNull(fingerprint),
- contentOrNull(operationID));
+ final AuditEventIterator it = service.queryAuditTrailEventsByIterator(from, to, contentOrNull(fileID),
+ collectionID, contentOrNull(reportingComponent), contentOrNull(actor), filterAction(action),
+ contentOrNull(fingerprint), contentOrNull(operationID));
if (it != null) {
return output -> {
JsonFactory jf = new JsonFactory();
@@ -114,17 +115,26 @@ public StreamingOutput queryAuditTrailEvents(
}
};
} else {
- throw new WebApplicationException(Response.status(Response.Status.NO_CONTENT).entity("Failed to get audit trails from database")
- .type(MediaType.TEXT_PLAIN).build());
+ throw new WebApplicationException(
+ Response.status(Response.Status.NO_CONTENT).entity("Failed to get audit trails from database")
+ .type(MediaType.TEXT_PLAIN).build());
}
}
@POST
- @Path("/collectAuditTrails/")
+ @Path("/collectAuditTrails")
@Produces("text/html")
public String collectAuditTrails() {
service.collectAuditTrails();
- return "Started audit trails collection";
+ return "Started audit trails collection.";
+ }
+
+ @POST
+ @Path("/collectSpecificAuditTrail")
+ @Produces("text/html")
+ public String collectSpecificAuditTrail(@QueryParam("collectionID") String collectionID) {
+ service.collectAuditTrails(collectionID);
+ return String.format(Locale.ROOT, "Started collecting audit trails for collection: %s.", collectionID);
}
@GET
@@ -143,7 +153,7 @@ public PreservationInfo getPreservationSchedule() {
return preservationInfo;
} else {
throw new WebApplicationException(Response.status(Response.Status.NOT_FOUND)
- .entity("404: Preservation must be enabled in settings to use this endpoint")
+ .entity("404: Preservation must be enabled in settings to use this endpoint.")
.type(MediaType.TEXT_PLAIN).build());
}
}
@@ -154,7 +164,8 @@ private void writeAuditResult(AuditTrailEvent event, JsonGenerator jg) throws IO
jg.writeObjectField("reportingComponent", event.getReportingComponent());
jg.writeObjectField("actor", contentOrEmptyString(event.getActorOnFile()));
jg.writeObjectField("action", event.getActionOnFile().toString());
- jg.writeObjectField("timeStamp", TimeUtils.shortDate(CalendarUtils.convertFromXMLGregorianCalendar(event.getActionDateTime())));
+ jg.writeObjectField("timeStamp",
+ TimeUtils.shortDate(CalendarUtils.convertFromXMLGregorianCalendar(event.getActionDateTime())));
jg.writeObjectField("info", contentOrEmptyString(event.getInfo()));
jg.writeObjectField("auditTrailInfo", contentOrEmptyString(event.getAuditTrailInformation()));
jg.writeObjectField("fingerprint", contentOrEmptyString(event.getCertificateID()));
diff --git a/bitrepository-audit-trail-service/src/test/java/org/bitrepository/audittrails/AuditTrailServiceTest.java b/bitrepository-audit-trail-service/src/test/java/org/bitrepository/audittrails/AuditTrailServiceTest.java
index 47e9e77a0..511377311 100644
--- a/bitrepository-audit-trail-service/src/test/java/org/bitrepository/audittrails/AuditTrailServiceTest.java
+++ b/bitrepository-audit-trail-service/src/test/java/org/bitrepository/audittrails/AuditTrailServiceTest.java
@@ -5,16 +5,16 @@
* Copyright (C) 2010 - 2012 The State and University Library, The Royal Library and The State Archives, Denmark
* %%
* This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Lesser General Public License as
- * published by the Free Software Foundation, either version 2.1 of the
+ * it under the terms of the GNU Lesser General Public License as
+ * published by the Free Software Foundation, either version 2.1 of the
* License, or (at your option) any later version.
- *
+ *
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Lesser Public License for more details.
- *
- * You should have received a copy of the GNU General Lesser Public
+ *
+ * You should have received a copy of the GNU General Lesser Public
* License along with this program. If not, see
* .
* #L%
@@ -54,35 +54,35 @@
import static org.mockito.Mockito.verify;
public class AuditTrailServiceTest extends ExtendedTestCase {
- /** The settings for the tests. Should be instantiated in the setup.*/
+ /** The settings for the tests. Should be instantiated in the setup. */
Settings settings;
-
+
public static final String TEST_COLLECTION = "dummy-collection";
public static final String DEFAULT_CONTRIBUTOR = "Contributor1";
private ThreadFactory threadFactory;
- @BeforeClass (alwaysRun = true)
+ @BeforeClass(alwaysRun = true)
public void setup() throws Exception {
settings = TestSettingsProvider.reloadSettings("AuditTrailServiceUnderTest");
Collection c = settings.getRepositorySettings().getCollections().getCollection().get(0);
settings.getRepositorySettings().getCollections().getCollection().clear();
c.setID(TEST_COLLECTION);
settings.getRepositorySettings().getCollections().getCollection().add(c);
- threadFactory = new DefaultThreadFactory(this.getClass().getSimpleName(),Thread.NORM_PRIORITY);
+ threadFactory = new DefaultThreadFactory(this.getClass().getSimpleName(), Thread.NORM_PRIORITY);
}
-
+
@Test(groups = {"unstable"})
public void auditTrailServiceTest() throws Exception {
addDescription("Test the Audit Trail Service");
DatatypeFactory factory = DatatypeFactory.newInstance();
settings.getRepositorySettings().getGetAuditTrailSettings().getNonPillarContributorIDs().clear();
- settings.getRepositorySettings().getGetAuditTrailSettings().getNonPillarContributorIDs().add(DEFAULT_CONTRIBUTOR);
+ settings.getRepositorySettings().getGetAuditTrailSettings().getNonPillarContributorIDs()
+ .add(DEFAULT_CONTRIBUTOR);
settings.getReferenceSettings().getAuditTrailServiceSettings()
.setCollectAuditInterval(factory.newDuration(800));
settings.getReferenceSettings().getAuditTrailServiceSettings().setTimerTaskCheckInterval(100L);
- settings.getReferenceSettings().getAuditTrailServiceSettings()
- .setGracePeriod(factory.newDuration(800));
+ settings.getReferenceSettings().getAuditTrailServiceSettings().setGracePeriod(factory.newDuration(800));
AuditTrailStore store = mock(AuditTrailStore.class);
AuditTrailClient client = mock(AuditTrailClient.class);
@@ -90,39 +90,37 @@ public void auditTrailServiceTest() throws Exception {
ContributorMediator mediator = mock(ContributorMediator.class);
AuditTrailCollector collector = new AuditTrailCollector(settings, client, store, alarmDispatcher);
-
+
addStep("Instantiate the service.", "Should work.");
AuditTrailService service = new AuditTrailService(store, collector, mediator, settings);
service.start();
-
+
addStep("Try to collect audit trails.", "Should make a call to the client.");
CollectionRunner collectionRunner = new CollectionRunner(service);
Thread t = threadFactory.newThread(collectionRunner);
t.start();
-
+
ArgumentCaptor eventHandlerCaptor = ArgumentCaptor.forClass(EventHandler.class);
verify(client, timeout(3000).times(1)).getAuditTrails(eq(TEST_COLLECTION), any(AuditTrailQuery[].class),
isNull(), isNull(), eventHandlerCaptor.capture(), any(String.class));
-
- AuditTrailResult event = new AuditTrailResult(DEFAULT_CONTRIBUTOR, TEST_COLLECTION, new ResultingAuditTrails(), false);
+
+ AuditTrailResult event = new AuditTrailResult(DEFAULT_CONTRIBUTOR, TEST_COLLECTION, new ResultingAuditTrails(),
+ false);
eventHandlerCaptor.getValue().handleEvent(event);
eventHandlerCaptor.getValue().handleEvent(new CompleteEvent(TEST_COLLECTION, null));
-
+
addStep("Retrieve audit trails with and without an action", "Should work.");
-
- verify(store, times(1)).addAuditTrails(any(AuditTrailEvents.class), eq(TEST_COLLECTION), eq(DEFAULT_CONTRIBUTOR));
+
+ verify(store, times(1)).addAuditTrails(any(AuditTrailEvents.class), eq(TEST_COLLECTION),
+ eq(DEFAULT_CONTRIBUTOR));
service.queryAuditTrailEventsByIterator(null, null, null, null, null, null, null, null, null);
- verify(store, times(1)).getAuditTrailsByIterator(isNull(), isNull(),
- isNull(), isNull(), isNull(), isNull(),
- isNull(), isNull(), isNull(), isNull(),
- isNull());
+ verify(store, times(1)).getAuditTrailsByIterator(isNull(), isNull(), isNull(), isNull(), isNull(), isNull(),
+ isNull(), isNull(), isNull(), isNull(), isNull());
service.queryAuditTrailEventsByIterator(null, null, null, null, null, null, FileAction.FAILURE, null, null);
- verify(store, times(1)).getAuditTrailsByIterator(isNull(), isNull(),
- isNull(), isNull(), isNull(), isNull(),
- eq(FileAction.FAILURE), isNull(), isNull(), isNull(),
- isNull());
+ verify(store, times(1)).getAuditTrailsByIterator(isNull(), isNull(), isNull(), isNull(), isNull(), isNull(),
+ eq(FileAction.FAILURE), isNull(), isNull(), isNull(), isNull());
+
-
addStep("Shutdown", "");
service.shutdown();
}
diff --git a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/reports/IntegrityReportConstants.java b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/reports/IntegrityReportConstants.java
index ed0e106d1..c348cf2d9 100644
--- a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/reports/IntegrityReportConstants.java
+++ b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/reports/IntegrityReportConstants.java
@@ -21,6 +21,10 @@
*/
package org.bitrepository.integrityservice.reports;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.Set;
+
/**
* Class containing constants related to integrity reports
*/
@@ -59,4 +63,8 @@ public enum ReportPart {
public static final String SECTION_HEADER_START_STOP = "========";
public static final String PILLAR_HEADER_START_STOP = "--------";
public static final String NO_ISSUE_HEADER_START_STOP = "++++++++";
+
+ public static Set getReportParts() {
+ return new HashSet<>(Arrays.asList(ReportPart.values()));
+ }
}
diff --git a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/reports/IntegrityReportProvider.java b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/reports/IntegrityReportProvider.java
index 4371a25e4..529926e52 100644
--- a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/reports/IntegrityReportProvider.java
+++ b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/reports/IntegrityReportProvider.java
@@ -48,7 +48,8 @@ public IntegrityReportProvider(File reportsDir) {
* @return IntegrityReportReader with the latest integrity report
* @throws FileNotFoundException if no report could be found
*/
- public synchronized IntegrityReportReader getLatestIntegrityReportReader(String collectionID) throws FileNotFoundException {
+ public synchronized IntegrityReportReader getLatestIntegrityReportReader(String collectionID)
+ throws FileNotFoundException {
IntegrityReportReader reader = reports.get(collectionID);
if (reader == null) {
log.info("Trying to lookup the latest report on disk for collection {}", collectionID);
@@ -63,6 +64,26 @@ public synchronized IntegrityReportReader getLatestIntegrityReportReader(String
return reader;
}
+ /**
+ * Get the latest integrity report of a collection, for a specific report part and specific pillar.
+ *
+ * @param collectionID The specific collectionID.
+ * @param pillarID The specific pillarID.
+ * @param reportPart The specific ReportPart.
+ * @return Returns the given report part {@link File} if it exists.
+ * @throws FileNotFoundException If no such report part can be found for the given pillar and collection.
+ */
+ public synchronized File getIntegrityReportPart(String collectionID, String pillarID, String reportPart)
+ throws FileNotFoundException {
+ log.info("Trying to lookup the '{}' report on disk for collection '{}'", reportPart, collectionID);
+ File latestReportPart = getReportPartFromDisk(collectionID, pillarID, reportPart);
+ if (latestReportPart != null) {
+ return latestReportPart;
+ } else {
+ throw new FileNotFoundException("Could not find latest integrity report");
+ }
+ }
+
/**
* Register the latest integrity report for a given collection
*
@@ -75,7 +96,7 @@ public synchronized void setLatestReport(String collectionID, File reportDir) {
private File getLatestReportFromDisk(String collectionID) {
File collectionReports = new File(reportsDir, collectionID);
- log.info("Looking for latest report dir in {}", collectionReports);
+ log.info("Looking for latest report dir in '{}'", collectionReports);
File[] reports = collectionReports.listFiles(File::isDirectory);
long lastModification = Long.MIN_VALUE;
File latestReport = null;
@@ -87,7 +108,37 @@ private File getLatestReportFromDisk(String collectionID) {
latestReport = reportDir;
lastModification = latestReport.lastModified();
} else {
- log.debug("Candidate report dir {} had no report file", reportDir);
+ log.debug("Candidate report dir '{}' had no report file", reportDir);
+ }
+ }
+ }
+
+ return latestReport;
+ }
+
+ /**
+ * Returns the latest report part for the given collection and pillar.
+ *
+ * @param collectionID The collection ID.
+ * @param pillarID The pillar ID.
+ * @param reportPart The wanted report part as {@link String}.
+ * @return Returns a {@link File} containing the latest report part for the given collection and pillar if such a file exists.
+ */
+ private File getReportPartFromDisk(String collectionID, String pillarID, String reportPart) {
+ File collectionReports = new File(reportsDir, collectionID);
+ log.info("Looking for latest report dir in '{}'", collectionReports);
+ File[] reports = collectionReports.listFiles(File::isDirectory);
+ long lastModification = Long.MIN_VALUE;
+ File latestReport = null;
+ assert reports != null;
+ for (File reportDir : reports) {
+ if (reportDir.lastModified() > lastModification) {
+ File reportFile = new File(reportDir, reportPart + "-" + pillarID);
+ if (reportFile.exists()) {
+ latestReport = reportFile;
+ lastModification = latestReport.lastModified();
+ } else {
+ log.debug("Report dir '{}' had no '{}'-report for pillar '{}'", reportDir, reportPart, pillarID);
}
}
}
diff --git a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/web/RestIntegrityService.java b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/web/RestIntegrityService.java
index e169b96f6..dab0e0f3f 100644
--- a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/web/RestIntegrityService.java
+++ b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/web/RestIntegrityService.java
@@ -34,6 +34,7 @@
import org.bitrepository.integrityservice.cache.IntegrityModel;
import org.bitrepository.integrityservice.cache.PillarCollectionStat;
import org.bitrepository.integrityservice.cache.database.IntegrityIssueIterator;
+import org.bitrepository.integrityservice.reports.IntegrityReportConstants;
import org.bitrepository.integrityservice.reports.IntegrityReportConstants.ReportPart;
import org.bitrepository.integrityservice.reports.IntegrityReportProvider;
import org.bitrepository.integrityservice.reports.IntegrityReportReader;
@@ -61,7 +62,9 @@
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
+import java.io.OutputStream;
import java.io.StringWriter;
+import java.nio.file.Files;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
@@ -69,7 +72,14 @@
import java.util.Locale;
import java.util.Map;
import java.util.Objects;
+import java.util.Set;
import java.util.stream.Collectors;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipOutputStream;
+
+import static javax.ws.rs.core.Response.ResponseBuilder;
+import static javax.ws.rs.core.Response.Status;
+import static javax.ws.rs.core.Response.status;
@Path("/IntegrityService")
public class RestIntegrityService {
@@ -95,30 +105,24 @@ public RestIntegrityService() {
@GET
@Path("/getTotalFileIDs")
@Produces(MediaType.APPLICATION_JSON)
- public HashMap> getTotalFileIDs(
- @QueryParam("collectionID")
- String collectionID,
- @QueryParam("pillarID")
- String pillarID,
- @QueryParam("page")
- int page,
- @DefaultValue("100")
- @QueryParam("pageSize")
- int pageSize) {
+ public HashMap> getTotalFileIDs(@QueryParam("collectionID") String collectionID,
+ @QueryParam("pillarID") String pillarID,
+ @QueryParam("page") int page,
+ @DefaultValue("100") @QueryParam("pageSize") int pageSize) {
IntegrityIssueIterator it = model.getFilesOnPillar(pillarID, getOffset(page, pageSize), pageSize, collectionID);
if (it == null) {
throw new WebApplicationException(
- Response.status(Response.Status.NO_CONTENT).entity("Failed to get missing files from database")
+ status(Status.NO_CONTENT).entity("Failed to get missing files from database")
.type(MediaType.TEXT_PLAIN).build());
}
List iteratorAsList = StreamingTools.iteratorToList(it);
if (iteratorAsList.isEmpty()) {
- throw new WebApplicationException(Response.status(Response.Status.NOT_FOUND)
- .entity(String.format(Locale.ROOT, "No fileIDs found for collection: '%s' and pillar: '%s'", collectionID, pillarID))
- .type(MediaType.TEXT_PLAIN).build());
+ throw new WebApplicationException(status(Status.NOT_FOUND).entity(
+ String.format(Locale.ROOT, "No fileIDs found for collection: '%s' and pillar: '%s'", collectionID,
+ pillarID)).type(MediaType.TEXT_PLAIN).build());
}
return new HashMap<>(Map.of(pillarID, iteratorAsList));
@@ -133,16 +137,10 @@ public HashMap> getTotalFileIDs(
@GET
@Path("/getMissingFileIDs")
@Produces(MediaType.APPLICATION_JSON)
- public HashMap> getMissingFileIDs(
- @QueryParam("collectionID")
- String collectionID,
- @QueryParam("pillarID")
- String pillarID,
- @QueryParam("page")
- int page,
- @DefaultValue("100")
- @QueryParam("pageSize")
- int pageSize) {
+ public HashMap> getMissingFileIDs(@QueryParam("collectionID") String collectionID,
+ @QueryParam("pillarID") String pillarID,
+ @QueryParam("page") int page,
+ @DefaultValue("100") @QueryParam("pageSize") int pageSize) {
HashMap> output = new HashMap<>();
ReportPart part = ReportPart.MISSING_FILE;
List missingOnPillar;
@@ -155,9 +153,9 @@ public HashMap> getMissingFileIDs(
missingOnPillar = getReportPart(part, collectionID, pillarID, page, pageSize);
output.put(pillarID, missingOnPillar);
} catch (FileNotFoundException e) {
- throw new WebApplicationException(Response.status(Response.Status.NOT_FOUND)
- .entity(String.format(Locale.ROOT, "No integrity '%s' report part for collection: '%s' and pillar: '%s' found!",
- part.getHumanString(), collectionID, pillarID)).type(MediaType.TEXT_PLAIN).build());
+ throw new WebApplicationException(status(Status.NOT_FOUND).entity(String.format(Locale.ROOT,
+ "No integrity '%s' report part for collection: '%s' and pillar: '%s' found!", part.getHumanString(),
+ collectionID, pillarID)).type(MediaType.TEXT_PLAIN).build());
}
for (String otherPillar : otherPillars) {
@@ -178,17 +176,12 @@ public HashMap> getMissingFileIDs(
@GET
@Path("/getMissingChecksumsFileIDs")
@Produces(MediaType.APPLICATION_JSON)
- public HashMap> getMissingChecksums(
- @QueryParam("collectionID")
- String collectionID,
- @QueryParam("pillarID")
- String pillarID,
- @QueryParam("page")
- int page,
- @DefaultValue("100")
- @QueryParam("pageSize")
- int pageSize) {
- List streamingOutput = getReportPartForPillar(ReportPart.MISSING_CHECKSUM, collectionID, pillarID, page, pageSize);
+ public HashMap> getMissingChecksums(@QueryParam("collectionID") String collectionID,
+ @QueryParam("pillarID") String pillarID,
+ @QueryParam("page") int page,
+ @DefaultValue("100") @QueryParam("pageSize") int pageSize) {
+ List streamingOutput = getReportPartForPillar(ReportPart.MISSING_CHECKSUM, collectionID, pillarID, page,
+ pageSize);
return new HashMap<>(Map.of(pillarID, streamingOutput));
}
@@ -202,17 +195,12 @@ public HashMap> getMissingChecksums(
@GET
@Path("/getObsoleteChecksumsFileIDs")
@Produces(MediaType.APPLICATION_JSON)
- public HashMap> geObsoleteChecksums(
- @QueryParam("collectionID")
- String collectionID,
- @QueryParam("pillarID")
- String pillarID,
- @QueryParam("page")
- int page,
- @DefaultValue("100")
- @QueryParam("pageSize")
- int pageSize) {
- List streamingOutput = getReportPartForPillar(ReportPart.OBSOLETE_CHECKSUM, collectionID, pillarID, page, pageSize);
+ public HashMap> geObsoleteChecksums(@QueryParam("collectionID") String collectionID,
+ @QueryParam("pillarID") String pillarID,
+ @QueryParam("page") int page,
+ @DefaultValue("100") @QueryParam("pageSize") int pageSize) {
+ List streamingOutput = getReportPartForPillar(ReportPart.OBSOLETE_CHECKSUM, collectionID, pillarID,
+ page, pageSize);
return new HashMap<>(Map.of(pillarID, streamingOutput));
}
@@ -226,17 +214,12 @@ public HashMap> geObsoleteChecksums(
@GET
@Path("/getChecksumErrorFileIDs")
@Produces(MediaType.APPLICATION_JSON)
- public HashMap> getChecksumErrors(
- @QueryParam("collectionID")
- String collectionID,
- @QueryParam("pillarID")
- String pillarID,
- @QueryParam("page")
- int page,
- @DefaultValue("100")
- @QueryParam("pageSize")
- int pageSize) {
- List streamingOutput = getReportPartForPillar(ReportPart.CHECKSUM_ERROR, collectionID, pillarID, page, pageSize);
+ public HashMap> getChecksumErrors(@QueryParam("collectionID") String collectionID,
+ @QueryParam("pillarID") String pillarID,
+ @QueryParam("page") int page,
+ @DefaultValue("100") @QueryParam("pageSize") int pageSize) {
+ List streamingOutput = getReportPartForPillar(ReportPart.CHECKSUM_ERROR, collectionID, pillarID, page,
+ pageSize);
return new HashMap<>(Map.of(pillarID, streamingOutput));
}
@@ -246,9 +229,7 @@ public HashMap> getChecksumErrors(
@GET
@Path("/getIntegrityStatus")
@Produces(MediaType.APPLICATION_JSON)
- public String getIntegrityStatus(
- @QueryParam("collectionID")
- String collectionID) throws IOException {
+ public String getIntegrityStatus(@QueryParam("collectionID") String collectionID) throws IOException {
StringWriter writer = new StringWriter();
JsonFactory jf = new JsonFactory();
JsonGenerator jg = jf.createGenerator(writer);
@@ -264,17 +245,16 @@ public String getIntegrityStatus(
String pillarName = Objects.requireNonNullElse(SettingsUtils.getPillarName(pillar), "N/A");
PillarType pillarTypeObject = SettingsUtils.getPillarType(pillar);
String pillarType = pillarTypeObject != null ? pillarTypeObject.value() : null;
- PillarCollectionStat emptyStat = new PillarCollectionStat(pillar, collectionID, pillarName,
- pillarType, 0L, 0L, 0L, 0L, 0L,
- 0L, "", null, new Date(0), new Date(0));
+ PillarCollectionStat emptyStat = new PillarCollectionStat(pillar, collectionID, pillarName, pillarType,
+ 0L, 0L, 0L, 0L, 0L, 0L, "", null, new Date(0), new Date(0));
stats.put(pillar, emptyStat);
}
}
jg.writeStartArray();
for (PillarCollectionStat stat : stats.values()) {
writeIntegrityStatusObject(stat, jg);
- log.debug(String.format(Locale.ROOT, "IntegrityStatus: Wrote pillar name: '%s' to pillar '%s'", stat.getPillarName(),
- stat.getPillarID()));
+ log.debug(String.format(Locale.ROOT, "IntegrityStatus: Wrote pillar name: '%s' to pillar '%s'",
+ stat.getPillarName(), stat.getPillarID()));
}
jg.writeEndArray();
jg.flush();
@@ -288,9 +268,7 @@ public String getIntegrityStatus(
@GET
@Path("/getWorkflowSetup")
@Produces(MediaType.APPLICATION_JSON)
- public String getWorkflowSetup(
- @QueryParam("collectionID")
- String collectionID) throws IOException {
+ public String getWorkflowSetup(@QueryParam("collectionID") String collectionID) throws IOException {
try {
StringWriter writer = new StringWriter();
JsonFactory jf = new JsonFactory();
@@ -315,9 +293,7 @@ public String getWorkflowSetup(
@GET
@Path("/getWorkflowList")
@Produces(MediaType.APPLICATION_JSON)
- public List getWorkflowList(
- @QueryParam("collectionID")
- String collectionID) {
+ public List getWorkflowList(@QueryParam("collectionID") String collectionID) {
List workflowIDs = new ArrayList<>();
for (JobID workflowID : workflowManager.getWorkflows(collectionID)) {
workflowIDs.add(workflowID.getWorkflowName());
@@ -325,36 +301,118 @@ public List getWorkflowList(
return workflowIDs;
}
+ /**
+ * Gets a {@link HashMap} with key {@link String} representing the {@link ReportPart} and value {@link List} of {@link String}s
+ * representing
+ * which pillars have a file for the given report part.
+ * {@link ReportPart}
+ *
+ * @param collectionID The collection ID for which the look at.
+ * @return {@link HashMap} mapping {@link ReportPart} to a {@link List} of pillars.
+ */
+ @GET
+ @Path("/getAvailableIntegrityReports")
+ @Produces(MediaType.APPLICATION_JSON)
+ public HashMap> getAvailableIntegrityReports(@QueryParam("collectionID") String collectionID) {
+ HashMap> availableIntegrityReports = new HashMap<>();
+ List pillars = SettingsUtils.getPillarIDsForCollection(collectionID);
+ Set reportParts = IntegrityReportConstants.getReportParts();
+
+ for (String pillarID : pillars) {
+ List reportPartOnPillar = new ArrayList<>();
+ for (ReportPart currentReportPart : reportParts) {
+ try {
+ getReportPart(currentReportPart, collectionID, pillarID, 0, Integer.MAX_VALUE);
+ reportPartOnPillar.add(currentReportPart.getPartName());
+ } catch (FileNotFoundException e) {
+ log.debug(e.getMessage());
+ }
+ }
+ availableIntegrityReports.put(pillarID, reportPartOnPillar);
+ }
+
+ return availableIntegrityReports;
+ }
+
/**
* Get the latest integrity report, or an error message telling no such report found.
*/
@GET
@Path("/getLatestIntegrityReport")
@Produces(MediaType.TEXT_PLAIN)
- public StreamingOutput getLatestIntegrityReport(
- @QueryParam("collectionID")
- String collectionID) {
+ public StreamingOutput getLatestIntegrityReport(@QueryParam("collectionID") String collectionID) {
final File fullReport;
try {
fullReport = integrityReportProvider.getLatestIntegrityReportReader(collectionID).getFullReport();
} catch (FileNotFoundException e) {
- throw new WebApplicationException(Response.status(Response.Status.NOT_FOUND)
- .entity(String.format(Locale.ROOT, "No integrity report for collection: '%s' found!", collectionID))
+ throw new WebApplicationException(status(Status.NOT_FOUND).entity(
+ String.format(Locale.ROOT, "No integrity report for collection: '%s' found!", collectionID))
.type(MediaType.TEXT_PLAIN).build());
}
- return output -> {
- try {
- int i;
- byte[] data = new byte[4096];
- FileInputStream is = new FileInputStream(fullReport);
- while ((i = is.read(data)) >= 0) {
- output.write(data, 0, i);
+ return output -> streamFile(fullReport, output);
+ }
+
+ @GET
+ @Path("/getIntegrityReportsAsZIP")
+ @Produces(MediaType.APPLICATION_OCTET_STREAM)
+ public Response getIntegrityReportsAsZIP(@QueryParam("collectionID") String collectionID,
+ @QueryParam("reports") List reports) {
+ String fileName = "IntegrityReports.zip";
+ HashMap files = new HashMap<>();
+
+ for (String report : reports) {
+ String[] parts = report.split("-", 2);
+ files.put(report, getLatestIntegrityReportPartFile(collectionID, parts[0], parts[1]));
+ }
+
+ StreamingOutput streamingOutput = output -> {
+ ZipOutputStream zipOut = new ZipOutputStream(output);
+ // Add the full integrity report to the hashmap
+ files.put("report", integrityReportProvider.getLatestIntegrityReportReader(collectionID).getFullReport());
+
+ // Zip each file in the files hashmap
+ files.forEach((key, value) -> {
+ try {
+ zipOut.putNextEntry(new ZipEntry(key));
+ zipOut.write(Files.readAllBytes(value.toPath()));
+ zipOut.flush();
+ } catch (IOException e) {
+ throw new WebApplicationException(status(Status.INTERNAL_SERVER_ERROR).entity(
+ "Something went wrong when trying to zip the file " + key + ".").type(MediaType.TEXT_PLAIN)
+ .build());
}
- is.close();
- } catch (Exception e) {
- throw new WebApplicationException(e);
- }
+ });
+ zipOut.close();
};
+ ResponseBuilder response = Response.ok();
+ response.type("application/zip");
+ response.header("Content-Disposition", "attachment; filename=" + fileName);
+ response.entity(streamingOutput);
+
+ return response.build();
+ }
+
+ /**
+ * Get the latest integrity report, or an error message telling no such report found.
+ *
+ * @param collectionID The collection ID.
+ * @param pillarID The pillar ID.
+ * @param reportPart The report part.
+ * @return {@link StreamingOutput} of the report part for the given collection and pillar.
+ */
+ public File getLatestIntegrityReportPartFile(String collectionID, String reportPart, String pillarID) {
+ final File reportPartFile;
+ try {
+ reportPartFile = integrityReportProvider.getIntegrityReportPart(collectionID, pillarID, reportPart);
+ } catch (FileNotFoundException e) {
+ String errorMessage = String.format(Locale.ROOT,
+ "No '%s' report part for collection: '%s' and pillar: '%s' found!", reportPart, collectionID,
+ pillarID);
+ log.error(errorMessage);
+ throw new WebApplicationException(
+ status(Status.NOT_FOUND).entity(errorMessage).type(MediaType.TEXT_PLAIN).build());
+ }
+ return reportPartFile;
}
/**
@@ -364,11 +422,8 @@ public StreamingOutput getLatestIntegrityReport(
@Path("/startWorkflow")
@Consumes("application/x-www-form-urlencoded")
@Produces("text/html")
- public String startWorkflow(
- @FormParam("workflowID")
- String workflowID,
- @FormParam("collectionID")
- String collectionID) {
+ public String startWorkflow(@FormParam("workflowID") String workflowID,
+ @FormParam("collectionID") String collectionID) {
log.debug("Starting workflow '" + workflowID + "' on collection '" + collectionID + "'.");
return workflowManager.startWorkflow(new JobID(workflowID, collectionID));
}
@@ -379,9 +434,7 @@ public String startWorkflow(
@GET
@Path("/getCollectionInformation")
@Produces(MediaType.APPLICATION_JSON)
- public String getCollectionInformation(
- @QueryParam("collectionID")
- String collectionID) throws IOException {
+ public String getCollectionInformation(@QueryParam("collectionID") String collectionID) throws IOException {
StringWriter writer = new StringWriter();
JsonFactory jf = new JsonFactory();
JsonGenerator jg = jf.createGenerator(writer);
@@ -435,14 +488,18 @@ private List getReportPart(ReportPart part, String collectionID, String
*
* @return Returns either a {@link List} of fileIDs or throws a {@link WebApplicationException}.
*/
- private List getReportPartForPillar(ReportPart part, String collectionID, String pillarID, int page, int pageSize) {
+ private List getReportPartForPillar(ReportPart part,
+ String collectionID,
+ String pillarID,
+ int page,
+ int pageSize) {
List reportPartContent;
try {
reportPartContent = getReportPart(part, collectionID, pillarID, page, pageSize);
} catch (FileNotFoundException e) {
- throw new WebApplicationException(Response.status(Response.Status.NOT_FOUND)
- .entity(String.format(Locale.ROOT, "No integrity '%s' report part for collection: '%s' and pillar: '%s' found!",
- part.getHumanString(), collectionID, pillarID)).type(MediaType.TEXT_PLAIN).build());
+ throw new WebApplicationException(status(Status.NOT_FOUND).entity(String.format(Locale.ROOT,
+ "No integrity '%s' report part for collection: '%s' and pillar: '%s' found!", part.getHumanString(),
+ collectionID, pillarID)).type(MediaType.TEXT_PLAIN).build());
}
return reportPartContent;
}
@@ -460,7 +517,10 @@ private List getReportPartForPillar(ReportPart part, String collectionID
* @param pageSize The paging size.
* @return Returns a {@link List} containing the files that were also missing on the given pillar.
*/
- private List compareMissingFiles(List missingOnPillar, String collectionID, String pillar, int pageSize) {
+ private List compareMissingFiles(List missingOnPillar,
+ String collectionID,
+ String pillar,
+ int pageSize) {
List batchToCheck;
List agreedMissingFileIDs = new ArrayList<>();
for (String missingFileID : missingOnPillar) {
@@ -505,6 +565,26 @@ private int getOffset(int page, int pageSize) {
return (page - 1) * pageSize;
}
+ /**
+ * Streams the given file, allowing it to be downloaded on REST api call.
+ *
+ * @param file The {@link File} to stream.
+ * @param output The {@link OutputStream} to write the stream to.
+ */
+ private void streamFile(File file, OutputStream output) {
+ try {
+ int i;
+ byte[] data = new byte[4096];
+ FileInputStream is = new FileInputStream(file);
+ while ((i = is.read(data)) >= 0) {
+ output.write(data, 0, i);
+ }
+ is.close();
+ } catch (Exception e) {
+ throw new WebApplicationException(e);
+ }
+ }
+
private void writeIntegrityStatusObject(PillarCollectionStat stat, JsonGenerator jg) throws IOException {
jg.writeStartObject();
jg.writeObjectField("pillarID", stat.getPillarID());
diff --git a/bitrepository-reference-pillar/src/main/java/org/bitrepository/pillar/Pillar.java b/bitrepository-reference-pillar/src/main/java/org/bitrepository/pillar/Pillar.java
index 9734045ad..a77ef963d 100644
--- a/bitrepository-reference-pillar/src/main/java/org/bitrepository/pillar/Pillar.java
+++ b/bitrepository-reference-pillar/src/main/java/org/bitrepository/pillar/Pillar.java
@@ -90,8 +90,8 @@ public Pillar(MessageBus messageBus, Settings settings, StorageModel pillarModel
*/
private void initializeWorkflows() {
Long interval = DEFAULT_RECALCULATION_WORKFLOW_TIME;
- Duration recalculateOldChecksumsInterval =
- settings.getReferenceSettings().getPillarSettings().getRecalculateOldChecksumsInterval();
+ Duration recalculateOldChecksumsInterval = settings.getReferenceSettings().getPillarSettings()
+ .getRecalculateOldChecksumsInterval();
if (recalculateOldChecksumsInterval != null) {
interval = XmlUtils.xmlDurationToMilliseconds(recalculateOldChecksumsInterval);
}
diff --git a/bitrepository-webclient/src/main/webapp/download-modal.js b/bitrepository-webclient/src/main/webapp/download-modal.js
new file mode 100644
index 000000000..4c3195782
--- /dev/null
+++ b/bitrepository-webclient/src/main/webapp/download-modal.js
@@ -0,0 +1,150 @@
+/*
+ * #%L
+ * Bitrepository Webclient
+ * %%
+ * Copyright (C) 2010 - 2013 The State and University Library, The Royal Library and The State Archives, Denmark
+ * %%
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as
+ * published by the Free Software Foundation, either version 2.1 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Lesser Public License for more details.
+ *
+ * You should have received a copy of the GNU General Lesser Public
+ * License along with this program. If not, see
+ * .
+ * #L%
+ */
+
+function DownloadModal(collectionID, contentElement, url) {
+ this.url = url;
+ this.collectionID = collectionID;
+
+ this.getModal = function () {
+ let self = this;
+ $.getJSON(`${self.url}/getAvailableIntegrityReports?collectionID=${self.collectionID}`, {}, function (json) {
+ let html = `
`;
+ html += `Select the reports you wish to download. If none are selected, only the latest full integrity report will be downloaded.`;
+ html += `
`;
+
+ // Create "select all" checkbox and "reload table" button.
+ html += `
`;
+ html += `Select all: `;
+ html += `🔄`;
+ html += `
`;
+
+ // Create table
+ html += `
`;
+
+ // Populate the header of the table.
+ html += ``;
+ html += `
`;
+ html += `
Pillar ID
`;
+ html += `
Missing Files
`;
+ html += `
Missing Checksums
`;
+ html += `
Obsolete Checksums
`;
+ html += `
Inconsistent Checksums
`;
+ html += `
Deleted Files
`;
+ html += `
`;
+ html += ``;
+
+ // Init table body
+ html += ``;
+ // Populate the table body
+ html += getTableBody(json);
+
+ html += ``;
+ html += `
`;
+
+ // Init download button
+ html += `
`;
+ let zipURL = `${self.url}/getIntegrityReportsAsZIP?collectionID=${self.collectionID}`;
+ html += `Download Reports`;
+ html += `
`;
+
+ // Assign content and apply listener functions.
+ $(contentElement).html(html);
+ updateDownloadLink(zipURL);
+ enableOnClickFunctionality(self);
+ }).fail(function () {
+ let html = "
";
+ html += "Failed to load page";
+ html += "
";
+ $(contentElement).html(html);
+ });
+ }
+
+ function getTableBody(json) {
+ let html = ``;
+ let pillars = Object.keys(json);
+
+ for (let i = 0; i < pillars.length; i++) {
+ html += `
`;
+ // Pillar information
+ html += `
${pillars[i]}
`;
+
+ // Integrity Information
+ html += getReportPartTD(json, pillars[i], "missingFile");
+ html += getReportPartTD(json, pillars[i], "missingChecksum");
+ html += getReportPartTD(json, pillars[i], "obsoleteChecksum");
+ html += getReportPartTD(json, pillars[i], "checksumIssue");
+ html += getReportPartTD(json, pillars[i], "deletedFile");
+
+ html += `
`;
+ }
+
+ return html;
+ }
+
+ function getReportPartTD(json, pillarID, reportPart) {
+ let html = "";
+ if (json[pillarID].includes(reportPart)) {
+ html += `
+
`;
+ } else {
+ html += `
`;
+ }
+ return html;
+ }
+
+ function updateDownloadLink(zipURL) {
+ $("a[class=download-button]").on("click", function (e) {
+ e.originalEvent.currentTarget.href = zipURL + getSelected();
+ });
+ }
+
+ function getSelected() {
+ let output = "";
+ $("input:checkbox[name=report-checkbox]:checked").each(function () {
+ output += `&reports=${$(this).val()}`;
+ });
+
+ return output;
+ }
+
+ function enableOnClickFunctionality(self) {
+ // On click refresh table
+ $("span#refresh").on("click", function () {
+ self.getModal();
+ });
+
+ // On click either select all or deselect all
+ $("input:checkbox[name=select-all]").change(function () {
+ if (this.checked) {
+ changeAllCheckboxes(true);
+ } else {
+ changeAllCheckboxes(false);
+ }
+ });
+ }
+
+ function changeAllCheckboxes(bool) {
+ $("input:checkbox").each(function () {
+ $(this).prop("checked", bool);
+ });
+ }
+}
diff --git a/bitrepository-webclient/src/main/webapp/flot/jquery.flot.pie.js b/bitrepository-webclient/src/main/webapp/flot/jquery.flot.pie.js
index ff06a92a6..2cecbbddc 100644
--- a/bitrepository-webclient/src/main/webapp/flot/jquery.flot.pie.js
+++ b/bitrepository-webclient/src/main/webapp/flot/jquery.flot.pie.js
@@ -613,7 +613,7 @@ More detail and specific examples can be found in the included HTML file.
arrPoly = [[0, 0], [p1X, p1Y], [p2X, p2Y], [p3X, p3Y], [p4X, p4Y], [p5X, p5Y]],
arrPoint = [x, y];
- // TODO: perhaps do some mathmatical trickery here with the Y-coordinate to compensate for pie tilt?
+ // TODO: perhaps do some mathematical trickery here with the Y-coordinate to compensate for pie tilt?
if (isPointInPoly(arrPoly, arrPoint)) {
ctx.restore();
diff --git a/bitrepository-webclient/src/main/webapp/integrity-service.html b/bitrepository-webclient/src/main/webapp/integrity-service.html
index b29a85bec..b8b4a715a 100644
--- a/bitrepository-webclient/src/main/webapp/integrity-service.html
+++ b/bitrepository-webclient/src/main/webapp/integrity-service.html
@@ -111,6 +111,7 @@
Modal header
+
@@ -119,6 +120,7 @@
Modal header
let pillarIntegrityStatuses = {};
let workflows = {};
let modal;
+ let downloadModal;
let updatePageInterval;
let nameMapper;
let integrityServiceUrl;
@@ -240,14 +242,28 @@
Modal header
}
}
+ function showDownloadModal(integrityURL) {
+ return function () {
+ downloadModal = new DownloadModal(getCollectionID(), "#modalBody", integrityURL);
+ $("#modalLabel").html("Select Reports to Download");
+ $("#modalBody").html("
Loading
");
+ downloadModal.getModal();
+ $("#modalDialog").modal('show');
+ }
+ }
+
function getBodyContext(id, type) {
let context = {};
context.url = integrityServiceUrl + "/integrity/IntegrityService/";
if (type === "Pillar Name") {
context.element = id + "-pillarName";
+ context.title = type + " on " + id.toUpperCase();
+ context.propertyName = "pillarName";
} else if (type === "Pillar Type") {
context.element = id + "-pillarType";
+ context.title = type + " on " + id.toUpperCase();
+ context.propertyName = "pillarType";
} else if (type === "Total files") {
context.element = id + "-totalFileCount";
context.title = type + " on " + id.toUpperCase();
@@ -388,8 +404,10 @@
Modal header
function collectionChanged(collectionID) {
clearContent();
$("#integrityLegend").html("Integrity information for collection " + nameMapper.getName(collectionID));
- let reportUrl = `${integrityServiceUrl}/integrity/IntegrityService/getLatestIntegrityReport?collectionID=${collectionID}&workflowID=CompleteIntegrityCheck`;
- $("#integrityReportGetter").html(`Get latest integrity report`);
+ let reportUrl = `${integrityServiceUrl}/integrity/IntegrityService`;
+ $("#integrityReportGetter").html(`Download latest integrity reports`).on("click",
+ showDownloadModal(reportUrl));
+
clearInterval(updatePageInterval);
loadWorkflows();
getCollectionInformation(collectionID);