Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -50,20 +50,27 @@ public class FileUpdater {
* tuple is not already found in the database
*/
private final String insertFileInfoSql =
"INSERT INTO fileinfo (" + " collectionID, pillarID, fileID, filesize, file_timestamp, last_seen_getfileids)" +
" (SELECT collectionID, ?, ?, ?, ?, ? FROM collections" + " WHERE collectionID = ? " + " AND NOT EXISTS (" +
" SELECT * FROM fileinfo " + " WHERE fileID = ?" + " AND collectionID = ?" + " AND pillarID = ?))";

private final String updateFileInfoSql =
"UPDATE fileinfo " + " SET filesize = ?," + " file_timestamp = ?," + " last_seen_getfileids = ?" + " WHERE fileID = ?" +
" AND collectionID = ?" + " AND pillarID = ?";

private final String insertLatestFileTime = "INSERT INTO collection_progress " + "(collectionID, pillarID, latest_file_timestamp)" +
" ( SELECT collectionID, ?, ? FROM collections" + " WHERE collectionID = ?" +
" AND NOT EXISTS ( SELECT * FROM collection_progress" + " WHERE collectionID = ?" + " AND pillarID = ?))";
"INSERT INTO fileinfo (collectionID, pillarID, fileID, filesize, file_timestamp, last_seen_getfileids)" +
" (SELECT collectionID, ?, ?, ?, ?, ? FROM collections" +
" WHERE collectionID = ? " +
" AND NOT EXISTS (" +
" SELECT * FROM fileinfo " +
" WHERE fileID = ?" +
" AND collectionID = ?" +
" AND pillarID = ?))";

private final String updateFileInfoSql = "UPDATE fileinfo " +
" SET filesize = ?, file_timestamp = ?, last_seen_getfileids = ?" +
" WHERE fileID = ? AND collectionID = ? AND pillarID = ?";

private final String insertLatestFileTime =
"INSERT INTO collection_progress (collectionID, pillarID, latest_file_timestamp)" +
" (SELECT collectionID, ?, ? FROM collections" +
" WHERE collectionID = ? AND NOT EXISTS" +
" (SELECT * FROM collection_progress WHERE collectionID = ? AND pillarID = ?))";

private final String updateLatestFileTime =
"UPDATE collection_progress" + " SET latest_file_timestamp = ? " + " WHERE collectionID = ?" + " AND pillarID = ?";
"UPDATE collection_progress SET latest_file_timestamp = ? WHERE collectionID = ? AND pillarID = ?";

private final Logger log = LoggerFactory.getLogger(getClass());

Expand Down

Large diffs are not rendered by default.

Original file line number Diff line number Diff line change
Expand Up @@ -49,8 +49,8 @@ public class StatisticsCreator {
+ " (SELECT MAX(stat_key), ?, ?, ?, ? FROM stats WHERE collectionID = ?)";

private final String insertPillarStatEntrySql = "INSERT INTO pillarstats"
+ " (stat_key, pillarID, file_count, file_size, missing_files_count, "
+ "checksum_errors_count, missing_checksums_count, obsolete_checksums_count, oldest_checksum_timestamp)"
+ " (stat_key, pillarID, file_count, file_size, missing_files_count,"
+ " checksum_errors_count, missing_checksums_count, obsolete_checksums_count, oldest_checksum_timestamp)"
+ " (SELECT MAX(stat_key), ?, ?, ?, ?, ?, ?, ?, ? FROM stats WHERE collectionID = ?)";

private final Logger log = LoggerFactory.getLogger(getClass());
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -253,8 +253,8 @@ public String getIntegrityStatus(@QueryParam("collectionID") String collectionID
jg.writeStartArray();
for (PillarCollectionStat stat : stats.values()) {
writeIntegrityStatusObject(stat, jg);
log.debug(String.format(Locale.ROOT, "IntegrityStatus: Wrote pillar name: '%s' to pillar '%s'",
stat.getPillarName(), stat.getPillarID()));
log.debug("IntegrityStatus: Wrote pillar name: '{}' to pillar '{}'", stat.getPillarName(),
stat.getPillarID());
}
jg.writeEndArray();
jg.flush();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -41,20 +41,21 @@ public CompleteIntegrityCheck() {}

@Override
public String getDescription() {
return "Retrieves all fileIDs and checksums from all pillars and checks for all potential integrity " + "problems.";
return "Retrieves all fileIDs and checksums from all pillars and checks for all potential integrity problems.";
}

@Override
protected UpdateFileIDsStep getUpdateFileIDsStep() {
return new FullUpdateFileIDsStep(context.getCollector(), context.getStore(), context.getAlerter(), context.getSettings(),
collectionID, integrityContributors);
return new FullUpdateFileIDsStep(context.getCollector(), context.getStore(), context.getAlerter(),
context.getSettings(), collectionID, integrityContributors);
}


@Override
protected UpdateChecksumsStep getUpdateChecksumsStep() {
return new FullUpdateChecksumsStep(context.getCollector(), context.getStore(), context.getAlerter(),
ChecksumUtils.getDefault(context.getSettings()), context.getSettings(), collectionID, integrityContributors);
ChecksumUtils.getDefault(context.getSettings()), context.getSettings(), collectionID,
integrityContributors);
}


Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -53,15 +53,15 @@
* And finally it is verified whether any missing or obsolete checksums can be found.
*/
public abstract class IntegrityCheckWorkflow extends Workflow {
/**
* The default number of retries if none is set in ReferenceSettings
*/
private static final int DEFAULT_MAX_RETRIES = 3;
private final Logger log = LoggerFactory.getLogger(getClass());
protected IntegrityWorkflowContext context;
protected String collectionID;
protected IntegrityContributors integrityContributors;
protected Date workflowStart;
/**
* The default number of retries if none is set in ReferenceSettings
*/
private static final int DEFAULT_MAX_RETRIES = 3;

/**
* Remember to call the initialise method needs to be called before the start method.
Expand All @@ -85,21 +85,20 @@ public void initialise(WorkflowContext context, String collectionID) {

@Override
public void start() {

workflowStart = new Date();

if (context == null) {
throw new IllegalStateException("The workflow can not be started before the initialise method has been " +
"called.");
throw new IllegalStateException(
"The workflow can not be started before the initialise method has been " + "called.");
}
IntegrityReporter reporter = new BasicIntegrityReporter(jobID.getCollectionID(), jobID.getWorkflowName(),
IntegrityServiceManager.getIntegrityReportStorageDir());

super.start();
try {
StatisticsCollector statisticsCollector = new StatisticsCollector(collectionID);
Integer maxRetries
= context.getSettings().getReferenceSettings().getIntegrityServiceSettings().getComponentRetries();
Integer maxRetries = context.getSettings().getReferenceSettings().getIntegrityServiceSettings()
.getComponentRetries();
integrityContributors = new IntegrityContributors(SettingsUtils.getPillarIDsForCollection(collectionID),
maxRetries == null ? DEFAULT_MAX_RETRIES : maxRetries);

Expand All @@ -112,35 +111,33 @@ public void start() {
performStep(updateChecksumStep);

if (cleanDeletedFiles()) {
HandleDeletedFilesStep handleDeletedFilesStep = new HandleDeletedFilesStep(context.getStore(),
reporter, workflowStart, integrityContributors.getFinishedContributors());
HandleDeletedFilesStep handleDeletedFilesStep = new HandleDeletedFilesStep(context.getStore(), reporter,
workflowStart, integrityContributors.getFinishedContributors());
performStep(handleDeletedFilesStep);
}

statisticsCollector.getCollectionStat().setStatsTime(new Date());
javax.xml.datatype.Duration timeBeforeMissingFileCheck =
context.getSettings().getReferenceSettings().getIntegrityServiceSettings().getTimeBeforeMissingFileCheck();
javax.xml.datatype.Duration timeBeforeMissingFileCheck = context.getSettings().getReferenceSettings()
.getIntegrityServiceSettings().getTimeBeforeMissingFileCheck();
Duration missingFileGracePeriod = XmlUtils.xmlDurationToDuration(timeBeforeMissingFileCheck);
HandleMissingFilesStep handleMissingFilesStep = new HandleMissingFilesStep(context.getStore(), reporter,
statisticsCollector, missingFileGracePeriod);
performStep(handleMissingFilesStep);

HandleChecksumValidationStep handleChecksumValidationStep
= new HandleChecksumValidationStep(context.getStore(), context.getAuditManager(), reporter,
statisticsCollector);
HandleChecksumValidationStep handleChecksumValidationStep = new HandleChecksumValidationStep(
context.getStore(), context.getAuditManager(), reporter, statisticsCollector);
performStep(handleChecksumValidationStep);

HandleMissingChecksumsStep handleMissingChecksumsStep = new HandleMissingChecksumsStep(context.getStore(),
reporter, statisticsCollector, getChecksumUpdateCutoffDate());
performStep(handleMissingChecksumsStep);

HandleObsoleteChecksumsStep handleObsoleteChecksumsStep
= new HandleObsoleteChecksumsStep(context.getSettings(), context.getStore(), reporter,
statisticsCollector);
HandleObsoleteChecksumsStep handleObsoleteChecksumsStep = new HandleObsoleteChecksumsStep(
context.getSettings(), context.getStore(), reporter, statisticsCollector);
performStep(handleObsoleteChecksumsStep);

CreateStatisticsEntryStep createStatistics = new CreateStatisticsEntryStep(
context.getStore(), collectionID, statisticsCollector);
CreateStatisticsEntryStep createStatistics = new CreateStatisticsEntryStep(context.getStore(), collectionID,
statisticsCollector);
performStep(createStatistics);

if (currentState() != WorkflowState.ABORTED) {
Expand All @@ -149,7 +146,8 @@ public void start() {
}
try {
reporter.generateReport();
IntegrityServiceManager.getIntegrityReportProvider().setLatestReport(collectionID, reporter.getReportDir());
IntegrityServiceManager.getIntegrityReportProvider()
.setLatestReport(collectionID, reporter.getReportDir());
} catch (IOException e) {
log.error("Failed to generate integrity report", e);
context.getAlerter().integrityComponentFailure("Failed to generate integrity report", collectionID);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -52,8 +52,8 @@ public class HandleChecksumValidationStep extends AbstractWorkFlowStep {
private Long allPillarChecksumErrors = 0L;
private Long collectionChecksumErrors = 0L;

public HandleChecksumValidationStep(IntegrityModel store, AuditTrailManager auditManager, IntegrityReporter reporter,
StatisticsCollector statisticsCollector) {
public HandleChecksumValidationStep(IntegrityModel store, AuditTrailManager auditManager,
IntegrityReporter reporter, StatisticsCollector statisticsCollector) {
this.store = store;
this.auditManager = auditManager;
this.reporter = reporter;
Expand All @@ -71,11 +71,14 @@ public String getName() {
}

/**
* Queries the IntegrityModel for missing files on each pillar. Reports them if any is returned.
* Queries the IntegrityModel for inconsistent checksums in the collection.
* Checks every reported inconsistent checksum, to verify that it's actually inconsistent.
* Updates database model to reflect the discovered situation.
*/
@Override
public synchronized void performStep() throws StepFailedException {
try (IntegrityIssueIterator inconsistentFilesIterator = store.getFilesWithInconsistentChecksums(reporter.getCollectionID())) {
try (IntegrityIssueIterator inconsistentFilesIterator = store.getFilesWithInconsistentChecksums(
reporter.getCollectionID())) {
String fileID;
while ((fileID = inconsistentFilesIterator.getNextIntegrityIssue()) != null) {
handleChecksumInconsistency(store.getFileInfos(fileID, reporter.getCollectionID()), fileID);
Expand Down Expand Up @@ -120,7 +123,7 @@ private void handleChecksumInconsistency(Collection<FileInfo> infos, String file
}

/**
* Creates a audit-trail for inconsistency between checksums.
* Creates an audit-trail for inconsistency between checksums.
* If only one pillar is alone with a checksum compared to all the others, then it is pointed out at the possible
* cause.
*
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -41,14 +41,14 @@ public class HandleMissingChecksumsStep extends AbstractWorkFlowStep {
private final IntegrityModel store;
private final IntegrityReporter reporter;
private final StatisticsCollector sc;
private final Date cutoff;
private final Date cutoffDate;

public HandleMissingChecksumsStep(IntegrityModel store, IntegrityReporter reporter, StatisticsCollector statisticsCollector,
Date latestChecksumUpdate) {
this.store = store;
this.reporter = reporter;
this.sc = statisticsCollector;
this.cutoff = latestChecksumUpdate;
this.cutoffDate = latestChecksumUpdate;
}

@Override
Expand All @@ -72,7 +72,7 @@ public synchronized void performStep() throws StepFailedException {

String missingFile;
try (IntegrityIssueIterator missingChecksumsIterator = store.findFilesWithMissingChecksum(reporter.getCollectionID(), pillar,
cutoff)) {
cutoffDate)) {
while ((missingFile = missingChecksumsIterator.getNextIntegrityIssue()) != null) {
try {
reporter.reportMissingChecksum(missingFile, pillar);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -55,8 +55,8 @@ public class HandleMissingFilesStep extends AbstractWorkFlowStep {
private final StatisticsCollector sc;
private final Duration gracePeriod;

public HandleMissingFilesStep(IntegrityModel store, IntegrityReporter reporter, StatisticsCollector statisticsCollector,
Duration missingFileGracePeriod) {
public HandleMissingFilesStep(IntegrityModel store, IntegrityReporter reporter,
StatisticsCollector statisticsCollector, Duration missingFileGracePeriod) {
this.store = store;
this.reporter = reporter;
this.sc = statisticsCollector;
Expand All @@ -69,9 +69,7 @@ public String getName() {
}

/**
* Queries the IntegrityModel for inconsistent checksums in the collection.
* Checks every reported inconsistent checksum, to verify that it's actually inconsistent.
* Updates database model to reflect the discovered situation.
* Queries the IntegrityModel for missing files on each pillar. Reports them if any is returned.
*/
@Override
public synchronized void performStep() throws StepFailedException {
Expand All @@ -82,10 +80,11 @@ public synchronized void performStep() throws StepFailedException {
}
Instant missingAfterInstant = Instant.now().minus(gracePeriod);
Date missingAfterDate = Date.from(missingAfterInstant);
log.info("Looking for missing files, files need to be older than {} to be considered missing.", missingAfterDate);
log.info("Looking for missing files, files need to be older than {} to be considered missing.",
missingAfterDate);

try (IntegrityIssueIterator issueIterator = store.findFilesWithMissingCopies(reporter.getCollectionID(), pillars.size(), 0L,
Long.MAX_VALUE)) {
try (IntegrityIssueIterator issueIterator = store.findFilesWithMissingCopies(reporter.getCollectionID(),
pillars.size(), 0L, Long.MAX_VALUE)) {

String missingFile;
while ((missingFile = issueIterator.getNextIntegrityIssue()) != null) {
Expand Down
Loading