From eec993c1fd6a5ec23cbc985d8f9e525a63b24f90 Mon Sep 17 00:00:00 2001 From: bohlski Date: Mon, 26 Sep 2022 16:35:04 +0200 Subject: [PATCH 1/7] Redid readability indentations that was undone by auto-formatting --- .../cache/database/FileUpdater.java | 31 +++--- .../cache/database/IntegrityDAO.java | 101 +++++++++++------- .../cache/database/StatisticsCreator.java | 4 +- .../web/RestIntegrityService.java | 4 +- 4 files changed, 85 insertions(+), 55 deletions(-) diff --git a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/database/FileUpdater.java b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/database/FileUpdater.java index 902726956..a5ea1598f 100644 --- a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/database/FileUpdater.java +++ b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/database/FileUpdater.java @@ -50,20 +50,27 @@ public class FileUpdater { * tuple is not already found in the database */ private final String insertFileInfoSql = - "INSERT INTO fileinfo (" + " collectionID, pillarID, fileID, filesize, file_timestamp, last_seen_getfileids)" + - " (SELECT collectionID, ?, ?, ?, ?, ? FROM collections" + " WHERE collectionID = ? " + " AND NOT EXISTS (" + - " SELECT * FROM fileinfo " + " WHERE fileID = ?" + " AND collectionID = ?" + " AND pillarID = ?))"; - - private final String updateFileInfoSql = - "UPDATE fileinfo " + " SET filesize = ?," + " file_timestamp = ?," + " last_seen_getfileids = ?" + " WHERE fileID = ?" + - " AND collectionID = ?" + " AND pillarID = ?"; - - private final String insertLatestFileTime = "INSERT INTO collection_progress " + "(collectionID, pillarID, latest_file_timestamp)" + - " ( SELECT collectionID, ?, ? FROM collections" + " WHERE collectionID = ?" + - " AND NOT EXISTS ( SELECT * FROM collection_progress" + " WHERE collectionID = ?" + " AND pillarID = ?))"; + "INSERT INTO fileinfo (collectionID, pillarID, fileID, filesize, file_timestamp, last_seen_getfileids)" + + " (SELECT collectionID, ?, ?, ?, ?, ? FROM collections" + + " WHERE collectionID = ? " + + " AND NOT EXISTS (" + + " SELECT * FROM fileinfo " + + " WHERE fileID = ?" + + " AND collectionID = ?" + + " AND pillarID = ?))"; + + private final String updateFileInfoSql = "UPDATE fileinfo " + + " SET filesize = ?, file_timestamp = ?, last_seen_getfileids = ?" + + " WHERE fileID = ? AND collectionID = ? AND pillarID = ?"; + + private final String insertLatestFileTime = + "INSERT INTO collection_progress (collectionID, pillarID, latest_file_timestamp)" + + " (SELECT collectionID, ?, ? FROM collections" + + " WHERE collectionID = ? AND NOT EXISTS" + + " (SELECT * FROM collection_progress WHERE collectionID = ? AND pillarID = ?))"; private final String updateLatestFileTime = - "UPDATE collection_progress" + " SET latest_file_timestamp = ? " + " WHERE collectionID = ?" + " AND pillarID = ?"; + "UPDATE collection_progress SET latest_file_timestamp = ? WHERE collectionID = ? AND pillarID = ?"; private final Logger log = LoggerFactory.getLogger(getClass()); diff --git a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/database/IntegrityDAO.java b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/database/IntegrityDAO.java index ac406051e..ce92582b5 100644 --- a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/database/IntegrityDAO.java +++ b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/database/IntegrityDAO.java @@ -156,7 +156,8 @@ public Date getLatestFileDate(String collectionID, String pillarID) { ArgumentValidator.checkNotNullOrEmpty(pillarID, "String pillarID"); ArgumentValidator.checkNotNullOrEmpty(collectionID, "String collectionID"); - String retrieveSql = "SELECT latest_file_timestamp FROM collection_progress" + " WHERE collectionID = ? " + " AND pillarID = ?"; + String retrieveSql = "SELECT latest_file_timestamp FROM collection_progress" + + " WHERE collectionID = ? AND pillarID = ?"; Long time = DatabaseUtils.selectFirstLongValue(dbConnector, retrieveSql, collectionID, pillarID); return (time == null ? null : new Date(time)); @@ -171,7 +172,7 @@ public Date getLatestFileDate(String collectionID, String pillarID) { public Date getLatestFileDateInCollection(String collectionID) { ArgumentValidator.checkNotNullOrEmpty(collectionID, "String collectionID"); - String retrieveSql = "SELECT MAX(latest_file_timestamp) FROM collection_progress" + " WHERE collectionID = ?"; + String retrieveSql = "SELECT MAX(latest_file_timestamp) FROM collection_progress WHERE collectionID = ?"; Long time = DatabaseUtils.selectFirstLongValue(dbConnector, retrieveSql, collectionID); return (time == null ? null : new Date(time)); @@ -188,7 +189,9 @@ public Date getLatestChecksumDate(String collectionID, String pillarID) { ArgumentValidator.checkNotNullOrEmpty(pillarID, "String pillarID"); ArgumentValidator.checkNotNullOrEmpty(collectionID, "String collectionID"); - String retrieveSql = "SELECT latest_checksum_timestamp FROM collection_progress" + " WHERE collectionID = ? " + " AND pillarID = ?"; + String retrieveSql = "SELECT latest_checksum_timestamp FROM collection_progress" + + " WHERE collectionID = ? " + + " AND pillarID = ?"; Long time = DatabaseUtils.selectFirstLongValue(dbConnector, retrieveSql, collectionID, pillarID); return (time == null ? null : new Date(time)); } @@ -200,7 +203,9 @@ public Date getLatestChecksumDate(String collectionID, String pillarID) { */ public void resetFileCollectionProgress(String collectionID) { ArgumentValidator.checkNotNullOrEmpty(collectionID, "String collectionID"); - String resetSql = "UPDATE collection_progress" + " SET latest_file_timestamp = NULL" + " WHERE collectionID = ?"; + String resetSql = "UPDATE collection_progress" + + " SET latest_file_timestamp = NULL" + + " WHERE collectionID = ?"; DatabaseUtils.executeStatement(dbConnector, resetSql, collectionID); } @@ -212,7 +217,9 @@ public void resetFileCollectionProgress(String collectionID) { */ public void resetChecksumCollectionProgress(String collectionID) { ArgumentValidator.checkNotNullOrEmpty(collectionID, "String collectionID"); - String resetSql = "UPDATE collection_progress" + " SET latest_checksum_timestamp = NULL" + " WHERE collectionID = ?"; + String resetSql = "UPDATE collection_progress" + + " SET latest_checksum_timestamp = NULL" + + " WHERE collectionID = ?"; DatabaseUtils.executeStatement(dbConnector, resetSql, collectionID); } @@ -230,8 +237,10 @@ public IntegrityIssueIterator getFilesWithOutdatedChecksums(String collectionID, ArgumentValidator.checkNotNullOrEmpty(collectionID, "String collectionID"); ArgumentValidator.checkNotNull(maxDate, "Date maxDate"); - String retrieveSql = - "SELECT fileID from fileinfo" + " WHERE collectionID = ?" + " AND pillarID = ?" + " AND checksum_timestamp < ?"; + String retrieveSql = "SELECT fileID FROM fileinfo" + + " WHERE collectionID = ?" + + " AND pillarID = ?" + + " AND checksum_timestamp < ?"; return makeIntegrityIssueIterator(retrieveSql, collectionID, pillarID, maxDate.getTime()); } @@ -250,8 +259,10 @@ public IntegrityIssueIterator getFilesWithMissingChecksums(String collectionID, ArgumentValidator.checkNotNullOrEmpty(collectionID, "String collectionID"); ArgumentValidator.checkNotNullOrEmpty(pillarID, "String pillarID"); - String retrieveSql = "SELECT fileID from fileinfo" + " WHERE collectionID = ?" + " AND pillarID = ?" + " AND (checksum is NULL" + - " OR last_seen_getchecksums < ?)"; + String retrieveSql = "SELECT fileID FROM fileinfo" + + " WHERE collectionID = ?" + + " AND pillarID = ?" + + " AND (checksum IS NULL OR last_seen_getchecksums < ?)"; return makeIntegrityIssueIterator(retrieveSql, collectionID, pillarID, cutoffDate.getTime()); } @@ -269,8 +280,10 @@ public IntegrityIssueIterator getOrphanFilesOnPillar(String collectionID, String ArgumentValidator.checkNotNullOrEmpty(pillarID, "String pillarID"); ArgumentValidator.checkNotNull(cutoffDate, "Date cutoffDate"); - String findOrphansSql = - "SELECT fileID from fileinfo" + " WHERE collectionID = ?" + " AND pillarID = ?" + " AND last_seen_getfileids < ?"; + String findOrphansSql = "SELECT fileID FROM fileinfo" + + " WHERE collectionID = ?" + + " AND pillarID = ?" + + " AND last_seen_getfileids < ?"; return makeIntegrityIssueIterator(findOrphansSql, collectionID, pillarID, cutoffDate.getTime()); } @@ -287,7 +300,9 @@ public void removeFile(String collectionID, String pillarID, String fileID) { ArgumentValidator.checkNotNullOrEmpty(pillarID, "String pillarID"); ArgumentValidator.checkNotNullOrEmpty(fileID, "String fileID"); - String removeSql = "DELETE FROM fileinfo" + " WHERE collectionID = ?" + " AND pillarID = ?" + " AND fileID = ?"; + String removeSql = "DELETE FROM fileinfo" + + " WHERE collectionID = ?" + + " AND pillarID = ? AND fileID = ?"; DatabaseUtils.executeStatement(dbConnector, removeSql, collectionID, pillarID, fileID); } @@ -328,8 +343,12 @@ public IntegrityIssueIterator findFilesWithChecksumInconsistencies(String collec ArgumentValidator.checkNotNullOrEmpty(collectionID, "String collectionID"); String findInconsistentChecksumsSql = - "SELECT fileID FROM (" + " SELECT fileID, count(distinct(checksum)) as checksums FROM fileinfo" + - " WHERE collectionID = ?" + " GROUP BY fileID) as subselect" + " WHERE checksums > 1"; + "SELECT fileID FROM (" + + " SELECT fileID, COUNT(distinct(checksum)) AS checksums" + + " FROM fileinfo" + + " WHERE collectionID = ?" + + " GROUP BY fileID) AS subselect" + + " WHERE checksums > 1"; return makeIntegrityIssueIterator(findInconsistentChecksumsSql, collectionID); } @@ -392,8 +411,9 @@ public List getFileInfosForFile(String fileID, String collectionID) { List res = new ArrayList<>(); String getFileInfoSql = "SELECT pillarID, filesize, checksum, file_timestamp," + - " checksum_timestamp, last_seen_getfileids, last_seen_getchecksums FROM fileinfo" + " WHERE collectionID = ?" + - " AND fileID = ?"; + " checksum_timestamp, last_seen_getfileids, last_seen_getchecksums" + + " FROM fileinfo" + + " WHERE collectionID = ? AND fileID = ?"; try (Connection conn = dbConnector.getConnection(); PreparedStatement ps = DatabaseUtils.createPreparedStatement(conn, getFileInfoSql, collectionID, fileID)) { @@ -407,16 +427,16 @@ public List getFileInfosForFile(String fileID, String collectionID) { Date lastSeenGetFileIDs = new Date(dbResult.getLong("last_seen_getfileids")); Date lastSeenGetChecksums = new Date(dbResult.getLong("last_seen_getchecksums")); - FileInfo f = new FileInfo(fileID, CalendarUtils.getXmlGregorianCalendar(lastFileCheck), checksum, fileSize, - CalendarUtils.getXmlGregorianCalendar(lastChecksumCheck), pillarID); + FileInfo f = new FileInfo(fileID, CalendarUtils.getXmlGregorianCalendar(lastFileCheck), checksum, + fileSize, CalendarUtils.getXmlGregorianCalendar(lastChecksumCheck), pillarID); f.setLastSeenGetFileIDs(lastSeenGetFileIDs); f.setLastSeenGetChecksums(lastSeenGetChecksums); res.add(f); } } } catch (SQLException e) { - throw new IllegalStateException("Could not retrieve the FileInfo for '" + fileID + "' with the SQL '" + getFileInfoSql + "'.", - e); + throw new IllegalStateException("Could not retrieve the FileInfo for '" + fileID + "' with the SQL '" + + getFileInfoSql + "'.", e); } return res; } @@ -436,7 +456,7 @@ public void createStatistics(String collectionID, StatisticsCollector statistics /** - * Method to retrieves the metrics for the pillars in the given collection + * Method to retrieve the metrics for the pillars in the given collection * I.e. the summed filesize and file count per pillar * * @param collectionID The ID of the collection to get metrics for @@ -446,8 +466,8 @@ public void createStatistics(String collectionID, StatisticsCollector statistics public Map getPillarCollectionMetrics(String collectionID) { Map metrics = new HashMap<>(); String selectSql = - "SELECT pillarID, COUNT(fileID) as filecount, SUM(filesize) as sizesum," + - " MIN(checksum_timestamp) as oldest_checksum_timestamp" + + "SELECT pillarID, COUNT(fileID) AS filecount, SUM(filesize) AS sizesum," + + " MIN(checksum_timestamp) AS oldest_checksum_timestamp" + " FROM fileinfo" + " WHERE collectionID = ?" + " GROUP BY pillarID"; @@ -467,9 +487,8 @@ public Map getPillarCollectionMetrics(String col metrics.put(pillarID, metric); } } catch (SQLException e) { - throw new IllegalStateException( - "Could not retrieve PillarCollectionMetrics for collection '" + collectionID + "' with the SQL '" + selectSql + "'.", - e); + throw new IllegalStateException("Could not retrieve PillarCollectionMetrics for collection '" + + collectionID + "' with the SQL '" + selectSql + "'.", e); } return metrics; @@ -485,7 +504,8 @@ public long getCollectionSize(String collectionID) { ArgumentValidator.checkNotNullOrEmpty(collectionID, "String collectionID"); String getCollectionSizeSql = - "SELECT SUM(filesize) FROM " + "(SELECT distinct(fileID), filesize from fileinfo" + " WHERE collectionID = ?) AS subselect"; + "SELECT SUM(filesize) FROM " + + "(SELECT distinct(fileID), filesize FROM fileinfo WHERE collectionID = ?) AS subselect"; Long size = DatabaseUtils.selectFirstLongValue(dbConnector, getCollectionSizeSql, collectionID); return (size == null ? 0 : size); } @@ -499,7 +519,7 @@ public long getCollectionSize(String collectionID) { public Long getNumberOfFilesInCollection(String collectionID) { ArgumentValidator.checkNotNullOrEmpty(collectionID, "String collectionID"); - String getNumberOfFilesSql = "SELECT COUNT(DISTINCT(fileid)) FROM fileinfo" + " WHERE collectionID = ?"; + String getNumberOfFilesSql = "SELECT COUNT(DISTINCT(fileid)) FROM fileinfo WHERE collectionID = ?"; return DatabaseUtils.selectFirstLongValue(dbConnector, getNumberOfFilesSql, collectionID); } @@ -550,7 +570,7 @@ public List getLatestPillarStats(String collectionID) { } } catch (SQLException e) { throw new IllegalStateException( - "Could not retrieve the latest PillarCollectionStat's for '" + collectionID + "' " + "with the SQL '" + + "Could not retrieve the latest PillarCollectionStat's for '" + collectionID + "' with the SQL '" + latestPillarStatsSql + "'.", e); } @@ -594,7 +614,8 @@ public List getLatestCollectionStats(String collectionID, int co String latestCollectionStatSql = getLatestCollectionStatsSql(); try (Connection conn = dbConnector.getConnection(); - PreparedStatement ps = DatabaseUtils.createPreparedStatement(conn, latestCollectionStatSql, collectionID, count)) { + PreparedStatement ps = DatabaseUtils.createPreparedStatement( + conn, latestCollectionStatSql, collectionID, count)) { try (ResultSet dbResult = ps.executeQuery()) { while (dbResult.next()) { Long fileCount = dbResult.getLong("file_count"); @@ -604,15 +625,15 @@ public List getLatestCollectionStats(String collectionID, int co Date statsTime = new Date(dbResult.getLong("stat_time")); Date updateTime = new Date(dbResult.getLong("last_update")); - CollectionStat stat = new CollectionStat(collectionID, fileCount, dataSize, checksumErrors, latestFile, statsTime, - updateTime); + CollectionStat stat = new CollectionStat(collectionID, fileCount, dataSize, checksumErrors, + latestFile, statsTime, updateTime); stats.add(stat); } } } catch (SQLException e) { - throw new IllegalStateException( - "Could not retrieve the latest PillarStat's for '" + collectionID + "' " + "with the SQL '" + latestCollectionStatSql + - "' with arguments '" + Arrays.asList(collectionID, count) + "'.", e); + throw new IllegalStateException("Could not retrieve the latest PillarStat's for '" + collectionID + + "' with the SQL '" + latestCollectionStatSql + + "' with arguments '" + Arrays.asList(collectionID, count) + "'.", e); } java.util.Collections.reverse(stats); return stats; @@ -629,8 +650,10 @@ public Date getEarliestFileDate(String collectionID, String fileID) { ArgumentValidator.checkNotNullOrEmpty(collectionID, "String collectionID"); ArgumentValidator.checkNotNullOrEmpty(fileID, "String fileID"); - String getEarliestFileDateSql = "SELECT MIN(file_timestamp) FROM fileinfo" + " WHERE collectionID = ?" + " AND fileID = ?"; - long time = Optional.ofNullable(DatabaseUtils.selectFirstLongValue(dbConnector, getEarliestFileDateSql, collectionID, fileID)) + String getEarliestFileDateSql = "SELECT MIN(file_timestamp) FROM fileinfo" + + " WHERE collectionID = ? AND fileID = ?"; + long time = Optional.ofNullable( + DatabaseUtils.selectFirstLongValue(dbConnector, getEarliestFileDateSql, collectionID, fileID)) .orElse(0L); return new Date(time); } @@ -643,8 +666,8 @@ private IntegrityIssueIterator makeIntegrityIssueIterator(String query, Object.. ps = DatabaseUtils.createPreparedStatement(conn, query, args); return new IntegrityIssueIterator(ps); } catch (Exception e) { - throw new IllegalStateException( - "Failed to create IntegrityIssueIterator for query '" + query + "' with arguments" + Arrays.asList(args), e); + throw new IllegalStateException("Failed to create IntegrityIssueIterator for query '" + query + + "' with arguments" + Arrays.asList(args), e); } } } diff --git a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/database/StatisticsCreator.java b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/database/StatisticsCreator.java index 0ab1e389a..8211759a9 100644 --- a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/database/StatisticsCreator.java +++ b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/database/StatisticsCreator.java @@ -49,8 +49,8 @@ public class StatisticsCreator { + " (SELECT MAX(stat_key), ?, ?, ?, ? FROM stats WHERE collectionID = ?)"; private final String insertPillarStatEntrySql = "INSERT INTO pillarstats" - + " (stat_key, pillarID, file_count, file_size, missing_files_count, " - + "checksum_errors_count, missing_checksums_count, obsolete_checksums_count, oldest_checksum_timestamp)" + + " (stat_key, pillarID, file_count, file_size, missing_files_count," + + " checksum_errors_count, missing_checksums_count, obsolete_checksums_count, oldest_checksum_timestamp)" + " (SELECT MAX(stat_key), ?, ?, ?, ?, ?, ?, ?, ? FROM stats WHERE collectionID = ?)"; private final Logger log = LoggerFactory.getLogger(getClass()); diff --git a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/web/RestIntegrityService.java b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/web/RestIntegrityService.java index e169b96f6..9a2082020 100644 --- a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/web/RestIntegrityService.java +++ b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/web/RestIntegrityService.java @@ -273,8 +273,8 @@ public String getIntegrityStatus( jg.writeStartArray(); for (PillarCollectionStat stat : stats.values()) { writeIntegrityStatusObject(stat, jg); - log.debug(String.format(Locale.ROOT, "IntegrityStatus: Wrote pillar name: '%s' to pillar '%s'", stat.getPillarName(), - stat.getPillarID())); + log.debug("IntegrityStatus: Wrote pillar name: '{}' to pillar '{}'", stat.getPillarName(), + stat.getPillarID()); } jg.writeEndArray(); jg.flush(); From ef20f8512e4c57c296f1a175b2306c89047945b6 Mon Sep 17 00:00:00 2001 From: bohlski Date: Mon, 26 Sep 2022 16:57:17 +0200 Subject: [PATCH 2/7] Some auto-format/readability fixes --- .../workflow/IntegrityCheckWorkflow.java | 30 +++++++++---------- .../step/HandleChecksumValidationStep.java | 7 +++-- .../step/HandleMissingChecksumsStep.java | 6 ++-- 3 files changed, 21 insertions(+), 22 deletions(-) diff --git a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/workflow/IntegrityCheckWorkflow.java b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/workflow/IntegrityCheckWorkflow.java index 05babead5..cfba3f1b6 100644 --- a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/workflow/IntegrityCheckWorkflow.java +++ b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/workflow/IntegrityCheckWorkflow.java @@ -53,15 +53,15 @@ * And finally it is verified whether any missing or obsolete checksums can be found. */ public abstract class IntegrityCheckWorkflow extends Workflow { + /** + * The default number of retries if none is set in ReferenceSettings + */ + private static final int DEFAULT_MAX_RETRIES = 3; private final Logger log = LoggerFactory.getLogger(getClass()); protected IntegrityWorkflowContext context; protected String collectionID; protected IntegrityContributors integrityContributors; protected Date workflowStart; - /** - * The default number of retries if none is set in ReferenceSettings - */ - private static final int DEFAULT_MAX_RETRIES = 3; /** * Remember to call the initialise method needs to be called before the start method. @@ -85,7 +85,6 @@ public void initialise(WorkflowContext context, String collectionID) { @Override public void start() { - workflowStart = new Date(); if (context == null) { @@ -98,8 +97,8 @@ public void start() { super.start(); try { StatisticsCollector statisticsCollector = new StatisticsCollector(collectionID); - Integer maxRetries - = context.getSettings().getReferenceSettings().getIntegrityServiceSettings().getComponentRetries(); + Integer maxRetries = context.getSettings().getReferenceSettings().getIntegrityServiceSettings() + .getComponentRetries(); integrityContributors = new IntegrityContributors(SettingsUtils.getPillarIDsForCollection(collectionID), maxRetries == null ? DEFAULT_MAX_RETRIES : maxRetries); @@ -118,25 +117,23 @@ public void start() { } statisticsCollector.getCollectionStat().setStatsTime(new Date()); - javax.xml.datatype.Duration timeBeforeMissingFileCheck = - context.getSettings().getReferenceSettings().getIntegrityServiceSettings().getTimeBeforeMissingFileCheck(); + javax.xml.datatype.Duration timeBeforeMissingFileCheck = context.getSettings().getReferenceSettings() + .getIntegrityServiceSettings().getTimeBeforeMissingFileCheck(); Duration missingFileGracePeriod = XmlUtils.xmlDurationToDuration(timeBeforeMissingFileCheck); HandleMissingFilesStep handleMissingFilesStep = new HandleMissingFilesStep(context.getStore(), reporter, statisticsCollector, missingFileGracePeriod); performStep(handleMissingFilesStep); - HandleChecksumValidationStep handleChecksumValidationStep - = new HandleChecksumValidationStep(context.getStore(), context.getAuditManager(), reporter, - statisticsCollector); + HandleChecksumValidationStep handleChecksumValidationStep = new HandleChecksumValidationStep( + context.getStore(), context.getAuditManager(), reporter, statisticsCollector); performStep(handleChecksumValidationStep); HandleMissingChecksumsStep handleMissingChecksumsStep = new HandleMissingChecksumsStep(context.getStore(), reporter, statisticsCollector, getChecksumUpdateCutoffDate()); performStep(handleMissingChecksumsStep); - HandleObsoleteChecksumsStep handleObsoleteChecksumsStep - = new HandleObsoleteChecksumsStep(context.getSettings(), context.getStore(), reporter, - statisticsCollector); + HandleObsoleteChecksumsStep handleObsoleteChecksumsStep = new HandleObsoleteChecksumsStep( + context.getSettings(), context.getStore(), reporter, statisticsCollector); performStep(handleObsoleteChecksumsStep); CreateStatisticsEntryStep createStatistics = new CreateStatisticsEntryStep( @@ -149,7 +146,8 @@ public void start() { } try { reporter.generateReport(); - IntegrityServiceManager.getIntegrityReportProvider().setLatestReport(collectionID, reporter.getReportDir()); + IntegrityServiceManager.getIntegrityReportProvider() + .setLatestReport(collectionID, reporter.getReportDir()); } catch (IOException e) { log.error("Failed to generate integrity report", e); context.getAlerter().integrityComponentFailure("Failed to generate integrity report", collectionID); diff --git a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/workflow/step/HandleChecksumValidationStep.java b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/workflow/step/HandleChecksumValidationStep.java index 6480bf704..977e2469c 100644 --- a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/workflow/step/HandleChecksumValidationStep.java +++ b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/workflow/step/HandleChecksumValidationStep.java @@ -52,8 +52,8 @@ public class HandleChecksumValidationStep extends AbstractWorkFlowStep { private Long allPillarChecksumErrors = 0L; private Long collectionChecksumErrors = 0L; - public HandleChecksumValidationStep(IntegrityModel store, AuditTrailManager auditManager, IntegrityReporter reporter, - StatisticsCollector statisticsCollector) { + public HandleChecksumValidationStep(IntegrityModel store, AuditTrailManager auditManager, + IntegrityReporter reporter, StatisticsCollector statisticsCollector) { this.store = store; this.auditManager = auditManager; this.reporter = reporter; @@ -75,7 +75,8 @@ public String getName() { */ @Override public synchronized void performStep() throws StepFailedException { - try (IntegrityIssueIterator inconsistentFilesIterator = store.getFilesWithInconsistentChecksums(reporter.getCollectionID())) { + try (IntegrityIssueIterator inconsistentFilesIterator = store.getFilesWithInconsistentChecksums( + reporter.getCollectionID())) { String fileID; while ((fileID = inconsistentFilesIterator.getNextIntegrityIssue()) != null) { handleChecksumInconsistency(store.getFileInfos(fileID, reporter.getCollectionID()), fileID); diff --git a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/workflow/step/HandleMissingChecksumsStep.java b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/workflow/step/HandleMissingChecksumsStep.java index bc0339467..f3d48efcf 100644 --- a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/workflow/step/HandleMissingChecksumsStep.java +++ b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/workflow/step/HandleMissingChecksumsStep.java @@ -41,14 +41,14 @@ public class HandleMissingChecksumsStep extends AbstractWorkFlowStep { private final IntegrityModel store; private final IntegrityReporter reporter; private final StatisticsCollector sc; - private final Date cutoff; + private final Date cutoffDate; public HandleMissingChecksumsStep(IntegrityModel store, IntegrityReporter reporter, StatisticsCollector statisticsCollector, Date latestChecksumUpdate) { this.store = store; this.reporter = reporter; this.sc = statisticsCollector; - this.cutoff = latestChecksumUpdate; + this.cutoffDate = latestChecksumUpdate; } @Override @@ -72,7 +72,7 @@ public synchronized void performStep() throws StepFailedException { String missingFile; try (IntegrityIssueIterator missingChecksumsIterator = store.findFilesWithMissingChecksum(reporter.getCollectionID(), pillar, - cutoff)) { + cutoffDate)) { while ((missingFile = missingChecksumsIterator.getNextIntegrityIssue()) != null) { try { reporter.reportMissingChecksum(missingFile, pillar); From 341a7787d0710047afb204e2f439e06c77f8d831 Mon Sep 17 00:00:00 2001 From: bohlski Date: Mon, 26 Sep 2022 16:59:43 +0200 Subject: [PATCH 3/7] Fixed what seemed like a copy-pasted javadoc --- .../workflow/step/HandleChecksumValidationStep.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/workflow/step/HandleChecksumValidationStep.java b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/workflow/step/HandleChecksumValidationStep.java index 977e2469c..9b6cf69d6 100644 --- a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/workflow/step/HandleChecksumValidationStep.java +++ b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/workflow/step/HandleChecksumValidationStep.java @@ -71,7 +71,7 @@ public String getName() { } /** - * Queries the IntegrityModel for missing files on each pillar. Reports them if any is returned. + * Queries the IntegrityModel for inconsistent checksums on each pillar. Reports them if any is returned. */ @Override public synchronized void performStep() throws StepFailedException { @@ -121,7 +121,7 @@ private void handleChecksumInconsistency(Collection infos, String file } /** - * Creates a audit-trail for inconsistency between checksums. + * Creates an audit-trail for inconsistency between checksums. * If only one pillar is alone with a checksum compared to all the others, then it is pointed out at the possible * cause. * From 40515d25528d0a04001f098bee00ec44234322b3 Mon Sep 17 00:00:00 2001 From: bohlski Date: Tue, 27 Sep 2022 13:38:19 +0200 Subject: [PATCH 4/7] Small typo --- .../integrityservice/workflow/step/UpdateChecksumsStep.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/workflow/step/UpdateChecksumsStep.java b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/workflow/step/UpdateChecksumsStep.java index 073f30b33..edc526466 100644 --- a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/workflow/step/UpdateChecksumsStep.java +++ b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/workflow/step/UpdateChecksumsStep.java @@ -133,7 +133,7 @@ private void handleFailureEvent(OperationEvent event) throws WorkflowAbortedExce integrityContributors.getFailedContributors(), collectionID); throw new WorkflowAbortedException("Aborting workflow due to failure collecting checksums. " + "Cause: " + ofe.toString()); } else { - log.info("Failure occured collecting fileIDs, continuing collecting checksums. Failure {}", ofe.toString()); + log.info("Failure occurred collecting fileIDs, continuing collecting checksums. Failure {}", ofe.toString()); alerter.integrityFailed("Failure while collecting checksums, the check will continue " + "with the information available. The failed contributors were: " + integrityContributors.getFailedContributors(), collectionID); From 0e3969082080c893af627062f9b8ec17e0309976 Mon Sep 17 00:00:00 2001 From: bohlski Date: Tue, 27 Sep 2022 13:41:31 +0200 Subject: [PATCH 5/7] Auto-format UpdateChecksumsStep --- .../workflow/step/UpdateChecksumsStep.java | 32 +++++++++++-------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/workflow/step/UpdateChecksumsStep.java b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/workflow/step/UpdateChecksumsStep.java index edc526466..1e9e7fef1 100644 --- a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/workflow/step/UpdateChecksumsStep.java +++ b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/workflow/step/UpdateChecksumsStep.java @@ -49,16 +49,16 @@ * The step for collecting the checksums of all files from all pillars. */ public abstract class UpdateChecksumsStep extends AbstractWorkFlowStep { + protected final IntegrityModel store; + protected final String collectionID; private final Logger log = LoggerFactory.getLogger(getClass()); private final IntegrityInformationCollector collector; - protected final IntegrityModel store; private final ChecksumSpecTYPE checksumType; private final IntegrityAlerter alerter; private final Duration timeout; private final Integer maxNumberOfResultsPerConversation; - protected final String collectionID; - private boolean abortInCaseOfFailure = true; private final IntegrityContributors integrityContributors; + private boolean abortInCaseOfFailure = true; /** * @param collector The client for collecting the checksums. @@ -67,8 +67,8 @@ public abstract class UpdateChecksumsStep extends AbstractWorkFlowStep { * @param checksumType The type of checksum to collect. */ public UpdateChecksumsStep(IntegrityInformationCollector collector, IntegrityModel store, IntegrityAlerter alerter, - ChecksumSpecTYPE checksumType, Settings settings, String collectionID, - IntegrityContributors integrityContributors) { + ChecksumSpecTYPE checksumType, Settings settings, String collectionID, + IntegrityContributors integrityContributors) { this.collector = collector; this.store = store; this.checksumType = checksumType; @@ -78,7 +78,8 @@ public UpdateChecksumsStep(IntegrityInformationCollector collector, IntegrityMod this.timeout = settings.getIdentificationTimeout().plus(settings.getOperationTimeout()); this.maxNumberOfResultsPerConversation = SettingsUtils.getMaxClientPageSize(); if (settings.getReferenceSettings().getIntegrityServiceSettings().isSetAbortOnFailedContributor()) { - abortInCaseOfFailure = settings.getReferenceSettings().getIntegrityServiceSettings().isAbortOnFailedContributor(); + abortInCaseOfFailure = settings.getReferenceSettings().getIntegrityServiceSettings() + .isAbortOnFailedContributor(); } } @@ -98,11 +99,14 @@ public synchronized void performStep() throws WorkflowAbortedException { initialStepAction(); Set pillarsToCollectFrom = integrityContributors.getActiveContributors(); - log.debug("Collecting checksums from '" + pillarsToCollectFrom + "' for collection '" + collectionID + "'."); + log.debug( + "Collecting checksums from '" + pillarsToCollectFrom + "' for collection '" + collectionID + "'."); while (!pillarsToCollectFrom.isEmpty()) { - IntegrityCollectorEventHandler eventHandler = new IntegrityCollectorEventHandler(store, timeout, integrityContributors); + IntegrityCollectorEventHandler eventHandler = new IntegrityCollectorEventHandler(store, timeout, + integrityContributors); ContributorQuery[] queries = getQueries(pillarsToCollectFrom); - collector.getChecksums(collectionID, pillarsToCollectFrom, checksumType, null, "IntegrityService: " + getName(), queries, + collector.getChecksums(collectionID, pillarsToCollectFrom, checksumType, null, + "IntegrityService: " + getName(), queries, eventHandler); OperationEvent event = eventHandler.getFinish(); @@ -131,12 +135,14 @@ private void handleFailureEvent(OperationEvent event) throws WorkflowAbortedExce if (abortInCaseOfFailure) { alerter.integrityFailed("Integrity check aborted while getting checksums due to failed contributors: " + integrityContributors.getFailedContributors(), collectionID); - throw new WorkflowAbortedException("Aborting workflow due to failure collecting checksums. " + "Cause: " + ofe.toString()); + throw new WorkflowAbortedException("Aborting workflow due to failure collecting checksums. " + + "Cause: " + ofe.toString()); } else { - log.info("Failure occurred collecting fileIDs, continuing collecting checksums. Failure {}", ofe.toString()); + log.info("Failure occurred collecting fileIDs, continuing collecting checksums. Failure {}", + ofe.toString()); alerter.integrityFailed("Failure while collecting checksums, the check will continue " + - "with the information available. The failed contributors were: " + integrityContributors.getFailedContributors(), - collectionID); + "with the information available. The failed contributors were: " + + integrityContributors.getFailedContributors(), collectionID); } } } From 32a9b1eef2d41d82f2e3f180b9c1ae3671d333b1 Mon Sep 17 00:00:00 2001 From: bohlski Date: Wed, 28 Sep 2022 14:43:31 +0200 Subject: [PATCH 6/7] Javadoc fix for HandleMissingFiles- and HandleChecksumValidationStep --- .../workflow/step/HandleChecksumValidationStep.java | 4 +++- .../workflow/step/HandleMissingFilesStep.java | 4 +--- .../java/org/bitrepository/pillar/store/FileStorageModel.java | 3 +-- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/workflow/step/HandleChecksumValidationStep.java b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/workflow/step/HandleChecksumValidationStep.java index 9b6cf69d6..f0baa1c9a 100644 --- a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/workflow/step/HandleChecksumValidationStep.java +++ b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/workflow/step/HandleChecksumValidationStep.java @@ -71,7 +71,9 @@ public String getName() { } /** - * Queries the IntegrityModel for inconsistent checksums on each pillar. Reports them if any is returned. + * Queries the IntegrityModel for inconsistent checksums in the collection. + * Checks every reported inconsistent checksum, to verify that it's actually inconsistent. + * Updates database model to reflect the discovered situation. */ @Override public synchronized void performStep() throws StepFailedException { diff --git a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/workflow/step/HandleMissingFilesStep.java b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/workflow/step/HandleMissingFilesStep.java index 35571dbc8..6c2492a5c 100644 --- a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/workflow/step/HandleMissingFilesStep.java +++ b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/workflow/step/HandleMissingFilesStep.java @@ -69,9 +69,7 @@ public String getName() { } /** - * Queries the IntegrityModel for inconsistent checksums in the collection. - * Checks every reported inconsistent checksum, to verify that it's actually inconsistent. - * Updates database model to reflect the discovered situation. + * Queries the IntegrityModel for missing files on each pillar. Reports them if any is returned. */ @Override public synchronized void performStep() throws StepFailedException { diff --git a/bitrepository-reference-pillar/src/main/java/org/bitrepository/pillar/store/FileStorageModel.java b/bitrepository-reference-pillar/src/main/java/org/bitrepository/pillar/store/FileStorageModel.java index a5c3b7e50..80df927a7 100644 --- a/bitrepository-reference-pillar/src/main/java/org/bitrepository/pillar/store/FileStorageModel.java +++ b/bitrepository-reference-pillar/src/main/java/org/bitrepository/pillar/store/FileStorageModel.java @@ -78,8 +78,7 @@ public FileStorageModel(FileStore archives, ChecksumStore cache, AlarmDispatcher } @Override - public void verifyFileToCacheConsistencyOfAllDataIfRequired( - String collectionID) { + public void verifyFileToCacheConsistencyOfAllDataIfRequired(String collectionID) { Boolean verify = settings.getReferenceSettings().getPillarSettings().isVerifyDataConsistencyOnMessage(); if (verify != null && verify) { verifyFileToCacheConsistencyOfAllData(collectionID); From b23b813961f7da8e4cff89247e538c8d6c731ffa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mathias=20S=C3=B8by?= Date: Thu, 29 Sep 2022 10:37:24 +0200 Subject: [PATCH 7/7] Small changes. --- .../workflow/CompleteIntegrityCheck.java | 9 +-- .../workflow/IntegrityCheckWorkflow.java | 12 ++-- .../workflow/step/HandleMissingFilesStep.java | 11 ++-- .../pillar/store/FileStorageModel.java | 66 +++++++++---------- .../pillar/store/StorageModel.java | 59 ++++++++++------- 5 files changed, 85 insertions(+), 72 deletions(-) diff --git a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/workflow/CompleteIntegrityCheck.java b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/workflow/CompleteIntegrityCheck.java index 3f8175280..12a0e3fa9 100644 --- a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/workflow/CompleteIntegrityCheck.java +++ b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/workflow/CompleteIntegrityCheck.java @@ -41,20 +41,21 @@ public CompleteIntegrityCheck() {} @Override public String getDescription() { - return "Retrieves all fileIDs and checksums from all pillars and checks for all potential integrity " + "problems."; + return "Retrieves all fileIDs and checksums from all pillars and checks for all potential integrity problems."; } @Override protected UpdateFileIDsStep getUpdateFileIDsStep() { - return new FullUpdateFileIDsStep(context.getCollector(), context.getStore(), context.getAlerter(), context.getSettings(), - collectionID, integrityContributors); + return new FullUpdateFileIDsStep(context.getCollector(), context.getStore(), context.getAlerter(), + context.getSettings(), collectionID, integrityContributors); } @Override protected UpdateChecksumsStep getUpdateChecksumsStep() { return new FullUpdateChecksumsStep(context.getCollector(), context.getStore(), context.getAlerter(), - ChecksumUtils.getDefault(context.getSettings()), context.getSettings(), collectionID, integrityContributors); + ChecksumUtils.getDefault(context.getSettings()), context.getSettings(), collectionID, + integrityContributors); } diff --git a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/workflow/IntegrityCheckWorkflow.java b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/workflow/IntegrityCheckWorkflow.java index cfba3f1b6..5202ace4f 100644 --- a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/workflow/IntegrityCheckWorkflow.java +++ b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/workflow/IntegrityCheckWorkflow.java @@ -88,8 +88,8 @@ public void start() { workflowStart = new Date(); if (context == null) { - throw new IllegalStateException("The workflow can not be started before the initialise method has been " + - "called."); + throw new IllegalStateException( + "The workflow can not be started before the initialise method has been " + "called."); } IntegrityReporter reporter = new BasicIntegrityReporter(jobID.getCollectionID(), jobID.getWorkflowName(), IntegrityServiceManager.getIntegrityReportStorageDir()); @@ -111,8 +111,8 @@ public void start() { performStep(updateChecksumStep); if (cleanDeletedFiles()) { - HandleDeletedFilesStep handleDeletedFilesStep = new HandleDeletedFilesStep(context.getStore(), - reporter, workflowStart, integrityContributors.getFinishedContributors()); + HandleDeletedFilesStep handleDeletedFilesStep = new HandleDeletedFilesStep(context.getStore(), reporter, + workflowStart, integrityContributors.getFinishedContributors()); performStep(handleDeletedFilesStep); } @@ -136,8 +136,8 @@ public void start() { context.getSettings(), context.getStore(), reporter, statisticsCollector); performStep(handleObsoleteChecksumsStep); - CreateStatisticsEntryStep createStatistics = new CreateStatisticsEntryStep( - context.getStore(), collectionID, statisticsCollector); + CreateStatisticsEntryStep createStatistics = new CreateStatisticsEntryStep(context.getStore(), collectionID, + statisticsCollector); performStep(createStatistics); if (currentState() != WorkflowState.ABORTED) { diff --git a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/workflow/step/HandleMissingFilesStep.java b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/workflow/step/HandleMissingFilesStep.java index 6c2492a5c..0a27513d4 100644 --- a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/workflow/step/HandleMissingFilesStep.java +++ b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/workflow/step/HandleMissingFilesStep.java @@ -55,8 +55,8 @@ public class HandleMissingFilesStep extends AbstractWorkFlowStep { private final StatisticsCollector sc; private final Duration gracePeriod; - public HandleMissingFilesStep(IntegrityModel store, IntegrityReporter reporter, StatisticsCollector statisticsCollector, - Duration missingFileGracePeriod) { + public HandleMissingFilesStep(IntegrityModel store, IntegrityReporter reporter, + StatisticsCollector statisticsCollector, Duration missingFileGracePeriod) { this.store = store; this.reporter = reporter; this.sc = statisticsCollector; @@ -80,10 +80,11 @@ public synchronized void performStep() throws StepFailedException { } Instant missingAfterInstant = Instant.now().minus(gracePeriod); Date missingAfterDate = Date.from(missingAfterInstant); - log.info("Looking for missing files, files need to be older than {} to be considered missing.", missingAfterDate); + log.info("Looking for missing files, files need to be older than {} to be considered missing.", + missingAfterDate); - try (IntegrityIssueIterator issueIterator = store.findFilesWithMissingCopies(reporter.getCollectionID(), pillars.size(), 0L, - Long.MAX_VALUE)) { + try (IntegrityIssueIterator issueIterator = store.findFilesWithMissingCopies(reporter.getCollectionID(), + pillars.size(), 0L, Long.MAX_VALUE)) { String missingFile; while ((missingFile = issueIterator.getNextIntegrityIssue()) != null) { diff --git a/bitrepository-reference-pillar/src/main/java/org/bitrepository/pillar/store/FileStorageModel.java b/bitrepository-reference-pillar/src/main/java/org/bitrepository/pillar/store/FileStorageModel.java index 80df927a7..347952048 100644 --- a/bitrepository-reference-pillar/src/main/java/org/bitrepository/pillar/store/FileStorageModel.java +++ b/bitrepository-reference-pillar/src/main/java/org/bitrepository/pillar/store/FileStorageModel.java @@ -63,6 +63,8 @@ */ public class FileStorageModel extends StorageModel { private final Logger log = LoggerFactory.getLogger(getClass()); + private final Boolean verify = settings.getReferenceSettings().getPillarSettings() + .isVerifyDataConsistencyOnMessage(); /** * @param archives The file archives. @@ -71,15 +73,14 @@ public class FileStorageModel extends StorageModel { * @param settings The settings. * @param fileExchange The file exchange. */ - public FileStorageModel(FileStore archives, ChecksumStore cache, AlarmDispatcher alarmDispatcher, - Settings settings, FileExchange fileExchange) { + public FileStorageModel(FileStore archives, ChecksumStore cache, AlarmDispatcher alarmDispatcher, Settings settings, + FileExchange fileExchange) { super(archives, cache, alarmDispatcher, settings, fileExchange); log.info("Instantiating the FileStorageModel: " + getPillarID()); } @Override public void verifyFileToCacheConsistencyOfAllDataIfRequired(String collectionID) { - Boolean verify = settings.getReferenceSettings().getPillarSettings().isVerifyDataConsistencyOnMessage(); if (verify != null && verify) { verifyFileToCacheConsistencyOfAllData(collectionID); } @@ -87,7 +88,6 @@ public void verifyFileToCacheConsistencyOfAllDataIfRequired(String collectionID) @Override protected void verifyFileToCacheConsistencyIfRequired(String fileID, String collectionID) { - Boolean verify = settings.getReferenceSettings().getPillarSettings().isVerifyDataConsistencyOnMessage(); if (verify != null && verify) { recalculateChecksum(fileID, collectionID); } @@ -106,7 +106,7 @@ public FileInfo getFileInfoForActualFile(String fileID, String collectionID) { @Override public ExtractedFileIDsResultSet getFileIDsResultSet(String fileID, XMLGregorianCalendar minTimestamp, - XMLGregorianCalendar maxTimestamp, Long maxResults, String collectionID) { + XMLGregorianCalendar maxTimestamp, Long maxResults, String collectionID) { Long minTime = null; if (minTimestamp != null) { minTime = CalendarUtils.convertFromXMLGregorianCalendar(minTimestamp).getTime(); @@ -131,20 +131,20 @@ public ExtractedFileIDsResultSet getFileIDsResultSet(String fileID, XMLGregorian } @Override - public void verifyEnoughFreeSpaceLeftForFile(Long fileSize, - String collectionID) throws RequestHandlerException { - long useableSizeLeft = fileArchive.sizeLeftInArchive(collectionID) - - settings.getReferenceSettings().getPillarSettings().getMinimumSizeLeft(); - - if (useableSizeLeft < fileSize) { - throw new IdentifyContributorException(ResponseCode.FAILURE, "Not enough space left in this pillar. " - + "Requires '" + fileSize + "' but has only '" + useableSizeLeft + "'"); + public void verifyEnoughFreeSpaceLeftForFile(Long fileSize, String collectionID) throws RequestHandlerException { + long usableSizeLeft = fileArchive.sizeLeftInArchive(collectionID) - + settings.getReferenceSettings().getPillarSettings().getMinimumSizeLeft(); + + if (usableSizeLeft < fileSize) { + throw new IdentifyContributorException(ResponseCode.FAILURE, + "Not enough space left in this pillar. " + "Requires '" + fileSize + "' but has only '" + + usableSizeLeft + "'"); } } @Override - protected ExtractedChecksumResultSet getNonDefaultChecksumResultSet( - Long maxResults, String collectionID, ChecksumSpecTYPE csSpec) { + protected ExtractedChecksumResultSet getNonDefaultChecksumResultSet(Long maxResults, String collectionID, + ChecksumSpecTYPE csSpec) { ExtractedChecksumResultSet res = new ExtractedChecksumResultSet(); long i = 0; @@ -162,11 +162,10 @@ protected ExtractedChecksumResultSet getNonDefaultChecksumResultSet( } @Override - public void verifyFileExists(String fileID, String collectionID) - throws RequestHandlerException { + public void verifyFileExists(String fileID, String collectionID) throws RequestHandlerException { if (!hasFileID(fileID, collectionID)) { - log.warn("The file '" + fileID + "' has been requested, but we do not have that file in collection '" - + collectionID + "'!"); + log.warn("The file '" + fileID + "' has been requested, but we do not have that file in collection '" + + collectionID + "'!"); throw new InvalidMessageException(ResponseCode.FILE_NOT_FOUND_FAILURE, "File not found."); } } @@ -179,7 +178,7 @@ public ChecksumSpecTYPE getChecksumPillarSpec() { @Override public void putFile(String collectionID, String fileID, String fileAddress, - ChecksumDataForFileTYPE expectedChecksum) throws RequestHandlerException { + ChecksumDataForFileTYPE expectedChecksum) throws RequestHandlerException { transferFileToTmp(fileID, collectionID, fileAddress); verifyFileInTmp(fileID, collectionID, expectedChecksum); fileArchive.moveToArchive(fileID, collectionID); @@ -188,7 +187,7 @@ public void putFile(String collectionID, String fileID, String fileAddress, @Override public void replaceFile(String fileID, String collectionID, String fileAddress, - ChecksumDataForFileTYPE expectedChecksum) throws RequestHandlerException { + ChecksumDataForFileTYPE expectedChecksum) throws RequestHandlerException { transferFileToTmp(fileID, collectionID, fileAddress); verifyFileInTmp(fileID, collectionID, expectedChecksum); fileArchive.replaceFile(fileID, collectionID); @@ -231,7 +230,7 @@ private String getChecksumForTempFile(String fileID, String collectionID, Checks * @return The requested file ids. */ private ExtractedFileIDsResultSet getFileIds(Long minTime, Long maxTime, Long maxNumberOfResults, - String collectionID) { + String collectionID) { ExtractedFileIDsResultSet res = new ExtractedFileIDsResultSet(); // Map between lastModifiedDate and fileInfo. @@ -286,12 +285,12 @@ public void verifyFileToCacheConsistencyOfAllData(String collectionID) { */ private void verifyCacheToArchiveConsistencyForFile(String fileID, String collectionID) { if (!fileArchive.hasFile(fileID, collectionID)) { - log.warn("The file '" + fileID + "' in the ChecksumCache is no longer in the archive. " - + "Dispatching an alarm, and removing it from the cache."); + log.warn("The file '" + fileID + "' in the ChecksumCache is no longer in the archive. " + + "Dispatching an alarm, and removing it from the cache."); Alarm alarm = new Alarm(); alarm.setAlarmCode(AlarmCode.COMPONENT_FAILURE); - alarm.setAlarmText("The file '" + fileID + "' has been removed from the archive without it being removed " - + "from index. Removing it from index."); + alarm.setAlarmText("The file '" + fileID + "' has been removed from the archive without it being removed " + + "from index. Removing it from index."); alarm.setFileID(fileID); alarmDispatcher.error(alarm); @@ -326,8 +325,7 @@ private void transferFileToTmp(String fileID, String collectionID, String fileAd log.debug("Retrieving the data to be stored from URL: '" + fileAddress + "'"); try { - fileArchive.downloadFileForValidation(fileID, collectionID, - fileExchange.getFile(new URL(fileAddress))); + fileArchive.downloadFileForValidation(fileID, collectionID, fileExchange.getFile(new URL(fileAddress))); } catch (IOException e) { String errMsg = "Could not retrieve the file from '" + fileAddress + "'"; log.error(errMsg, e); @@ -349,13 +347,13 @@ private void verifyFileInTmp(String fileID, String collectionID, ChecksumDataFor String calculatedChecksum = getChecksumForTempFile(fileID, collectionID, expectedChecksum.getChecksumSpec()); String expectedChecksumValue = Base16Utils.decodeBase16(expectedChecksum.getChecksumValue()); - log.debug("Validating newly downloaded file, '" + fileID + "', against expected checksum '" - + expectedChecksumValue + "'."); + log.debug("Validating newly downloaded file, '" + fileID + "', against expected checksum '" + + expectedChecksumValue + "'."); if (!calculatedChecksum.equals(expectedChecksumValue)) { - log.warn("Wrong checksum! Expected: [" + expectedChecksumValue - + "], but calculated: [" + calculatedChecksum + "]"); - throw new IllegalOperationException(ResponseCode.NEW_FILE_CHECKSUM_FAILURE, "The downloaded file does " - + "not have the expected checksum", fileID); + log.warn("Wrong checksum! Expected: [" + expectedChecksumValue + "], but calculated: [" + + calculatedChecksum + "]"); + throw new IllegalOperationException(ResponseCode.NEW_FILE_CHECKSUM_FAILURE, + "The downloaded file does " + "not have the expected checksum", fileID); } } else { log.debug("No checksums for validating the newly downloaded file '" + fileID + "'."); diff --git a/bitrepository-reference-pillar/src/main/java/org/bitrepository/pillar/store/StorageModel.java b/bitrepository-reference-pillar/src/main/java/org/bitrepository/pillar/store/StorageModel.java index 243805eb2..4975d6e69 100644 --- a/bitrepository-reference-pillar/src/main/java/org/bitrepository/pillar/store/StorageModel.java +++ b/bitrepository-reference-pillar/src/main/java/org/bitrepository/pillar/store/StorageModel.java @@ -76,7 +76,7 @@ public abstract class StorageModel { * @param fileExchange The file exchange. */ protected StorageModel(FileStore archives, ChecksumStore cache, AlarmDispatcher alarmDispatcher, Settings settings, - FileExchange fileExchange) { + FileExchange fileExchange) { this.cache = cache; this.fileArchive = archives; this.alarmDispatcher = alarmDispatcher; @@ -125,7 +125,8 @@ public boolean hasFileID(String fileID, String collectionID) { * @return The requested type of checksum for the given file. * @throws RequestHandlerException If a non-default checksum is requested for a checksum-pillar. */ - public String getChecksumForFile(String fileID, String collectionID, ChecksumSpecTYPE csType) throws RequestHandlerException { + public String getChecksumForFile(String fileID, String collectionID, ChecksumSpecTYPE csType) + throws RequestHandlerException { if (csType.equals(defaultChecksumSpec)) { verifyFileToCacheConsistencyIfRequired(fileID, collectionID); return cache.getChecksum(fileID, collectionID); @@ -195,16 +196,15 @@ public ChecksumDataForFileTYPE getChecksumDataForFile(String fileID, String coll * @throws RequestHandlerException If it is a non-default checksum specification, which is not supported (e.g. if * it is a ChecksumPillar). */ - public ExtractedChecksumResultSet getChecksumResultSet(XMLGregorianCalendar minTimestamp, XMLGregorianCalendar maxTimestamp, - Long maxResults, String collectionID, ChecksumSpecTYPE csSpec) + public ExtractedChecksumResultSet getChecksumResultSet(XMLGregorianCalendar minTimestamp, + XMLGregorianCalendar maxTimestamp, Long maxResults, String collectionID, ChecksumSpecTYPE csSpec) throws RequestHandlerException { verifyFileToCacheConsistencyOfAllDataIfRequired(collectionID); if (csSpec.equals(defaultChecksumSpec)) { return cache.getChecksumResults(minTimestamp, maxTimestamp, maxResults, collectionID); } else { - log.info( - "Bulk-extraction of non-default checksums for spec: " + csSpec + ", on collection " + collectionID + ", with maximum " + - maxResults + " results."); + log.info("Bulk-extraction of non-default checksums for spec: {}, on collection {}, with maximum {} " + + "results.", csSpec, collectionID, maxResults); // We ignore minTimestamp and maxTimestamp when dealing with non-default checksums. return getNonDefaultChecksumResultSet(maxResults, collectionID, csSpec); } @@ -224,20 +224,31 @@ public ExtractedChecksumResultSet getChecksumResultSet(XMLGregorianCalendar minT * @throws RequestHandlerException If it is not possible to extract the checksum result, e.g. due to unsupported * checksum specification. */ - public ExtractedChecksumResultSet getSingleChecksumResultSet(String fileID, String collectionID, XMLGregorianCalendar minTimestamp, - XMLGregorianCalendar maxTimestamp, ChecksumSpecTYPE csSpec) + public ExtractedChecksumResultSet getSingleChecksumResultSet(String fileID, String collectionID, + XMLGregorianCalendar minTimestamp, XMLGregorianCalendar maxTimestamp, ChecksumSpecTYPE csSpec) throws RequestHandlerException { ExtractedChecksumResultSet res = new ExtractedChecksumResultSet(); ChecksumEntry entry = getChecksumEntryForFile(fileID, collectionID, csSpec); - if ((minTimestamp == null || - CalendarUtils.convertFromXMLGregorianCalendar(minTimestamp).getTime() <= entry.getCalculationDate().getTime()) && - (maxTimestamp == null || - CalendarUtils.convertFromXMLGregorianCalendar(maxTimestamp).getTime() >= entry.getCalculationDate().getTime())) { + + boolean lowerBound = minTimestamp == null || getTime(minTimestamp) <= entry.getCalculationDate().getTime(); + boolean upperBound = maxTimestamp == null || getTime(maxTimestamp) >= entry.getCalculationDate().getTime(); + if (lowerBound && upperBound) { res.insertChecksumEntry(entry); } return res; } + /** + * Helper method to convert from XMLGregorianCalender to a {@link Long}. + * + * @param xmlGregorianCalendar The {@link XMLGregorianCalendar} timestamp to convert. + * @return Returns the number of milliseconds since January 1, 1970, 00:00:00 GMT represented by this date, as a + * {@link Long}. + */ + private long getTime(XMLGregorianCalendar xmlGregorianCalendar) { + return CalendarUtils.convertFromXMLGregorianCalendar(xmlGregorianCalendar).getTime(); + } + /** * Removes the entry for the given file. * Both from the cache, and if an archive exists, then also removed/deprecated the actual file. @@ -278,8 +289,8 @@ public void verifyChecksumAlgorithm(ChecksumSpecTYPE checksumSpec) throws Reques // Validate against ChecksumPillar specific algorithm (if is a ChecksumPillar). if (getChecksumPillarSpec() != null && !(getChecksumPillarSpec().equals(checksumSpec))) { throw new InvalidMessageException(ResponseCode.REQUEST_NOT_SUPPORTED, - "Cannot handle the checksum " + "specification '" + checksumSpec + "'.This checksum pillar can only handle '" + - getChecksumPillarSpec() + "'"); + "Cannot handle the checksum " + "specification '" + checksumSpec + + "'.This checksum pillar can only handle '" + getChecksumPillarSpec() + "'"); } try { @@ -322,7 +333,8 @@ protected ChecksumEntry retrieveNonDefaultChecksumEntry(String fileID, String co * @throws RequestHandlerException If it is not possible to store a file with the given size within the * archive of the given collection. */ - public abstract void verifyEnoughFreeSpaceLeftForFile(Long fileSize, String collectionID) throws RequestHandlerException; + public abstract void verifyEnoughFreeSpaceLeftForFile(Long fileSize, String collectionID) + throws RequestHandlerException; /** * Handles the ReplaceFile operation. @@ -334,8 +346,8 @@ protected ChecksumEntry retrieveNonDefaultChecksumEntry(String fileID, String co * @throws RequestHandlerException If something goes wrong, e.g. the downloaded file does not have the expected * checksum. */ - public abstract void replaceFile(String fileID, String collectionID, String fileAddress, ChecksumDataForFileTYPE validationChecksum) - throws RequestHandlerException; + public abstract void replaceFile(String fileID, String collectionID, String fileAddress, + ChecksumDataForFileTYPE validationChecksum) throws RequestHandlerException; /** * Handles the PutFile operation. @@ -347,8 +359,8 @@ public abstract void replaceFile(String fileID, String collectionID, String file * @throws RequestHandlerException If something goes wrong, e.g. the downloaded file does not have the expected * checksum. */ - public abstract void putFile(String collectionID, String fileID, String fileAddress, ChecksumDataForFileTYPE expectedChecksum) - throws RequestHandlerException; + public abstract void putFile(String collectionID, String fileID, String fileAddress, + ChecksumDataForFileTYPE expectedChecksum) throws RequestHandlerException; /** * Verify the consistency between all the data in the file archive and in the cache, if it is required by the @@ -400,7 +412,8 @@ protected abstract String getNonDefaultChecksum(String fileID, String collection * @return The fileInfo for the file. * @throws RequestHandlerException If it is a ChecksumPillar. */ - public abstract FileInfo getFileInfoForActualFile(String fileID, String collectionID) throws RequestHandlerException; + public abstract FileInfo getFileInfoForActualFile(String fileID, String collectionID) + throws RequestHandlerException; /** * Extracts a set of file ids according to the given restrictions. @@ -413,7 +426,7 @@ protected abstract String getNonDefaultChecksum(String fileID, String collection * @return The extracted file ids. */ public abstract ExtractedFileIDsResultSet getFileIDsResultSet(String fileID, XMLGregorianCalendar minTimestamp, - XMLGregorianCalendar maxTimestamp, Long maxResults, String collectionID); + XMLGregorianCalendar maxTimestamp, Long maxResults, String collectionID); /** * Retrieves the checksums with a non-default checksum specification for some files. @@ -426,7 +439,7 @@ public abstract ExtractedFileIDsResultSet getFileIDsResultSet(String fileID, XML * ChecksumPillar. */ protected abstract ExtractedChecksumResultSet getNonDefaultChecksumResultSet(Long maxResults, String collectionID, - ChecksumSpecTYPE csSpec) throws RequestHandlerException; + ChecksumSpecTYPE csSpec) throws RequestHandlerException; /** * Throws an exception unless the actual file exists and is available.