diff --git a/bitrepository-core/src/main/java/org/bitrepository/common/utils/SettingsUtils.java b/bitrepository-core/src/main/java/org/bitrepository/common/utils/SettingsUtils.java index fea26d3ab..c97eb183e 100644 --- a/bitrepository-core/src/main/java/org/bitrepository/common/utils/SettingsUtils.java +++ b/bitrepository-core/src/main/java/org/bitrepository/common/utils/SettingsUtils.java @@ -22,6 +22,7 @@ package org.bitrepository.common.utils; import org.bitrepository.common.settings.Settings; +import org.bitrepository.settings.referencesettings.IntegrityServiceSettings; import org.bitrepository.settings.referencesettings.PillarIntegrityDetails; import org.bitrepository.settings.referencesettings.PillarType; import org.bitrepository.settings.repositorysettings.Collection; @@ -118,6 +119,10 @@ public static String getPillarName(String pillarID) { return null; } + public static IntegrityServiceSettings getIntegrityServiceSettings() { + return settings.getReferenceSettings().getIntegrityServiceSettings(); + } + /** * Get the {@link PillarType} for the given Pillar ID. * @@ -207,4 +212,5 @@ public static Set getStatusContributorsForCollection() { contributors.addAll(SettingsUtils.getAllPillarIDs()); return contributors; } + } diff --git a/bitrepository-core/src/main/java/org/bitrepository/common/utils/TimeUtils.java b/bitrepository-core/src/main/java/org/bitrepository/common/utils/TimeUtils.java index 28124ad4d..487dbc114 100644 --- a/bitrepository-core/src/main/java/org/bitrepository/common/utils/TimeUtils.java +++ b/bitrepository-core/src/main/java/org/bitrepository/common/utils/TimeUtils.java @@ -21,12 +21,20 @@ */ package org.bitrepository.common.utils; +import org.bitrepository.common.ArgumentValidator; +import org.jetbrains.annotations.NotNull; + import javax.xml.datatype.XMLGregorianCalendar; import java.text.DateFormat; import java.text.SimpleDateFormat; import java.time.Duration; +import java.time.Period; +import java.time.ZonedDateTime; import java.time.temporal.ChronoUnit; -import java.util.*; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; +import java.util.Locale; import java.util.concurrent.TimeUnit; /** @@ -125,6 +133,91 @@ public static String millisecondsToHuman(long ms) { return sb.toString(); } + /** + * Formats a non-negative Duration to an approximate human-readable string like "1y 2m" or "3h 45m". + * The conversion uses estimated/approximate average values for the lengths of days, months and years. + * The method is therefore suitable for durations longer than a month. + * + * The duration must be non-negative and not longer than 4 382 910 hours (approximately 500 years). + * + * @throws IllegalArgumentException if dur is negative or longer than 4 382 910 hours + */ + public static String durationToHumanUsingEstimates(Duration dur) { + ArgumentValidator.checkTrue(! dur.isNegative(), "Cannot handle a negative duration; got " + dur); + ArgumentValidator.checkTrue(dur.compareTo(Duration.ofHours(4_382_910)) <= 0, + "Duration is too long: " + dur); + + int years = Math.toIntExact(dur.dividedBy(ChronoUnit.YEARS.getDuration())); + dur = dur.minus(ChronoUnit.YEARS.getDuration().multipliedBy(years)); + int months = Math.toIntExact(dur.dividedBy(ChronoUnit.MONTHS.getDuration())); + dur = dur.minus(ChronoUnit.MONTHS.getDuration().multipliedBy(months)); + int days = Math.toIntExact(dur.dividedBy(ChronoUnit.DAYS.getDuration())); + dur = dur.minus(ChronoUnit.DAYS.getDuration().multipliedBy(days)); + + Period p = Period.of(years, months, days); + + return humanPeriodAndDuration(p, dur); + } + + /** + * Generate a human-readable difference between start and end like "5y 2m 23d" or "7d 23m". + * + * Include years, months and days if they are non-zero. Include hours if months are 6 or less. + * Include minutes if days are 8 or less. Never include seconds. + * This generally gives the user a precision of 0.5 % of the difference or finer. + */ + public static String humanDifference(ZonedDateTime start, ZonedDateTime end) { + ArgumentValidator.checkTrue(! end.isBefore(start), start + " > " + end); + + Period periodBetween = Period.between(start.toLocalDate(), end.toLocalDate()); + ZonedDateTime afterPeriod = start.plus(periodBetween); + if (afterPeriod.isAfter(end)) { // Too far + // One day fewer + periodBetween = Period.between(start.toLocalDate(), end.toLocalDate().minusDays(1)); + afterPeriod = start.plus(periodBetween); + } + Duration durationBetween = Duration.between(afterPeriod, end); + + return humanPeriodAndDuration(periodBetween, durationBetween); + } + + @NotNull + private static String humanPeriodAndDuration(Period period, Duration dur) { + // Round duration to whole minutes + dur = dur.plusSeconds(30).truncatedTo(ChronoUnit.MINUTES); + + if (period.isZero() && dur.isZero()) { + return "0m"; + } + + boolean includeHours = period.getYears() == 0 && period.getMonths() <= 6; + boolean includeMinutes = period.getYears() == 0 + && period.getMonths() == 0 + && period.getDays() <= 8; + + // The following gives an ambiguous string like "3m" + // in the very rare cases where months or minutes are non-zero and days and hours are zero. + // It is not expected to be a problem for the user in practice. + List elements = new ArrayList<>(6); + if (period.getYears() != 0) { + elements.add(period.getYears() + "y"); + } + if (period.getMonths() != 0) { + elements.add(period.getMonths() + "m"); + } + if (period.getDays() != 0) { + elements.add(period.getDays() + "d"); + } + if (includeHours && dur.toHours() != 0) { + elements.add(dur.toHours() + "h"); + } + if (includeMinutes && dur.toMinutesPart() != 0) { + elements.add(dur.toMinutesPart() + "m"); + } + + return String.join(" ", elements); + } + /** * @throws ArithmeticException if dur is negative by more than Long.MAX_VALUE seconds */ diff --git a/bitrepository-core/src/test/java/org/bitrepository/common/utils/TimeUtilsTest.java b/bitrepository-core/src/test/java/org/bitrepository/common/utils/TimeUtilsTest.java index 1d878cc6c..13ecbf7b5 100644 --- a/bitrepository-core/src/test/java/org/bitrepository/common/utils/TimeUtilsTest.java +++ b/bitrepository-core/src/test/java/org/bitrepository/common/utils/TimeUtilsTest.java @@ -28,7 +28,13 @@ import java.text.DateFormat; import java.text.SimpleDateFormat; import java.time.Duration; +import java.time.Instant; +import java.time.Period; +import java.time.ZoneId; +import java.time.ZoneOffset; +import java.time.ZonedDateTime; import java.time.temporal.ChronoUnit; +import java.time.temporal.TemporalAmount; import java.util.Date; import java.util.Locale; import java.util.concurrent.TimeUnit; @@ -37,6 +43,8 @@ import static org.testng.Assert.assertTrue; public class TimeUtilsTest extends ExtendedTestCase { + private static final ZonedDateTime BASE = Instant.EPOCH.atZone(ZoneOffset.UTC); + @Test(groups = {"regressiontest"}) public void timeTester() throws Exception { addDescription("Tests the TimeUtils. Pi days = 271433605 milliseconds"); @@ -73,10 +81,26 @@ public void timeTester() throws Exception { assertTrue(human.contains(expectedDays), human); } + @Test(groups = {"regressiontest"}) + public void printsHumanDuration() { + assertEquals(TimeUtils.durationToHumanUsingEstimates(ChronoUnit.YEARS.getDuration()), "1y"); + assertEquals(TimeUtils.durationToHumanUsingEstimates(ChronoUnit.MONTHS.getDuration()), "1m"); + assertEquals(TimeUtils.durationToHumanUsingEstimates(ChronoUnit.DAYS.getDuration()), "1d"); + assertEquals(TimeUtils.durationToHumanUsingEstimates(ChronoUnit.HOURS.getDuration()), "1h"); + assertEquals(TimeUtils.durationToHumanUsingEstimates(ChronoUnit.MINUTES.getDuration()), "1m"); + // Don’t print seconds + assertEquals(TimeUtils.durationToHumanUsingEstimates(ChronoUnit.SECONDS.getDuration()), "0m"); + assertEquals(TimeUtils.durationToHumanUsingEstimates(Duration.parse("PT2H3M5S")), "2h 3m"); + + addStep("Test the limits of what the method handles", "0m and 500y respectively"); + assertEquals(TimeUtils.durationToHumanUsingEstimates(Duration.ZERO), "0m"); + assertEquals(TimeUtils.durationToHumanUsingEstimates(Duration.ofHours(4_382_910)), "500y"); + } + @Test(groups = {"regressiontest"}) public void zeroIntervalTest() throws Exception { addDescription("Verifies that a 0 ms interval is represented correctly"); - addStep("Call the millisecondsToHuman with 0 ms", "The output should be '0 ms'"); + addStep("Call millisecondsToHuman with 0 ms", "The output should be '0 ms'"); String zeroTimeString = TimeUtils.millisecondsToHuman(0); assertEquals(zeroTimeString, " 0 ms"); } @@ -103,6 +127,89 @@ public void durationsPrintHumanly() { Duration allUnits = Duration.parse("P3DT5H7M11.013000017S"); assertEquals(TimeUtils.durationToHuman(allUnits), "3d 5h 7m 11s 13000017 ns"); } + @Test(groups = {"regressiontest"}) + public void differencesPrintHumanly() { + addDescription("TimeUtils.humanDifference() should return" + + " similar human readable strings to those from millisecondsToHuman()"); + + addStep("Call humanDifference() with same time twice", "The output should be '0m'"); + String zeroTimeString = TimeUtils.humanDifference(BASE, BASE); + assertEquals(zeroTimeString, "0m"); + + addStep("Call humanDifference() with a difference obtained from a Duration", + "Expect corresponding readable output"); + // Don’t print seconds + testHumanDifference("0m", Duration.ofSeconds(1)); + testHumanDifference("1m", Duration.ofMinutes(1)); + testHumanDifference("1h", Duration.ofHours(1)); + testHumanDifference("2h 3m", Duration.parse("PT2H3M5.000000007S")); + + addStep("Call humanDifference() with a difference obtained from a Period", + "Expect corresponding readable output"); + testHumanDifference("1d", Period.ofDays(1)); + testHumanDifference("1m", Period.ofMonths(1)); + testHumanDifference("1y", Period.ofYears(1)); + testHumanDifference("2y 3m 5d", Period.of(2, 3, 5)); + + addStep("Call humanDifference() with a difference obtained from a combo of a Period and a Duration", + "Expect corresponding readable output"); + testHumanDifference("3y 5m 7d", + Period.of(3, 5, 7), Duration.parse("PT11H13M17.023S")); + testHumanDifference("2m 7d 11h", + Period.of(0, 2, 7), Duration.parse("PT11H13M17.023S")); + testHumanDifference("1d 11h 13m", Period.ofDays(1), Duration.parse("PT11H13M17.023S")); + + addStep("Call humanDifference()" + + " with dates that are 2 days apart but times that cause the diff to be less than 2 full days", + "Expect output 1d something"); + ZoneId testZoneId = ZoneId.of("Europe/Vienna"); + String oneDaySomethingString = TimeUtils.humanDifference( + ZonedDateTime.of(2021, 1, 31, + 12, 0, 0, 0, testZoneId), + ZonedDateTime.of(2021, 2, 2, + 11, 59, 29, 0, testZoneId)); + assertEquals(oneDaySomethingString, "1d 23h 59m"); + } + + @Test(groups = {"regressiontest"}) + public void differencesPrintsWithAppropriatePrecision() { + // Include hours if months are 6 or less. + testHumanDifference("11m", Period.ofMonths(11), Duration.ofHours(23)); + testHumanDifference("1y 1d", Period.of(1, 0, 1), Duration.ofHours(23)); + testHumanDifference("2m 1h", Period.ofMonths(2), Duration.ofHours(1)); + // Include minutes if days are 8 or less. + testHumanDifference("1y", Period.ofYears(1), Duration.ofMinutes(23)); + testHumanDifference("1m", Period.ofMonths(1), Duration.ofMinutes(23)); + testHumanDifference("27d", Period.ofDays(27), Duration.ofMinutes(23)); + testHumanDifference("2d 3m", Period.ofDays(2), Duration.ofMinutes(3)); + // Round to whole minutes + testHumanDifference("2d 3m", Period.ofDays(2), Duration.ofMinutes(2).plusSeconds(30)); + testHumanDifference("2d 3m", Period.ofDays(2), Duration.ofMinutes(3).plusSeconds(29)); + // Never include seconds. + testHumanDifference("1y", Period.ofYears(1), Duration.ofSeconds(55)); + testHumanDifference("1m", Period.ofMonths(1), Duration.ofSeconds(55)); + testHumanDifference("1d", Period.ofDays(1), Duration.ofSeconds(29)); + testHumanDifference("22h", Duration.ofHours(22).plusSeconds(29)); + testHumanDifference("4m", Duration.ofMinutes(4).plusSeconds(29)); + testHumanDifference("0m", Duration.ofSeconds(2).plusMillis(1)); + testHumanDifference("0m", Duration.ofNanos(500_000_000)); + testHumanDifference("0m", Duration.ofNanos(499_999_999)); + testHumanDifference("0m", Duration.ofMillis(1)); + testHumanDifference("0m", Duration.ofNanos(1)); + } + + /** + * Note that the expected result comes first in the argument list + * so that we can use varargs to pass a number of amounts, for example both a Period and a Duration. + */ + private void testHumanDifference(String expected, TemporalAmount... amounts) { + ZonedDateTime end = BASE; + for (TemporalAmount amount: amounts) { + end = end.plus(amount); + } + String differenceString = TimeUtils.humanDifference(BASE, end); + assertEquals(differenceString, expected); + } /* * The test only ensures that the output format is fixed. Which timezone the date is diff --git a/bitrepository-integration/src/main/resources/quickstart/conf/checksumpillar/ReferenceSettings.xml b/bitrepository-integration/src/main/resources/quickstart/conf/checksumpillar/ReferenceSettings.xml index 5026645cf..cb599fdcc 100644 --- a/bitrepository-integration/src/main/resources/quickstart/conf/checksumpillar/ReferenceSettings.xml +++ b/bitrepository-integration/src/main/resources/quickstart/conf/checksumpillar/ReferenceSettings.xml @@ -40,7 +40,7 @@ 1000000000 PT1S - P114Y + P100Y org.apache.derby.jdbc.EmbeddedDriver jdbc:derby:var/checksumpillar/auditcontributerdb diff --git a/bitrepository-integration/src/main/resources/quickstart/conf/integrityservice/ReferenceSettings.xml b/bitrepository-integration/src/main/resources/quickstart/conf/integrityservice/ReferenceSettings.xml index f8198e933..4687a3d2d 100644 --- a/bitrepository-integration/src/main/resources/quickstart/conf/integrityservice/ReferenceSettings.xml +++ b/bitrepository-integration/src/main/resources/quickstart/conf/integrityservice/ReferenceSettings.xml @@ -93,6 +93,20 @@ org.apache.derby.jdbc.EmbeddedDriver jdbc:derby:conf/integrityservice/auditcontributerdb + + + checksum-pillar + P100Y + + + file1-pillar + PT1H + + + file2-pillar + PT1H + + conf/integrityservice/reportdir diff --git a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/IntegrityDatabaseMigrator.java b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/IntegrityDatabaseMigrator.java index f93b36edb..34f8b62a5 100644 --- a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/IntegrityDatabaseMigrator.java +++ b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/IntegrityDatabaseMigrator.java @@ -30,7 +30,7 @@ import java.util.Map; /** - * Migration class for the AuditTrailDatabase of the AuditTrailService. + * Migration class for the IntegrityDatabaseDatabase of the IntegrityDatabaseService. * Will only try to perform the migration on an embedded derby database. */ public class IntegrityDatabaseMigrator extends DatabaseMigrator { @@ -63,10 +63,13 @@ public class IntegrityDatabaseMigrator extends DatabaseMigrator { * The name of the update script for version 6 to 7. */ private static final String UPDATE_SCRIPT_VERSION_6_TO_7 = "sql/derby/integrityDB6to7migration.sql"; + + private static final String UPDATE_SCRIPT_VERSION_7_TO_8 = "sql/derby/integrityDB7to8migration.sql"; + /** * The current version of the database. */ - private final Integer currentVersion = 7; + private final int currentVersion = 8; /** * @param connector connection to the database. @@ -114,6 +117,15 @@ public void migrate() { log.warn("Migrating integrityDB from version 6 to 7"); migrateDerbyDatabase(UPDATE_SCRIPT_VERSION_6_TO_7); } + if (versions.get(DATABASE_VERSION_ENTRY) < 8) { + log.warn("Migrating integrityDB from version 7 to 8"); + migrateDerbyDatabase(UPDATE_SCRIPT_VERSION_7_TO_8); + } + + if (needsMigration()) { + log.error("Database still appears to need migration after it has been migrated. Expected version is {}.", + currentVersion); + } } @Override diff --git a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/PillarCollectionMetric.java b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/PillarCollectionMetric.java index 5157b58b2..43d3f2dd1 100644 --- a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/PillarCollectionMetric.java +++ b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/PillarCollectionMetric.java @@ -1,5 +1,7 @@ package org.bitrepository.integrityservice.cache; +import java.time.Instant; + /** * Class to carry information of collection specific pillar metrics. * The class exists as java is not able to handle simple tuples, @@ -19,9 +21,13 @@ public class PillarCollectionMetric { */ private final long pillarFileCount; - public PillarCollectionMetric(Long pillarCollectionSize, Long pillarFileCount) { + /** Timestamp of the oldest checksum on the pillar or null if no checksums yet */ + private final Instant oldestChecksumTimestamp; + + public PillarCollectionMetric(Long pillarCollectionSize, Long pillarFileCount, Instant oldestChecksumTimestamp) { this.pillarCollectionSize = pillarCollectionSize == null ? 0 : pillarCollectionSize; this.pillarFileCount = pillarFileCount == null ? 0 : pillarFileCount; + this.oldestChecksumTimestamp = oldestChecksumTimestamp; } public long getPillarCollectionSize() { @@ -32,4 +38,7 @@ public long getPillarFileCount() { return pillarFileCount; } + public Instant getOldestChecksumTimestamp() { + return oldestChecksumTimestamp; + } } diff --git a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/PillarCollectionStat.java b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/PillarCollectionStat.java index 611ce2361..0c006eeb2 100644 --- a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/PillarCollectionStat.java +++ b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/PillarCollectionStat.java @@ -21,6 +21,12 @@ */ package org.bitrepository.integrityservice.cache; +import org.bitrepository.common.utils.TimeUtils; +import org.jetbrains.annotations.NotNull; + +import java.time.Instant; +import java.time.ZoneId; +import java.time.ZonedDateTime; import java.util.Date; /** @@ -37,6 +43,8 @@ public class PillarCollectionStat { private Long obsoleteChecksums = 0L; private Long missingChecksums = 0L; private Long checksumErrors = 0L; + private String maxAgeForChecksums; + private Instant oldestChecksumTimestamp; private Date statsTime; private Date updateTime; @@ -47,9 +55,11 @@ public PillarCollectionStat(String pillarID, String collectionID, String pillarN this.pillarType = pillarType; } - public PillarCollectionStat(String pillarID, String collectionID, String pillarName, String pillarType, Long fileCount, - Long dataSize, Long missingFiles, Long checksumErrors, Long missingChecksums, Long obsoleteChecksum, - Date statsTime, Date updateTime) { + public PillarCollectionStat(String pillarID, String collectionID, String pillarName, String pillarType, + Long fileCount, Long dataSize, + Long missingFiles, Long checksumErrors, Long missingChecksums, Long obsoleteChecksum, + String maxAgeForChecksums, + Instant oldestChecksumTimestamp, Date statsTime, Date updateTime) { this.pillarID = pillarID; this.collectionID = collectionID; this.pillarName = pillarName; @@ -62,6 +72,8 @@ public PillarCollectionStat(String pillarID, String collectionID, String pillarN this.obsoleteChecksums = obsoleteChecksum; this.statsTime = statsTime; this.updateTime = updateTime; + this.maxAgeForChecksums = maxAgeForChecksums; + this.oldestChecksumTimestamp = oldestChecksumTimestamp; } public String getPillarID() { @@ -144,4 +156,31 @@ public void setMissingChecksums(Long missingChecksums) { this.missingChecksums = missingChecksums; } + /** @return Human-readable age of the oldest checksum, for example "3m 46s" */ + @NotNull + public String getAgeOfOldestChecksum() { + if (oldestChecksumTimestamp == null) { + return "N/A"; + } + ZoneId zone = ZoneId.systemDefault(); + return TimeUtils.humanDifference(oldestChecksumTimestamp.atZone(zone), ZonedDateTime.now(zone)); + } + + public boolean hasOldestChecksumTimestamp() { + return oldestChecksumTimestamp != null; + } + + /** @throws NullPointerException if hasOldestChecksumTimestamp() does not return true */ + public long getOldestChecksumTimestampMillis() { + return oldestChecksumTimestamp.toEpochMilli(); + } + + public void setOldestChecksumTimestamp(Instant oldestChecksumTimestamp) { + this.oldestChecksumTimestamp = oldestChecksumTimestamp; + } + + public String getMaxAgeForChecksums() { + return maxAgeForChecksums; + } + } diff --git a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/database/IntegrityDAO.java b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/database/IntegrityDAO.java index 72b40f378..ac406051e 100644 --- a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/database/IntegrityDAO.java +++ b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/database/IntegrityDAO.java @@ -26,13 +26,18 @@ import org.bitrepository.common.ArgumentValidator; import org.bitrepository.common.utils.CalendarUtils; import org.bitrepository.common.utils.SettingsUtils; +import org.bitrepository.common.utils.TimeUtils; import org.bitrepository.integrityservice.cache.CollectionStat; import org.bitrepository.integrityservice.cache.FileInfo; import org.bitrepository.integrityservice.cache.PillarCollectionMetric; import org.bitrepository.integrityservice.cache.PillarCollectionStat; +import org.bitrepository.integrityservice.checking.MaxChecksumAgeProvider; import org.bitrepository.integrityservice.statistics.StatisticsCollector; +import org.bitrepository.integrityservice.workflow.step.HandleObsoleteChecksumsStep; import org.bitrepository.service.database.DBConnector; import org.bitrepository.service.database.DatabaseUtils; +import org.bitrepository.settings.referencesettings.ObsoleteChecksumSettings; +import org.jetbrains.annotations.NotNull; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -40,6 +45,8 @@ import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; +import java.time.Duration; +import java.time.Instant; import java.util.ArrayList; import java.util.Arrays; import java.util.Date; @@ -75,12 +82,12 @@ public void close() { } /** - * Method to ensure that pillars found in RepositorySettings is present in the database + * Method to ensure that pillars found in RepositorySettings are present in the database */ protected abstract void initializePillars(); /** - * Method to ensure that collections found in RepositorySettings is present in the database + * Method to ensure that collections found in RepositorySettings are present in the database */ protected abstract void initializeCollections(); @@ -439,7 +446,10 @@ public void createStatistics(String collectionID, StatisticsCollector statistics public Map getPillarCollectionMetrics(String collectionID) { Map metrics = new HashMap<>(); String selectSql = - "SELECT pillarID, COUNT(fileID) as filecount, SUM(filesize) as sizesum FROM fileinfo" + " WHERE collectionID = ?" + + "SELECT pillarID, COUNT(fileID) as filecount, SUM(filesize) as sizesum," + + " MIN(checksum_timestamp) as oldest_checksum_timestamp" + + " FROM fileinfo" + + " WHERE collectionID = ?" + " GROUP BY pillarID"; try (Connection conn = dbConnector.getConnection(); @@ -447,9 +457,13 @@ public Map getPillarCollectionMetrics(String col ResultSet dbResult = ps.executeQuery()) { while (dbResult.next()) { String pillarID = dbResult.getString("pillarID"); - Long fileCount = dbResult.getLong("filecount"); - Long fileSize = dbResult.getLong("sizesum"); - PillarCollectionMetric metric = new PillarCollectionMetric(fileSize, fileCount); + long fileCount = dbResult.getLong("filecount"); + // In case SUM(filesize) returned null, dbResult.getLong() will return 0, which is the sum we want + long fileSize = dbResult.getLong("sizesum"); + long oldestChecksumTimestampMillis = dbResult.getLong("oldest_checksum_timestamp"); + Instant oldestChecksumTimestamp = + dbResult.wasNull() ? null : Instant.ofEpochMilli(oldestChecksumTimestampMillis); + PillarCollectionMetric metric = new PillarCollectionMetric(fileSize, fileCount, oldestChecksumTimestamp); metrics.put(pillarID, metric); } } catch (SQLException e) { @@ -501,8 +515,13 @@ public List getLatestPillarStats(String collectionID) { List stats = new ArrayList<>(); String latestPillarStatsSql = "SELECT pillarID, file_count, file_size, missing_files_count," + - " checksum_errors_count, missing_checksums_count, obsolete_checksums_count" + " FROM pillarstats" + " WHERE stat_key = (" + - " SELECT MAX(stat_key) FROM stats" + " WHERE collectionID = ?)"; + " checksum_errors_count, missing_checksums_count, obsolete_checksums_count," + + " oldest_checksum_timestamp" + + " FROM pillarstats" + + " WHERE stat_key = (" + + " SELECT MAX(stat_key)" + + " FROM stats" + + " WHERE collectionID = ?)"; try (Connection conn = dbConnector.getConnection(); PreparedStatement ps = DatabaseUtils.createPreparedStatement(conn, latestPillarStatsSql, collectionID)) { @@ -520,8 +539,12 @@ public List getLatestPillarStats(String collectionID) { String pillarName = Objects.requireNonNullElse(SettingsUtils.getPillarName(pillarID), "N/A"); String pillarType = (SettingsUtils.getPillarType(pillarID) != null) ? Objects.requireNonNull(SettingsUtils.getPillarType(pillarID)).value() : "Unknown"; - PillarCollectionStat p = new PillarCollectionStat(pillarID, collectionID, pillarName, pillarType, fileCount, - dataSize, missingFiles, checksumErrors, missingChecksums, obsoleteChecksums, statsTime, updateTime); + String maxAgeForChecksums = getMaxAgeForChecksums(pillarID); + Instant oldestChecksumTimestamp = getOldestChecksumTimestamp(dbResult); + PillarCollectionStat p = new PillarCollectionStat(pillarID, collectionID, + pillarName, pillarType, fileCount, dataSize, + missingFiles, checksumErrors, missingChecksums, obsoleteChecksums, + maxAgeForChecksums, oldestChecksumTimestamp, statsTime, updateTime); stats.add(p); } } @@ -534,6 +557,22 @@ public List getLatestPillarStats(String collectionID) { return stats; } + @NotNull + private String getMaxAgeForChecksums(String pillarID) { + ObsoleteChecksumSettings obsoleteChecksumSettings = + SettingsUtils.getIntegrityServiceSettings().getObsoleteChecksumSettings(); + MaxChecksumAgeProvider maxChecksumAgeProvider = + new MaxChecksumAgeProvider(HandleObsoleteChecksumsStep.DEFAULT_MAX_CHECKSUM_AGE, + obsoleteChecksumSettings); + Duration maxAge = maxChecksumAgeProvider.getMaxChecksumAge(pillarID); + return maxAge.isZero() ? "unlimited" : TimeUtils.durationToHumanUsingEstimates(maxAge); + } + + private Instant getOldestChecksumTimestamp(ResultSet dbResult) throws SQLException { + long oldestChecksumTimestamp = dbResult.getLong("oldest_checksum_timestamp"); + return dbResult.wasNull() ? null : Instant.ofEpochMilli(oldestChecksumTimestamp); + } + /** * Method that should deliver the database specific SQL for getting the latest N collection statistics * diff --git a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/database/StatisticsCreator.java b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/database/StatisticsCreator.java index cefb53bd9..0ab1e389a 100644 --- a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/database/StatisticsCreator.java +++ b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/database/StatisticsCreator.java @@ -31,6 +31,7 @@ import java.sql.Connection; import java.sql.PreparedStatement; import java.sql.SQLException; +import java.sql.Types; import java.util.Date; import java.util.List; @@ -49,8 +50,8 @@ public class StatisticsCreator { private final String insertPillarStatEntrySql = "INSERT INTO pillarstats" + " (stat_key, pillarID, file_count, file_size, missing_files_count, " - + "checksum_errors_count, missing_checksums_count, obsolete_checksums_count)" - + " (SELECT MAX(stat_key), ?, ?, ?, ?, ?, ?, ? FROM stats WHERE collectionID = ?)"; + + "checksum_errors_count, missing_checksums_count, obsolete_checksums_count, oldest_checksum_timestamp)" + + " (SELECT MAX(stat_key), ?, ?, ?, ?, ?, ?, ?, ? FROM stats WHERE collectionID = ?)"; private final Logger log = LoggerFactory.getLogger(getClass()); @@ -112,19 +113,24 @@ private void addCollectionStatistics(CollectionStat cs) throws SQLException { insertCollectionStatPS.setString(5, cs.getCollectionID()); } - private void addPillarStat(PillarCollectionStat ps) throws SQLException { - insertPillarStatPS.setString(1, ps.getPillarID()); - insertPillarStatPS.setLong(2, ps.getFileCount()); - insertPillarStatPS.setLong(3, ps.getDataSize()); - insertPillarStatPS.setLong(4, ps.getMissingFiles()); - insertPillarStatPS.setLong(5, ps.getChecksumErrors()); - insertPillarStatPS.setLong(6, ps.getMissingChecksums()); - insertPillarStatPS.setLong(7, ps.getObsoleteChecksums()); - insertPillarStatPS.setString(8, ps.getCollectionID()); + private void addPillarStat(PillarCollectionStat pcStat) throws SQLException { + insertPillarStatPS.setString(1, pcStat.getPillarID()); + insertPillarStatPS.setLong(2, pcStat.getFileCount()); + insertPillarStatPS.setLong(3, pcStat.getDataSize()); + insertPillarStatPS.setLong(4, pcStat.getMissingFiles()); + insertPillarStatPS.setLong(5, pcStat.getChecksumErrors()); + insertPillarStatPS.setLong(6, pcStat.getMissingChecksums()); + insertPillarStatPS.setLong(7, pcStat.getObsoleteChecksums()); + if (pcStat.hasOldestChecksumTimestamp()) { + insertPillarStatPS.setLong(8, pcStat.getOldestChecksumTimestampMillis()); + } else { + insertPillarStatPS.setNull(8, Types.BIGINT); + } + + insertPillarStatPS.setString(9, pcStat.getCollectionID()); insertPillarStatPS.addBatch(); } - private void execute() throws SQLException { insertStatisticsEntryPS.execute(); insertCollectionStatPS.execute(); diff --git a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/web/RestIntegrityService.java b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/web/RestIntegrityService.java index 5eb7b31eb..e169b96f6 100644 --- a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/web/RestIntegrityService.java +++ b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/web/RestIntegrityService.java @@ -264,8 +264,9 @@ public String getIntegrityStatus( String pillarName = Objects.requireNonNullElse(SettingsUtils.getPillarName(pillar), "N/A"); PillarType pillarTypeObject = SettingsUtils.getPillarType(pillar); String pillarType = pillarTypeObject != null ? pillarTypeObject.value() : null; - PillarCollectionStat emptyStat = new PillarCollectionStat(pillar, collectionID, pillarName, pillarType, 0L, 0L, 0L, 0L, 0L, - 0L, new Date(0), new Date(0)); + PillarCollectionStat emptyStat = new PillarCollectionStat(pillar, collectionID, pillarName, + pillarType, 0L, 0L, 0L, 0L, 0L, + 0L, "", null, new Date(0), new Date(0)); stats.put(pillar, emptyStat); } } @@ -282,7 +283,7 @@ public String getIntegrityStatus( } /*** - * Get the current workflow's setup as a JSON array + * Get the current workflow’s setup as a JSON array */ @GET @Path("/getWorkflowSetup") @@ -514,6 +515,8 @@ private void writeIntegrityStatusObject(PillarCollectionStat stat, JsonGenerator jg.writeObjectField("checksumErrorCount", stat.getChecksumErrors()); jg.writeObjectField("obsoleteChecksumsCount", stat.getObsoleteChecksums()); jg.writeObjectField("missingChecksumsCount", stat.getMissingChecksums()); + jg.writeObjectField("maxAgeForChecksums", stat.getMaxAgeForChecksums()); + jg.writeObjectField("ageOfOldestChecksum", stat.getAgeOfOldestChecksum()); jg.writeEndObject(); } diff --git a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/workflow/step/CreateStatisticsEntryStep.java b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/workflow/step/CreateStatisticsEntryStep.java index 40427b1f3..fab49ba3a 100644 --- a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/workflow/step/CreateStatisticsEntryStep.java +++ b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/workflow/step/CreateStatisticsEntryStep.java @@ -63,9 +63,11 @@ public synchronized void performStep() { if (metric == null) { sc.getPillarCollectionStat(pillar).setFileCount(0L); sc.getPillarCollectionStat(pillar).setDataSize(0L); + sc.getPillarCollectionStat(pillar).setOldestChecksumTimestamp(null); } else { sc.getPillarCollectionStat(pillar).setFileCount(metric.getPillarFileCount()); sc.getPillarCollectionStat(pillar).setDataSize(metric.getPillarCollectionSize()); + sc.getPillarCollectionStat(pillar).setOldestChecksumTimestamp(metric.getOldestChecksumTimestamp()); } } sc.getCollectionStat().setFileCount(store.getNumberOfFilesInCollection(collectionID)); diff --git a/bitrepository-integrity-service/src/main/resources/sql/derby/integrityDB7to8migration.sql b/bitrepository-integrity-service/src/main/resources/sql/derby/integrityDB7to8migration.sql new file mode 100644 index 000000000..44fcde9bf --- /dev/null +++ b/bitrepository-integrity-service/src/main/resources/sql/derby/integrityDB7to8migration.sql @@ -0,0 +1,31 @@ +--- +-- #%L +-- Bitrepository Integrity Client +-- %% +-- Copyright (C) 2010 - 2022 Royal Danish Library and The State Archives, Denmark +-- %% +-- This program is free software: you can redistribute it and/or modify +-- it under the terms of the GNU Lesser General Public License as +-- published by the Free Software Foundation, either version 2.1 of the +-- License, or (at your option) any later version. +-- +-- This program is distributed in the hope that it will be useful, +-- but WITHOUT ANY WARRANTY; without even the implied warranty of +-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +-- GNU General Lesser Public License for more details. +-- +-- You should have received a copy of the GNU General Lesser Public +-- License along with this program. If not, see +-- . +-- #L% +--- + +connect 'jdbc:derby:integritydb'; + +-- database version +UPDATE tableversions SET version = 8 WHERE tablename = 'integritydb'; + +-- table version +UPDATE tableversions SET version = 3 WHERE tablename = 'pillarstats'; + +ALTER TABLE pillarstats ADD COLUMN oldest_checksum_timestamp BIGINT DEFAULT NULL; diff --git a/bitrepository-integrity-service/src/main/resources/sql/postgres/integrityDB7to8migration.sql b/bitrepository-integrity-service/src/main/resources/sql/postgres/integrityDB7to8migration.sql new file mode 100644 index 000000000..ad5140807 --- /dev/null +++ b/bitrepository-integrity-service/src/main/resources/sql/postgres/integrityDB7to8migration.sql @@ -0,0 +1,29 @@ +--- +-- #%L +-- Bitrepository Integrity Client +-- %% +-- Copyright (C) 2010 - 2022 Royal Danish Library and The State Archives, Denmark +-- %% +-- This program is free software: you can redistribute it and/or modify +-- it under the terms of the GNU Lesser General Public License as +-- published by the Free Software Foundation, either version 2.1 of the +-- License, or (at your option) any later version. +-- +-- This program is distributed in the hope that it will be useful, +-- but WITHOUT ANY WARRANTY; without even the implied warranty of +-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +-- GNU General Lesser Public License for more details. +-- +-- You should have received a copy of the GNU General Lesser Public +-- License along with this program. If not, see +-- . +-- #L% +--- + +-- database version +UPDATE tableversions SET version = 8 WHERE tablename = 'integritydb'; + +-- table version +UPDATE tableversions SET version = 3 WHERE tablename = 'pillarstats'; + +ALTER TABLE pillarstats ADD COLUMN oldest_checksum_timestamp BIGINT DEFAULT NULL; diff --git a/bitrepository-webclient/src/main/webapp/integrity-service.html b/bitrepository-webclient/src/main/webapp/integrity-service.html index 487f03f45..b29a85bec 100644 --- a/bitrepository-webclient/src/main/webapp/integrity-service.html +++ b/bitrepository-webclient/src/main/webapp/integrity-service.html @@ -87,6 +87,8 @@

Integrity service

Number of missing checksums Number of obsolete checksums Number of inconsistent checksums + Configured max age of checksums + Age of oldest checksum @@ -213,7 +215,7 @@

Modal header

} /** - * Callback method to get the value of a specific property from a pillars integrity status. + * Callback method to get the value of a specific property from a pillar’s integrity status. * The callback is intended for use by {@see TableModal}. * * NOTE: The reason for returning a callback instead of the property directly is because of the modal's nature of @@ -229,7 +231,8 @@

Modal header

function showModal(type, propertyName, title, url, pillarID) { return function () { - modal = new TableModal(type, pillarID, url, "#modalBody", getIntegrityStatusPropertyCountCallback(pillarID, propertyName), 100); + modal = new TableModal(type, pillarID, url, "#modalBody", + getIntegrityStatusPropertyCountCallback(pillarID, propertyName), 100); $("#modalLabel").html(title); $("#modalBody").html("

Loading

"); modal.getModal(1); @@ -243,12 +246,8 @@

Modal header

if (type === "Pillar Name") { context.element = id + "-pillarName"; - context.title = type + " on " + id.toUpperCase(); - context.propertyName = "pillarName"; } else if (type === "Pillar Type") { context.element = id + "-pillarType"; - context.title = type + " on " + id.toUpperCase(); - context.propertyName = "pillarType"; } else if (type === "Total files") { context.element = id + "-totalFileCount"; context.title = type + " on " + id.toUpperCase(); @@ -274,6 +273,10 @@

Modal header

context.title = type + " on " + id.toUpperCase(); context.propertyName = "checksumErrorCount"; context.url += "getChecksumErrorFileIDs"; + } else if(type === "Checksum age limit") { + context.element = id + "-maxAgeForChecksums"; + } else if(type === "Oldest checksum age") { + context.element = id + "-ageOfOldestChecksum"; } context.url += "?pillarID=" + id + "&collectionID=" + getCollectionID(); return context; @@ -307,8 +310,11 @@

Modal header

html += ""; html += ""; html += ""; - html += ""; - return html; + html += ""; + html += ""; + html += ""; + html += ""; + return html; } function updateTableHeader() { @@ -316,6 +322,8 @@

Modal header

setHeader("Number of missing checksums"); setHeader("Number of obsolete checksums"); setHeader("Number of inconsistent checksums"); + setHeader("Configured max age of checksums"); + setHeader("Age of oldest checksum"); } function setHeader(type) { @@ -332,6 +340,8 @@

Modal header

updateIntCell(pillarID, "Missing checksums", pillarIntegrityStatuses[pillarID].missingChecksumsCount); updateIntCell(pillarID, "Obsolete checksums", pillarIntegrityStatuses[pillarID].obsoleteChecksumsCount); updateIntCell(pillarID, "Inconsistent checksums", pillarIntegrityStatuses[pillarID].checksumErrorCount); + updateStringCell(pillarID, "Checksum age limit", pillarIntegrityStatuses[pillarID].maxAgeForChecksums); + updateStringCell(pillarID, "Oldest checksum age", pillarIntegrityStatuses[pillarID].ageOfOldestChecksum); } function updateStringCell(id, type, cellValue) { @@ -412,7 +422,9 @@

Modal header

missingFilesCount: json[i].missingFilesCount, missingChecksumsCount: json[i].missingChecksumsCount, obsoleteChecksumsCount: json[i].obsoleteChecksumsCount, - checksumErrorCount: json[i].checksumErrorCount + checksumErrorCount: json[i].checksumErrorCount, + maxAgeForChecksums: json[i].maxAgeForChecksums, + ageOfOldestChecksum: json[i].ageOfOldestChecksum }; updateTableBody(json[i].pillarID); updateTableHeader();