Bitmag 1142 integrity gui display checksum age

bitrepository-core/src/main/java/org/bitrepository/common/utils/SettingsUtils.java

@@ -22,6 +22,7 @@
 package org.bitrepository.common.utils;
 
 import org.bitrepository.common.settings.Settings;
+import org.bitrepository.settings.referencesettings.IntegrityServiceSettings;
 import org.bitrepository.settings.referencesettings.PillarIntegrityDetails;
 import org.bitrepository.settings.referencesettings.PillarType;
 import org.bitrepository.settings.repositorysettings.Collection;
@@ -118,6 +119,10 @@ public static String getPillarName(String pillarID) {
         return null;
     }
 
+    public static IntegrityServiceSettings getIntegrityServiceSettings() {
+        return settings.getReferenceSettings().getIntegrityServiceSettings();
+    }
+
     /**
      * Get the {@link PillarType} for the given Pillar ID.
      *
@@ -207,4 +212,5 @@ public static Set<String> getStatusContributorsForCollection() {
         contributors.addAll(SettingsUtils.getAllPillarIDs());
         return contributors;
     }
+
 }

bitrepository-core/src/main/java/org/bitrepository/common/utils/TimeUtils.java

@@ -21,12 +21,20 @@
  */
 package org.bitrepository.common.utils;
 
+import org.bitrepository.common.ArgumentValidator;
+import org.jetbrains.annotations.NotNull;
+
 import javax.xml.datatype.XMLGregorianCalendar;
 import java.text.DateFormat;
 import java.text.SimpleDateFormat;
 import java.time.Duration;
+import java.time.Period;
+import java.time.ZonedDateTime;
 import java.time.temporal.ChronoUnit;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.List;
+import java.util.Locale;
 import java.util.concurrent.TimeUnit;
 
 /**
@@ -125,6 +133,91 @@ public static String millisecondsToHuman(long ms) {
         return sb.toString();
     }
 
+    /**
+     * Formats a non-negative Duration to an approximate human-readable string like "1y 2m" or "3h 45m".
+     * The conversion uses estimated/approximate average values for the lengths of days, months and years.
+     * The method is therefore suitable for durations longer than a month.
+     *
+     * The duration must be non-negative and not longer than 4 382 910 hours (approximately 500 years).
+     *
+     * @throws IllegalArgumentException if dur is negative or longer than 4 382 910 hours
+     */
+    public static String durationToHumanUsingEstimates(Duration dur) {
+        ArgumentValidator.checkTrue(! dur.isNegative(), "Cannot handle a negative duration; got " + dur);
+        ArgumentValidator.checkTrue(dur.compareTo(Duration.ofHours(4_382_910)) <= 0,
+                "Duration is too long: " + dur);
+
+        int years = Math.toIntExact(dur.dividedBy(ChronoUnit.YEARS.getDuration()));
+        dur = dur.minus(ChronoUnit.YEARS.getDuration().multipliedBy(years));
+        int months = Math.toIntExact(dur.dividedBy(ChronoUnit.MONTHS.getDuration()));
+        dur = dur.minus(ChronoUnit.MONTHS.getDuration().multipliedBy(months));
+        int days = Math.toIntExact(dur.dividedBy(ChronoUnit.DAYS.getDuration()));
+        dur = dur.minus(ChronoUnit.DAYS.getDuration().multipliedBy(days));
+
+        Period p = Period.of(years, months, days);
+
+        return humanPeriodAndDuration(p, dur);
+    }
+
+    /**
+     * Generate a human-readable difference between start and end like "5y 2m 23d" or "7d 23m".
+     *
+     * Include years, months and days if they are non-zero. Include hours if months are 6 or less.
+     * Include minutes if days are 8 or less. Never include seconds.
+     * This generally gives the user a precision of 0.5 % of the difference or finer.
+     */
+    public static String humanDifference(ZonedDateTime start, ZonedDateTime end) {
+        ArgumentValidator.checkTrue(! end.isBefore(start), start + " > " + end);
+
+        Period periodBetween = Period.between(start.toLocalDate(), end.toLocalDate());
+        ZonedDateTime afterPeriod = start.plus(periodBetween);
+        if (afterPeriod.isAfter(end)) { // Too far
+            // One day fewer
+            periodBetween = Period.between(start.toLocalDate(), end.toLocalDate().minusDays(1));
+            afterPeriod = start.plus(periodBetween);
+        }
+        Duration durationBetween = Duration.between(afterPeriod, end);
+
+        return humanPeriodAndDuration(periodBetween, durationBetween);
+    }
+
+    @NotNull
+    private static String humanPeriodAndDuration(Period period, Duration dur) {
+        // Round duration to whole minutes
+        dur = dur.plusSeconds(30).truncatedTo(ChronoUnit.MINUTES);
+
+        if (period.isZero() && dur.isZero()) {
+            return "0m";
+        }
+
+        boolean includeHours = period.getYears() == 0 && period.getMonths() <= 6;
+        boolean includeMinutes = period.getYears() == 0
+                && period.getMonths() == 0
+                && period.getDays() <= 8;
+
+        // The following gives an ambiguous string like "3m"
+        // in the very rare cases where months or minutes are non-zero and days and hours are zero.
+        // It is not expected to be a problem for the user in practice.
+        List<String> elements = new ArrayList<>(6);
+        if (period.getYears() != 0) {
+            elements.add(period.getYears() + "y");
+        }
+        if (period.getMonths() != 0) {
+            elements.add(period.getMonths() + "m");
+        }
+        if (period.getDays() != 0) {
+            elements.add(period.getDays() + "d");
+        }
+        if (includeHours && dur.toHours() != 0) {
+            elements.add(dur.toHours() + "h");
+        }
+        if (includeMinutes && dur.toMinutesPart() != 0) {
+            elements.add(dur.toMinutesPart() + "m");
+        }
+
+        return String.join(" ", elements);
+    }
+
     /**
      * @throws ArithmeticException if dur is negative by more than Long.MAX_VALUE seconds
      */

bitrepository-core/src/test/java/org/bitrepository/common/utils/TimeUtilsTest.java

@@ -28,7 +28,13 @@
 import java.text.DateFormat;
 import java.text.SimpleDateFormat;
 import java.time.Duration;
+import java.time.Instant;
+import java.time.Period;
+import java.time.ZoneId;
+import java.time.ZoneOffset;
+import java.time.ZonedDateTime;
 import java.time.temporal.ChronoUnit;
+import java.time.temporal.TemporalAmount;
 import java.util.Date;
 import java.util.Locale;
 import java.util.concurrent.TimeUnit;
@@ -37,6 +43,8 @@
 import static org.testng.Assert.assertTrue;
 
 public class TimeUtilsTest extends ExtendedTestCase {
+    private static final ZonedDateTime BASE = Instant.EPOCH.atZone(ZoneOffset.UTC);
+
     @Test(groups = {"regressiontest"})
     public void timeTester() throws Exception {
         addDescription("Tests the TimeUtils. Pi days = 271433605 milliseconds");
@@ -73,10 +81,26 @@ public void timeTester() throws Exception {
         assertTrue(human.contains(expectedDays), human);
     }
 
+    @Test(groups = {"regressiontest"})
+    public void printsHumanDuration() {
+        assertEquals(TimeUtils.durationToHumanUsingEstimates(ChronoUnit.YEARS.getDuration()), "1y");
+        assertEquals(TimeUtils.durationToHumanUsingEstimates(ChronoUnit.MONTHS.getDuration()), "1m");
+        assertEquals(TimeUtils.durationToHumanUsingEstimates(ChronoUnit.DAYS.getDuration()), "1d");
+        assertEquals(TimeUtils.durationToHumanUsingEstimates(ChronoUnit.HOURS.getDuration()), "1h");
+        assertEquals(TimeUtils.durationToHumanUsingEstimates(ChronoUnit.MINUTES.getDuration()), "1m");
+        // Don’t print seconds
+        assertEquals(TimeUtils.durationToHumanUsingEstimates(ChronoUnit.SECONDS.getDuration()), "0m");
+        assertEquals(TimeUtils.durationToHumanUsingEstimates(Duration.parse("PT2H3M5S")), "2h 3m");
+
+        addStep("Test the limits of what the method handles", "0m and 500y respectively");
+        assertEquals(TimeUtils.durationToHumanUsingEstimates(Duration.ZERO), "0m");
+        assertEquals(TimeUtils.durationToHumanUsingEstimates(Duration.ofHours(4_382_910)), "500y");
+    }
+
     @Test(groups = {"regressiontest"})
     public void zeroIntervalTest() throws Exception {
         addDescription("Verifies that a 0 ms interval is represented correctly");
-        addStep("Call the millisecondsToHuman with 0 ms", "The output should be '0 ms'");
+        addStep("Call millisecondsToHuman with 0 ms", "The output should be '0 ms'");
         String zeroTimeString = TimeUtils.millisecondsToHuman(0);
         assertEquals(zeroTimeString, " 0 ms");
     }
@@ -103,6 +127,89 @@ public void durationsPrintHumanly() {
         Duration allUnits = Duration.parse("P3DT5H7M11.013000017S");
         assertEquals(TimeUtils.durationToHuman(allUnits), "3d 5h 7m 11s 13000017 ns");
     }
+    @Test(groups = {"regressiontest"})
+    public void differencesPrintHumanly() {
+        addDescription("TimeUtils.humanDifference() should return" +
+                " similar human readable strings to those from millisecondsToHuman()");
+
+        addStep("Call humanDifference() with same time twice", "The output should be '0m'");
+        String zeroTimeString = TimeUtils.humanDifference(BASE, BASE);
+        assertEquals(zeroTimeString, "0m");
+
+        addStep("Call humanDifference() with a difference obtained from a Duration",
+                "Expect corresponding readable output");
+        // Don’t print seconds
+        testHumanDifference("0m", Duration.ofSeconds(1));
+        testHumanDifference("1m", Duration.ofMinutes(1));
+        testHumanDifference("1h", Duration.ofHours(1));
+        testHumanDifference("2h 3m", Duration.parse("PT2H3M5.000000007S"));
+
+        addStep("Call humanDifference() with a difference obtained from a Period",
+                "Expect corresponding readable output");
+        testHumanDifference("1d", Period.ofDays(1));
+        testHumanDifference("1m", Period.ofMonths(1));
+        testHumanDifference("1y", Period.ofYears(1));
+        testHumanDifference("2y 3m 5d", Period.of(2, 3, 5));
+
+        addStep("Call humanDifference() with a difference obtained from a combo of a Period and a Duration",
+                "Expect corresponding readable output");
+        testHumanDifference("3y 5m 7d",
+                Period.of(3, 5, 7), Duration.parse("PT11H13M17.023S"));
+        testHumanDifference("2m 7d 11h",
+                Period.of(0, 2, 7), Duration.parse("PT11H13M17.023S"));
+        testHumanDifference("1d 11h 13m", Period.ofDays(1), Duration.parse("PT11H13M17.023S"));
+
+        addStep("Call humanDifference()" +
+                        " with dates that are 2 days apart but times that cause the diff to be less than 2 full days",
+                "Expect output 1d something");
+        ZoneId testZoneId = ZoneId.of("Europe/Vienna");
+        String oneDaySomethingString = TimeUtils.humanDifference(
+                ZonedDateTime.of(2021, 1, 31,
+                        12, 0, 0, 0, testZoneId),
+                ZonedDateTime.of(2021, 2, 2,
+                        11, 59, 29, 0, testZoneId));
+        assertEquals(oneDaySomethingString, "1d 23h 59m");
+    }
+
+    @Test(groups = {"regressiontest"})
+    public void differencesPrintsWithAppropriatePrecision() {
+        // Include hours if months are 6 or less.
+        testHumanDifference("11m", Period.ofMonths(11), Duration.ofHours(23));
+        testHumanDifference("1y 1d", Period.of(1, 0, 1), Duration.ofHours(23));
+        testHumanDifference("2m 1h", Period.ofMonths(2), Duration.ofHours(1));
+        // Include minutes if days are 8 or less.
+        testHumanDifference("1y", Period.ofYears(1), Duration.ofMinutes(23));
+        testHumanDifference("1m", Period.ofMonths(1), Duration.ofMinutes(23));
+        testHumanDifference("27d", Period.ofDays(27), Duration.ofMinutes(23));
+        testHumanDifference("2d 3m", Period.ofDays(2), Duration.ofMinutes(3));
+        // Round to whole minutes
+        testHumanDifference("2d 3m", Period.ofDays(2), Duration.ofMinutes(2).plusSeconds(30));
+        testHumanDifference("2d 3m", Period.ofDays(2), Duration.ofMinutes(3).plusSeconds(29));
+        // Never include seconds.
+        testHumanDifference("1y", Period.ofYears(1), Duration.ofSeconds(55));
+        testHumanDifference("1m", Period.ofMonths(1), Duration.ofSeconds(55));
+        testHumanDifference("1d", Period.ofDays(1), Duration.ofSeconds(29));
+        testHumanDifference("22h", Duration.ofHours(22).plusSeconds(29));
+        testHumanDifference("4m", Duration.ofMinutes(4).plusSeconds(29));
+        testHumanDifference("0m", Duration.ofSeconds(2).plusMillis(1));
+        testHumanDifference("0m", Duration.ofNanos(500_000_000));
+        testHumanDifference("0m", Duration.ofNanos(499_999_999));
+        testHumanDifference("0m", Duration.ofMillis(1));
+        testHumanDifference("0m", Duration.ofNanos(1));
+    }
+
+    /**
+     * Note that the expected result comes first in the argument list
+     * so that we can use varargs to pass a number of amounts, for example both a Period and a Duration.
+     */
+    private void testHumanDifference(String expected, TemporalAmount... amounts) {
+        ZonedDateTime end = BASE;
+        for (TemporalAmount amount: amounts) {
+            end = end.plus(amount);
+        }
+        String differenceString = TimeUtils.humanDifference(BASE, end);
+        assertEquals(differenceString, expected);
+    }
 
     /*
      * The test only ensures that the output format is fixed. Which timezone the date is

bitrepository-integration/src/main/resources/quickstart/conf/checksumpillar/ReferenceSettings.xml

@@ -40,7 +40,7 @@
     </CollectionDirs>
     <MinimumSizeLeft>1000000000</MinimumSizeLeft>
     <TimeToStartDeliver>PT1S</TimeToStartDeliver>
-    <MaxAgeForChecksums>P114Y</MaxAgeForChecksums>
+    <MaxAgeForChecksums>P100Y</MaxAgeForChecksums>
     <AuditTrailContributerDatabase>
       <DriverClass>org.apache.derby.jdbc.EmbeddedDriver</DriverClass>
       <DatabaseURL>jdbc:derby:<!--foobarpattern-->var/checksumpillar/auditcontributerdb</DatabaseURL>

bitrepository-integration/src/main/resources/quickstart/conf/integrityservice/ReferenceSettings.xml

@@ -93,6 +93,20 @@
       <DriverClass>org.apache.derby.jdbc.EmbeddedDriver</DriverClass>
       <DatabaseURL>jdbc:derby:<!--foobarpattern-->conf/integrityservice/auditcontributerdb</DatabaseURL>
     </AuditTrailContributerDatabase>
+    <ObsoleteChecksumSettings>
+      <MaxChecksumAgeForPillar>
+        <PillarID>checksum-pillar</PillarID>
+        <MaxChecksumAge>P100Y</MaxChecksumAge>
+      </MaxChecksumAgeForPillar>
+      <MaxChecksumAgeForPillar>
+        <PillarID>file1-pillar</PillarID>
+        <MaxChecksumAge>PT1H</MaxChecksumAge>
+      </MaxChecksumAgeForPillar>
+      <MaxChecksumAgeForPillar>
+        <PillarID>file2-pillar</PillarID>
+        <MaxChecksumAge>PT1H</MaxChecksumAge>
+      </MaxChecksumAgeForPillar>
+    </ObsoleteChecksumSettings>
     <IntegrityReportsDir>conf/integrityservice/reportdir</IntegrityReportsDir>
     <PillarIntegrityDetails>
       <PillarDetails>

bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/IntegrityDatabaseMigrator.java

@@ -30,7 +30,7 @@
 import java.util.Map;
 
 /**
- * Migration class for the AuditTrailDatabase of the AuditTrailService.
+ * Migration class for the IntegrityDatabaseDatabase of the IntegrityDatabaseService.
  * Will only try to perform the migration on an embedded derby database.
  */
 public class IntegrityDatabaseMigrator extends DatabaseMigrator {
@@ -63,10 +63,13 @@ public class IntegrityDatabaseMigrator extends DatabaseMigrator {
      * The name of the update script for version 6 to 7.
      */
     private static final String UPDATE_SCRIPT_VERSION_6_TO_7 = "sql/derby/integrityDB6to7migration.sql";
+
+    private static final String UPDATE_SCRIPT_VERSION_7_TO_8 = "sql/derby/integrityDB7to8migration.sql";
+
     /**
      * The current version of the database.
      */
-    private final Integer currentVersion = 7;
+    private final int currentVersion = 8;
 
     /**
      * @param connector connection to the database.
@@ -114,6 +117,15 @@ public void migrate() {
             log.warn("Migrating integrityDB from version 6 to 7");
             migrateDerbyDatabase(UPDATE_SCRIPT_VERSION_6_TO_7);
         }
+        if (versions.get(DATABASE_VERSION_ENTRY) < 8) {
+            log.warn("Migrating integrityDB from version 7 to 8");
+            migrateDerbyDatabase(UPDATE_SCRIPT_VERSION_7_TO_8);
+        }
+
+        if (needsMigration()) {
+            log.error("Database still appears to need migration after it has been migrated. Expected version is {}.",
+                    currentVersion);
+        }
     }
 
     @Override

bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/PillarCollectionMetric.java

@@ -1,5 +1,7 @@
 package org.bitrepository.integrityservice.cache;
 
+import java.time.Instant;
+
 /**
  * Class to carry information of collection specific pillar metrics.
  * The class exists as java is not able to handle simple tuples,
@@ -19,9 +21,13 @@ public class PillarCollectionMetric {
      */
     private final long pillarFileCount;
 
-    public PillarCollectionMetric(Long pillarCollectionSize, Long pillarFileCount) {
+    /** Timestamp of the oldest checksum on the pillar or null if no checksums yet */
+    private final Instant oldestChecksumTimestamp;
+
+    public PillarCollectionMetric(Long pillarCollectionSize, Long pillarFileCount, Instant oldestChecksumTimestamp) {
         this.pillarCollectionSize = pillarCollectionSize == null ? 0 : pillarCollectionSize;
         this.pillarFileCount = pillarFileCount == null ? 0 : pillarFileCount;
+        this.oldestChecksumTimestamp = oldestChecksumTimestamp;
     }
 
     public long getPillarCollectionSize() {
@@ -32,4 +38,7 @@ public long getPillarFileCount() {
         return pillarFileCount;
     }
 
+    public Instant getOldestChecksumTimestamp() {
+        return oldestChecksumTimestamp;
+    }
 }