From ac4d1581c24c0d1d02f1e8180df291903e95429c Mon Sep 17 00:00:00 2001 From: "Ole V. Villumsen" Date: Tue, 26 Jul 2022 12:55:59 +0200 Subject: [PATCH 01/47] BITMAG-1142 Checksum age in GUI: underway --- .../common/utils/SettingsUtils.java | 20 ++++++++ .../cache/PillarCollectionMetric.java | 11 ++++- .../cache/PillarCollectionStat.java | 19 +++++-- .../cache/database/IntegrityDAO.java | 46 +++++++++++------ .../web/RestIntegrityService.java | 4 +- .../src/main/webapp/integrity-service.html | 49 +++++++++++++------ 6 files changed, 112 insertions(+), 37 deletions(-) diff --git a/bitrepository-core/src/main/java/org/bitrepository/common/utils/SettingsUtils.java b/bitrepository-core/src/main/java/org/bitrepository/common/utils/SettingsUtils.java index f46134535..6026b1b8c 100644 --- a/bitrepository-core/src/main/java/org/bitrepository/common/utils/SettingsUtils.java +++ b/bitrepository-core/src/main/java/org/bitrepository/common/utils/SettingsUtils.java @@ -118,6 +118,26 @@ public static String getHostname(String pillarID) { return null; } + /** + * Get the configured max age for checksums for the given pillarID from the ReferenceSettings. + * + * @param pillarID The pillarID for which the hostname is wanted. + * @return human-readable maximum age for checksums for the given pillar ID. + */ + public static String getMaxAgeForChecksums(String pillarID) { + // TODO how to use pillarID?? + BigInteger maxAge = settings.getReferenceSettings().getPillarSettings().getMaxAgeForChecksums(); + if (maxAge == null) { + return null; + } + try { + return TimeUtils.millisecondsToHuman(maxAge.longValueExact()); + } + catch (ArithmeticException) { + return "Extremely long"; + } + } + /** * Get the {@link PillarType} for the given Pillar ID. * diff --git a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/PillarCollectionMetric.java b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/PillarCollectionMetric.java index 5157b58b2..43d3f2dd1 100644 --- a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/PillarCollectionMetric.java +++ b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/PillarCollectionMetric.java @@ -1,5 +1,7 @@ package org.bitrepository.integrityservice.cache; +import java.time.Instant; + /** * Class to carry information of collection specific pillar metrics. * The class exists as java is not able to handle simple tuples, @@ -19,9 +21,13 @@ public class PillarCollectionMetric { */ private final long pillarFileCount; - public PillarCollectionMetric(Long pillarCollectionSize, Long pillarFileCount) { + /** Timestamp of the oldest checksum on the pillar or null if no checksums yet */ + private final Instant oldestChecksumTimestamp; + + public PillarCollectionMetric(Long pillarCollectionSize, Long pillarFileCount, Instant oldestChecksumTimestamp) { this.pillarCollectionSize = pillarCollectionSize == null ? 0 : pillarCollectionSize; this.pillarFileCount = pillarFileCount == null ? 0 : pillarFileCount; + this.oldestChecksumTimestamp = oldestChecksumTimestamp; } public long getPillarCollectionSize() { @@ -32,4 +38,7 @@ public long getPillarFileCount() { return pillarFileCount; } + public Instant getOldestChecksumTimestamp() { + return oldestChecksumTimestamp; + } } diff --git a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/PillarCollectionStat.java b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/PillarCollectionStat.java index fac5d8714..ed337a643 100644 --- a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/PillarCollectionStat.java +++ b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/PillarCollectionStat.java @@ -37,6 +37,9 @@ public class PillarCollectionStat { private Long obsoleteChecksums = 0L; private Long missingChecksums = 0L; private Long checksumErrors = 0L; + private String maxAgeForChecksums; + /** Human-readable age of the oldest checksum, for example " 3m 46s 299 ms" */ + private String ageOfOldestChecksum; private Date statsTime; private Date updateTime; @@ -47,9 +50,10 @@ public PillarCollectionStat(String pillarID, String collectionID, String pillarH this.pillarType = pillarType; } - public PillarCollectionStat(String pillarID, String collectionID, String pillarHostname, String pillarType, Long fileCount, - Long dataSize, Long missingFiles, Long checksumErrors, Long missingChecksums, Long obsoleteChecksum, - Date statsTime, Date updateTime) { + public PillarCollectionStat(String pillarID, String collectionID, String pillarHostname, String pillarType, + Long fileCount, Long dataSize, + Long missingFiles, Long checksumErrors, Long missingChecksums, Long obsoleteChecksum, + String maxAgeForChecksums, String ageOfOldestChecksum, Date statsTime, Date updateTime) { this.pillarID = pillarID; this.collectionID = collectionID; this.pillarHostname = pillarHostname; @@ -62,6 +66,8 @@ public PillarCollectionStat(String pillarID, String collectionID, String pillarH this.obsoleteChecksums = obsoleteChecksum; this.statsTime = statsTime; this.updateTime = updateTime; + this.maxAgeForChecksums = maxAgeForChecksums; + this.ageOfOldestChecksum = ageOfOldestChecksum; } public String getPillarID() { @@ -144,4 +150,11 @@ public void setMissingChecksums(Long missingChecksums) { this.missingChecksums = missingChecksums; } + public String getAgeOfOldestChecksum() { + return ageOfOldestChecksum; + } + + public String getMaxAgeForChecksums() { + return maxAgeForChecksums; + } } diff --git a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/database/IntegrityDAO.java b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/database/IntegrityDAO.java index 51e9d96b5..ef9c1b7e3 100644 --- a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/database/IntegrityDAO.java +++ b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/database/IntegrityDAO.java @@ -26,6 +26,7 @@ import org.bitrepository.common.ArgumentValidator; import org.bitrepository.common.utils.CalendarUtils; import org.bitrepository.common.utils.SettingsUtils; +import org.bitrepository.common.utils.TimeUtils; import org.bitrepository.integrityservice.cache.CollectionStat; import org.bitrepository.integrityservice.cache.FileInfo; import org.bitrepository.integrityservice.cache.PillarCollectionMetric; @@ -40,14 +41,8 @@ import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Date; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Optional; +import java.time.Instant; +import java.util.*; /** * Common parts of the implementation of the access to the integrity db. @@ -439,7 +434,10 @@ public void createStatistics(String collectionID, StatisticsCollector statistics public Map getPillarCollectionMetrics(String collectionID) { Map metrics = new HashMap<>(); String selectSql = - "SELECT pillarID, COUNT(fileID) as filecount, SUM(filesize) as sizesum FROM fileinfo" + " WHERE collectionID = ?" + + "SELECT pillarID, COUNT(fileID) as filecount, SUM(filesize) as sizesum," + + " MIN(checksum_timestamp) as oldest_checksum_timestamp" + + " FROM fileinfo" + + " WHERE collectionID = ?" + " GROUP BY pillarID"; try (Connection conn = dbConnector.getConnection(); @@ -447,9 +445,13 @@ public Map getPillarCollectionMetrics(String col ResultSet dbResult = ps.executeQuery()) { while (dbResult.next()) { String pillarID = dbResult.getString("pillarID"); - Long fileCount = dbResult.getLong("filecount"); - Long fileSize = dbResult.getLong("sizesum"); - PillarCollectionMetric metric = new PillarCollectionMetric(fileSize, fileCount); + long fileCount = dbResult.getLong("filecount"); + // In case SUM(filesize) returned null, dbResult.getLong() will return 0, which is the sum we want + long fileSize = dbResult.getLong("sizesum"); + long oldestChecksumTimestampMillis = dbResult.getLong("oldest_checksum_timestamp"); + Instant oldestChecksumTimestamp = + dbResult.wasNull() ? null : Instant.ofEpochMilli(oldestChecksumTimestampMillis); + PillarCollectionMetric metric = new PillarCollectionMetric(fileSize, fileCount, oldestChecksumTimestamp); metrics.put(pillarID, metric); } } catch (SQLException e) { @@ -501,8 +503,10 @@ public List getLatestPillarStats(String collectionID) { List stats = new ArrayList<>(); String latestPillarStatsSql = "SELECT pillarID, file_count, file_size, missing_files_count," + - " checksum_errors_count, missing_checksums_count, obsolete_checksums_count" + " FROM pillarstats" + " WHERE stat_key = (" + - " SELECT MAX(stat_key) FROM stats" + " WHERE collectionID = ?)"; + " checksum_errors_count, missing_checksums_count, obsolete_checksums_count," + + " 1000000000000 as oldest_checksum_timestamp" + // TODO extend table with new col + " FROM pillarstats" + + " WHERE stat_key = (" + " SELECT MAX(stat_key) FROM stats" + " WHERE collectionID = ?)"; try (Connection conn = dbConnector.getConnection(); PreparedStatement ps = DatabaseUtils.createPreparedStatement(conn, latestPillarStatsSql, collectionID)) { @@ -520,8 +524,18 @@ public List getLatestPillarStats(String collectionID) { String pillarHostname = Objects.requireNonNullElse(SettingsUtils.getHostname(pillarID), "N/A"); String pillarType = (SettingsUtils.getPillarType(pillarID) != null) ? Objects.requireNonNull(SettingsUtils.getPillarType(pillarID)).value() : "Unknown"; - PillarCollectionStat p = new PillarCollectionStat(pillarID, collectionID, pillarHostname, pillarType, fileCount, - dataSize, missingFiles, checksumErrors, missingChecksums, obsoleteChecksums, statsTime, updateTime); + long oldestChecksumTimestamp = dbResult.getLong("oldest_checksum_timestamp"); + String ageOfOldestChecksum; + if (dbResult.wasNull()) { + ageOfOldestChecksum = "N/A"; + } else { + ageOfOldestChecksum = TimeUtils.millisecondsToHuman( + System.currentTimeMillis() - oldestChecksumTimestamp); + } + PillarCollectionStat p = new PillarCollectionStat(pillarID, collectionID, + pillarHostname, pillarType, fileCount, dataSize, + missingFiles, checksumErrors, missingChecksums, obsoleteChecksums, + "TODO", ageOfOldestChecksum, statsTime, updateTime); stats.add(p); } } diff --git a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/web/RestIntegrityService.java b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/web/RestIntegrityService.java index 1b1e32293..24968062f 100644 --- a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/web/RestIntegrityService.java +++ b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/web/RestIntegrityService.java @@ -250,7 +250,7 @@ public String getIntegrityStatus( PillarType pillarTypeObject = SettingsUtils.getPillarType(pillar); String pillarType = pillarTypeObject != null ? pillarTypeObject.value() : null; PillarCollectionStat emptyStat = new PillarCollectionStat(pillar, collectionID, pillarHostname, pillarType, - 0L, 0L, 0L, 0L, 0L, 0L, new Date(0), new Date(0)); + 0L, 0L, 0L, 0L, 0L, 0L, "", "", new Date(0), new Date(0)); stats.put(pillar, emptyStat); } } @@ -422,6 +422,8 @@ private void writeIntegrityStatusObject(PillarCollectionStat stat, JsonGenerator jg.writeObjectField("checksumErrorCount", stat.getChecksumErrors()); jg.writeObjectField("obsoleteChecksumsCount", stat.getObsoleteChecksums()); jg.writeObjectField("missingChecksumsCount", stat.getMissingChecksums()); + jg.writeObjectField("maxAgeForChecksums", stat.getMaxAgeForChecksums()); + jg.writeObjectField("ageOfOldestChecksum", stat.getAgeOfOldestChecksum()); jg.writeEndObject(); } diff --git a/bitrepository-webclient/src/main/webapp/integrity-service.html b/bitrepository-webclient/src/main/webapp/integrity-service.html index 6d97edd61..a287b2279 100644 --- a/bitrepository-webclient/src/main/webapp/integrity-service.html +++ b/bitrepository-webclient/src/main/webapp/integrity-service.html @@ -85,6 +85,8 @@

Integrity service

Number of missing checksums Number of obsolete checksums Number of inconsistent checksums + Configured max age of checksums + Age of oldest checksum @@ -257,27 +259,38 @@

Modal header

context.member = "obsoleteChecksumsCount"; context.url += "getObsoleteChecksumsFileIDs/"; } else if(type == "Inconsistent checksums") { - context.element = id + "-checksumErrors"; - context.title = type + " on " + id; - context.member = "checksumErrorCount"; - context.url += "getChecksumErrorFileIDs/"; + context.element = id + "-checksumErrors"; + context.title = type + " on " + id; + context.member = "checksumErrorCount"; + context.url += "getChecksumErrorFileIDs/"; + } else if(type == "Checksum age limit") { + context.element = id + "-maxAgeForChecksums"; + context.title = type + " on " + id; + context.member = "maxAgeForChecksums"; + } else if(type == "Oldest checksum age") { + context.element = id + "-ageOfOldestChecksum"; + context.title = type + " on " + id; + context.member = "ageOfOldestChecksum"; } context.url += "?pillarID=" + id + "&collectionID=" + getCollectionID(); return context; } function makePillarRow(id) { - var html = ""; - html += ""; - html += "
" + id + "
"; - html += ""; - html += ""; - html += ""; - html += ""; - html += ""; - html += ""; - html += ""; - return html; + var html = ""; + html += ""; + html += "
" + id + "
"; + html += ""; + html += ""; + html += ""; + html += ""; + html += ""; + html += ""; + html += ""; + html += ""; + html += ""; + html += ""; + return html; } function updateCells(pillarID) { @@ -288,6 +301,8 @@

Modal header

updateIntCell(pillarID, "Missing checksums", pillars[pillarID].missingChecksumsCount); updateIntCell(pillarID, "Obsolete checksums", pillars[pillarID].obsoleteChecksumsCount); updateIntCell(pillarID, "Inconsistent checksums", pillars[pillarID].checksumErrorCount); + updateStringCell(pillarID, "Checksum age limit", pillars[pillarID].maxAgeForChecksums); + updateStringCell(pillarID, "Oldest checksum age", pillars[pillarID].ageOfOldestChecksum); } function updateStringCell(id, type, cellValue) { @@ -325,7 +340,9 @@

Modal header

missingFilesCount : j[i].missingFilesCount, missingChecksumsCount : j[i].missingChecksumsCount, obsoleteChecksumsCount : j[i].obsoleteChecksumsCount, - checksumErrorCount : j[i].checksumErrorCount}; + checksumErrorCount : j[i].checksumErrorCount, + maxAgeForChecksums : j[i].maxAgeForChecksums, + ageOfOldestChecksum : j[i].ageOfOldestChecksum}; updateCells(j[i].pillarID); } }); From 51c8beb4e9b5ee5d002f664d181dfaea1778d9e9 Mon Sep 17 00:00:00 2001 From: "Ole V. Villumsen" Date: Wed, 27 Jul 2022 12:16:38 +0200 Subject: [PATCH 02/47] Fix syntax error --- .../main/java/org/bitrepository/common/utils/SettingsUtils.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bitrepository-core/src/main/java/org/bitrepository/common/utils/SettingsUtils.java b/bitrepository-core/src/main/java/org/bitrepository/common/utils/SettingsUtils.java index 6026b1b8c..7276541db 100644 --- a/bitrepository-core/src/main/java/org/bitrepository/common/utils/SettingsUtils.java +++ b/bitrepository-core/src/main/java/org/bitrepository/common/utils/SettingsUtils.java @@ -133,7 +133,7 @@ public static String getMaxAgeForChecksums(String pillarID) { try { return TimeUtils.millisecondsToHuman(maxAge.longValueExact()); } - catch (ArithmeticException) { + catch (ArithmeticException ae) { return "Extremely long"; } } From 747f84e932d314f29f499b98d76ccf3d13aa2056 Mon Sep 17 00:00:00 2001 From: "Ole V. Villumsen" Date: Thu, 18 Aug 2022 11:04:42 +0200 Subject: [PATCH 03/47] BITMAG-1142 Show human-readable age of oldest checksum --- .../common/utils/SettingsUtils.java | 16 +++-- .../bitrepository/common/utils/TimeUtils.java | 59 ++++++++++++++++ .../common/utils/TimeUtilsTest.java | 67 ++++++++++++++++++- .../cache/IntegrityDatabaseMigrator.java | 16 ++++- .../cache/database/IntegrityDAO.java | 21 ++++-- .../sql/derby/integrityDB7to8migration.sql | 31 +++++++++ 6 files changed, 196 insertions(+), 14 deletions(-) create mode 100644 bitrepository-integrity-service/src/main/resources/sql/derby/integrityDB7to8migration.sql diff --git a/bitrepository-core/src/main/java/org/bitrepository/common/utils/SettingsUtils.java b/bitrepository-core/src/main/java/org/bitrepository/common/utils/SettingsUtils.java index 7276541db..c6a08b30c 100644 --- a/bitrepository-core/src/main/java/org/bitrepository/common/utils/SettingsUtils.java +++ b/bitrepository-core/src/main/java/org/bitrepository/common/utils/SettingsUtils.java @@ -23,6 +23,7 @@ import org.bitrepository.common.settings.Settings; import org.bitrepository.settings.referencesettings.PillarIntegrityDetails; +import org.bitrepository.settings.referencesettings.PillarSettings; import org.bitrepository.settings.referencesettings.PillarType; import org.bitrepository.settings.repositorysettings.Collection; @@ -121,20 +122,23 @@ public static String getHostname(String pillarID) { /** * Get the configured max age for checksums for the given pillarID from the ReferenceSettings. * - * @param pillarID The pillarID for which the hostname is wanted. + * @param pillarID The pillarID for which the max checksum age is wanted. * @return human-readable maximum age for checksums for the given pillar ID. */ public static String getMaxAgeForChecksums(String pillarID) { - // TODO how to use pillarID?? - BigInteger maxAge = settings.getReferenceSettings().getPillarSettings().getMaxAgeForChecksums(); - if (maxAge == null) { - return null; + PillarSettings pillarSettings = settings.getReferenceSettings().getPillarSettings(); + if (pillarSettings == null) { + return "Not set (no pillar settings)"; + } + if (! pillarSettings.getPillarID().equals(pillarID)) { + return "Unknown"; } + BigInteger maxAge = pillarSettings.getMaxAgeForChecksums(); try { return TimeUtils.millisecondsToHuman(maxAge.longValueExact()); } catch (ArithmeticException ae) { - return "Extremely long"; + return String.format(Locale.getDefault(Locale.Category.FORMAT), "Extremely long; %d ms", maxAge); } } diff --git a/bitrepository-core/src/main/java/org/bitrepository/common/utils/TimeUtils.java b/bitrepository-core/src/main/java/org/bitrepository/common/utils/TimeUtils.java index feccec24f..0fdf794f4 100644 --- a/bitrepository-core/src/main/java/org/bitrepository/common/utils/TimeUtils.java +++ b/bitrepository-core/src/main/java/org/bitrepository/common/utils/TimeUtils.java @@ -21,10 +21,17 @@ */ package org.bitrepository.common.utils; +import org.bitrepository.common.ArgumentValidator; + import javax.xml.datatype.XMLGregorianCalendar; import java.text.DateFormat; import java.text.SimpleDateFormat; +import java.time.Duration; +import java.time.Period; +import java.time.ZonedDateTime; +import java.util.ArrayList; import java.util.Date; +import java.util.List; import java.util.Locale; /** @@ -123,6 +130,57 @@ public static String millisecondsToHuman(long ms) { return sb.toString(); } + public static String humanDifference(ZonedDateTime start, ZonedDateTime end) { + ArgumentValidator.checkTrue(! end.isBefore(start), start + " > " + end); + + Period periodBetween = Period.between(start.toLocalDate(), end.toLocalDate()); + ZonedDateTime afterPeriod = start.plus(periodBetween); + if (afterPeriod.isAfter(end)) { // Too far + // One day fewer + periodBetween = Period.between(start.toLocalDate(), end.toLocalDate().minusDays(1)); + afterPeriod = start.plus(periodBetween); + } + Duration durationBetween = Duration.between(afterPeriod, end); + + if (periodBetween.isZero() && durationBetween.isZero()) { + return "0 ms"; + } + + // The following gives an ambiguous string like "3m" + // in the very rare cases where only months or only minutes are non-zero. + // Since in practice the text is updated a few seconds later, it is not expected to be a problem for the user. + List elements = new ArrayList<>(7); + if (periodBetween.getYears() != 0) { + elements.add(periodBetween.getYears() + "y"); + } + if (periodBetween.getMonths() != 0) { + elements.add(periodBetween.getMonths() + "m"); + } + if (periodBetween.getDays() != 0) { + elements.add(periodBetween.getDays() + "d"); + } + if (durationBetween.toHours() != 0) { + elements.add(durationBetween.toHours() + "h"); + } + if (durationBetween.toMinutesPart() != 0) { + elements.add(durationBetween.toMinutesPart() + "m"); + } + if (durationBetween.toSecondsPart() != 0) { + elements.add(durationBetween.toSecondsPart() + "s"); + } + if (durationBetween.toNanosPart() != 0) { + int millis = durationBetween.toMillisPart(); + int nanos = durationBetween.toNanosPart(); + if (Duration.ofMillis(millis).toNanos() == nanos) { // millis give full precision; print them + elements.add(millis + " ms"); + } else { // print only nanos + elements.add(nanos + " ns"); + } + } + + return String.join(" ", elements); + } + public static String shortDate(Date date) { return formatter.format(date); } @@ -145,4 +203,5 @@ public static Date getMaxDate(Date currentMax, Date itemDate) { return currentMax; } } + } diff --git a/bitrepository-core/src/test/java/org/bitrepository/common/utils/TimeUtilsTest.java b/bitrepository-core/src/test/java/org/bitrepository/common/utils/TimeUtilsTest.java index 8bc2186f7..a9a437352 100644 --- a/bitrepository-core/src/test/java/org/bitrepository/common/utils/TimeUtilsTest.java +++ b/bitrepository-core/src/test/java/org/bitrepository/common/utils/TimeUtilsTest.java @@ -27,6 +27,13 @@ import java.text.DateFormat; import java.text.SimpleDateFormat; +import java.time.Duration; +import java.time.Instant; +import java.time.Period; +import java.time.ZoneId; +import java.time.ZoneOffset; +import java.time.ZonedDateTime; +import java.time.temporal.TemporalAmount; import java.util.Date; import java.util.Locale; @@ -34,6 +41,8 @@ import static org.testng.Assert.assertTrue; public class TimeUtilsTest extends ExtendedTestCase { + private static final ZonedDateTime BASE = Instant.EPOCH.atZone(ZoneOffset.UTC); + @Test(groups = {"regressiontest"}) public void timeTester() throws Exception { addDescription("Tests the TimeUtils. Pi days = 271433605 milliseconds"); @@ -73,11 +82,67 @@ public void timeTester() throws Exception { @Test(groups = {"regressiontest"}) public void zeroIntervalTest() throws Exception { addDescription("Verifies that a 0 ms interval is represented correctly"); - addStep("Call the millisecondsToHuman with 0 ms", "The output should be '0 ms'"); + addStep("Call millisecondsToHuman with 0 ms", "The output should be '0 ms'"); String zeroTimeString = TimeUtils.millisecondsToHuman(0); assertEquals(zeroTimeString, " 0 ms"); } + @Test(groups = {"regressiontest"}) + public void differencesPrintHumanly() { + addDescription("TimeUtils.humanDifference() should return" + + " similar human readable strings to those from millisecondsToHuman()"); + + addStep("Call humanDifference() with same time twice", "The output should be '0 ms'"); + String zeroTimeString = TimeUtils.humanDifference(BASE, BASE); + assertEquals(zeroTimeString, "0 ms"); + + addStep("Call humanDifference() with a difference obtained from a Duration", + "Expect corresponding readable output"); + testHumanDifference("1 ns", Duration.ofNanos(1)); + testHumanDifference("1 ms", Duration.ofMillis(1)); + // If there are nanos, don’t print millis + testHumanDifference("2000003 ns", Duration.ofNanos(2_000_003)); + testHumanDifference("1s", Duration.ofSeconds(1)); + testHumanDifference("1m", Duration.ofMinutes(1)); + testHumanDifference("1h", Duration.ofHours(1)); + testHumanDifference("2h 3m 5s 7 ns", Duration.parse("PT2H3M5.000000007S")); + + addStep("Call humanDifference() with a difference obtained from a Period", + "Expect corresponding readable output"); + testHumanDifference("1d", Period.ofDays(1)); + testHumanDifference("1m", Period.ofMonths(1)); + testHumanDifference("1y", Period.ofYears(1)); + testHumanDifference("2y 3m 5d", Period.of(2, 3, 5)); + + addStep("Call humanDifference() with a difference obtained from a combo of a Period and a Duration", + "Expect corresponding readable output"); + testHumanDifference("3y 5m 7d 11h 13m 17s 23 ms", + Period.of(3, 5, 7), Duration.parse("PT11H13M17.023S")); + + addStep("Call humanDifference()" + + " with dates that are 2 days apart but times that cause the diff to be less than 2 full days", + "Expect output 1d something"); + ZoneId testZoneId = ZoneId.of("Europe/Vienna"); + String oneDaySomethingString = TimeUtils.humanDifference( + ZonedDateTime.of(2021, 1, 31, + 12, 0, 0, 0, testZoneId), + ZonedDateTime.of(2021, 2, 2, + 11, 59, 59, 0, testZoneId)); + assertEquals("1d 23h 59m 59s", oneDaySomethingString); + } + + /** + * Note that the expected result comes first in the argument list + * so that we can use varargs to pass a number of amounts, for example both a Period and a Duration. + */ + private void testHumanDifference(String expected, TemporalAmount... amounts) { + ZonedDateTime end = BASE; + for (TemporalAmount amount: amounts) { + end = end.plus(amount); + } + String differenceString = TimeUtils.humanDifference(BASE, end); + assertEquals(differenceString, expected); + } /* * The test only ensures that the output format is fixed. Which timezone the the date is diff --git a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/IntegrityDatabaseMigrator.java b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/IntegrityDatabaseMigrator.java index f93b36edb..34f8b62a5 100644 --- a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/IntegrityDatabaseMigrator.java +++ b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/IntegrityDatabaseMigrator.java @@ -30,7 +30,7 @@ import java.util.Map; /** - * Migration class for the AuditTrailDatabase of the AuditTrailService. + * Migration class for the IntegrityDatabaseDatabase of the IntegrityDatabaseService. * Will only try to perform the migration on an embedded derby database. */ public class IntegrityDatabaseMigrator extends DatabaseMigrator { @@ -63,10 +63,13 @@ public class IntegrityDatabaseMigrator extends DatabaseMigrator { * The name of the update script for version 6 to 7. */ private static final String UPDATE_SCRIPT_VERSION_6_TO_7 = "sql/derby/integrityDB6to7migration.sql"; + + private static final String UPDATE_SCRIPT_VERSION_7_TO_8 = "sql/derby/integrityDB7to8migration.sql"; + /** * The current version of the database. */ - private final Integer currentVersion = 7; + private final int currentVersion = 8; /** * @param connector connection to the database. @@ -114,6 +117,15 @@ public void migrate() { log.warn("Migrating integrityDB from version 6 to 7"); migrateDerbyDatabase(UPDATE_SCRIPT_VERSION_6_TO_7); } + if (versions.get(DATABASE_VERSION_ENTRY) < 8) { + log.warn("Migrating integrityDB from version 7 to 8"); + migrateDerbyDatabase(UPDATE_SCRIPT_VERSION_7_TO_8); + } + + if (needsMigration()) { + log.error("Database still appears to need migration after it has been migrated. Expected version is {}.", + currentVersion); + } } @Override diff --git a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/database/IntegrityDAO.java b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/database/IntegrityDAO.java index ef9c1b7e3..3c6ed29c9 100644 --- a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/database/IntegrityDAO.java +++ b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/database/IntegrityDAO.java @@ -42,7 +42,16 @@ import java.sql.ResultSet; import java.sql.SQLException; import java.time.Instant; -import java.util.*; +import java.time.ZoneId; +import java.time.ZonedDateTime; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; /** * Common parts of the implementation of the access to the integrity db. @@ -504,7 +513,7 @@ public List getLatestPillarStats(String collectionID) { String latestPillarStatsSql = "SELECT pillarID, file_count, file_size, missing_files_count," + " checksum_errors_count, missing_checksums_count, obsolete_checksums_count," + - " 1000000000000 as oldest_checksum_timestamp" + // TODO extend table with new col + " 1000000000000 as oldest_checksum_timestamp" + // TODO Ole V. use new col " FROM pillarstats" + " WHERE stat_key = (" + " SELECT MAX(stat_key) FROM stats" + " WHERE collectionID = ?)"; @@ -524,18 +533,20 @@ public List getLatestPillarStats(String collectionID) { String pillarHostname = Objects.requireNonNullElse(SettingsUtils.getHostname(pillarID), "N/A"); String pillarType = (SettingsUtils.getPillarType(pillarID) != null) ? Objects.requireNonNull(SettingsUtils.getPillarType(pillarID)).value() : "Unknown"; + String maxAgeForChecksums = SettingsUtils.getMaxAgeForChecksums(pillarID); long oldestChecksumTimestamp = dbResult.getLong("oldest_checksum_timestamp"); String ageOfOldestChecksum; if (dbResult.wasNull()) { ageOfOldestChecksum = "N/A"; } else { - ageOfOldestChecksum = TimeUtils.millisecondsToHuman( - System.currentTimeMillis() - oldestChecksumTimestamp); + ZoneId zone = ZoneId.systemDefault(); + ZonedDateTime oldestChecksumZdt = Instant.ofEpochMilli(oldestChecksumTimestamp).atZone(zone); + ageOfOldestChecksum = TimeUtils.humanDifference(oldestChecksumZdt, ZonedDateTime.now(zone)); } PillarCollectionStat p = new PillarCollectionStat(pillarID, collectionID, pillarHostname, pillarType, fileCount, dataSize, missingFiles, checksumErrors, missingChecksums, obsoleteChecksums, - "TODO", ageOfOldestChecksum, statsTime, updateTime); + maxAgeForChecksums, ageOfOldestChecksum, statsTime, updateTime); stats.add(p); } } diff --git a/bitrepository-integrity-service/src/main/resources/sql/derby/integrityDB7to8migration.sql b/bitrepository-integrity-service/src/main/resources/sql/derby/integrityDB7to8migration.sql new file mode 100644 index 000000000..44fcde9bf --- /dev/null +++ b/bitrepository-integrity-service/src/main/resources/sql/derby/integrityDB7to8migration.sql @@ -0,0 +1,31 @@ +--- +-- #%L +-- Bitrepository Integrity Client +-- %% +-- Copyright (C) 2010 - 2022 Royal Danish Library and The State Archives, Denmark +-- %% +-- This program is free software: you can redistribute it and/or modify +-- it under the terms of the GNU Lesser General Public License as +-- published by the Free Software Foundation, either version 2.1 of the +-- License, or (at your option) any later version. +-- +-- This program is distributed in the hope that it will be useful, +-- but WITHOUT ANY WARRANTY; without even the implied warranty of +-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +-- GNU General Lesser Public License for more details. +-- +-- You should have received a copy of the GNU General Lesser Public +-- License along with this program. If not, see +-- . +-- #L% +--- + +connect 'jdbc:derby:integritydb'; + +-- database version +UPDATE tableversions SET version = 8 WHERE tablename = 'integritydb'; + +-- table version +UPDATE tableversions SET version = 3 WHERE tablename = 'pillarstats'; + +ALTER TABLE pillarstats ADD COLUMN oldest_checksum_timestamp BIGINT DEFAULT NULL; From b4a25255a232dd8f5a9e7d3adb12d24ecec5068a Mon Sep 17 00:00:00 2001 From: "Ole V. Villumsen" Date: Thu, 18 Aug 2022 14:13:48 +0200 Subject: [PATCH 04/47] BITMAG-1142 Display configured max age from MaxChecksumAgeProvider --- .../common/utils/SettingsUtils.java | 50 +++++++++++-------- .../cache/database/IntegrityDAO.java | 45 ++++++++++++----- 2 files changed, 61 insertions(+), 34 deletions(-) diff --git a/bitrepository-core/src/main/java/org/bitrepository/common/utils/SettingsUtils.java b/bitrepository-core/src/main/java/org/bitrepository/common/utils/SettingsUtils.java index c6a08b30c..6c9fc1252 100644 --- a/bitrepository-core/src/main/java/org/bitrepository/common/utils/SettingsUtils.java +++ b/bitrepository-core/src/main/java/org/bitrepository/common/utils/SettingsUtils.java @@ -22,8 +22,8 @@ package org.bitrepository.common.utils; import org.bitrepository.common.settings.Settings; +import org.bitrepository.settings.referencesettings.IntegrityServiceSettings; import org.bitrepository.settings.referencesettings.PillarIntegrityDetails; -import org.bitrepository.settings.referencesettings.PillarSettings; import org.bitrepository.settings.referencesettings.PillarType; import org.bitrepository.settings.repositorysettings.Collection; @@ -119,27 +119,32 @@ public static String getHostname(String pillarID) { return null; } - /** - * Get the configured max age for checksums for the given pillarID from the ReferenceSettings. - * - * @param pillarID The pillarID for which the max checksum age is wanted. - * @return human-readable maximum age for checksums for the given pillar ID. - */ - public static String getMaxAgeForChecksums(String pillarID) { - PillarSettings pillarSettings = settings.getReferenceSettings().getPillarSettings(); - if (pillarSettings == null) { - return "Not set (no pillar settings)"; - } - if (! pillarSettings.getPillarID().equals(pillarID)) { - return "Unknown"; - } - BigInteger maxAge = pillarSettings.getMaxAgeForChecksums(); - try { - return TimeUtils.millisecondsToHuman(maxAge.longValueExact()); - } - catch (ArithmeticException ae) { - return String.format(Locale.getDefault(Locale.Category.FORMAT), "Extremely long; %d ms", maxAge); - } + // TODO Ole V. delete method +// /** +// * Get the configured max age for checksums for the given pillarID from the ReferenceSettings. +// * +// * @param pillarID The pillarID for which the max checksum age is wanted. +// * @return human-readable maximum age for checksums for the given pillar ID. +// */ +// public static String getMaxAgeForChecksums(String pillarID) { +// PillarSettings pillarSettings = settings.getReferenceSettings().getPillarSettings(); +// if (pillarSettings == null) { +// return "Not set (no pillar settings)"; +// } +// if (! pillarSettings.getPillarID().equals(pillarID)) { +// return "Unknown"; +// } +// BigInteger maxAge = pillarSettings.getMaxAgeForChecksums(); +// try { +// return TimeUtils.millisecondsToHuman(maxAge.longValueExact()); +// } +// catch (ArithmeticException ae) { +// return String.format(Locale.getDefault(Locale.Category.FORMAT), "Extremely long; %d ms", maxAge); +// } +// } + + public static IntegrityServiceSettings getIntegrityServiceSettings() { + return settings.getReferenceSettings().getIntegrityServiceSettings(); } /** @@ -231,4 +236,5 @@ public static Set getStatusContributorsForCollection() { contributors.addAll(SettingsUtils.getAllPillarIDs()); return contributors; } + } diff --git a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/database/IntegrityDAO.java b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/database/IntegrityDAO.java index 3c6ed29c9..fb7d49926 100644 --- a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/database/IntegrityDAO.java +++ b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/database/IntegrityDAO.java @@ -31,9 +31,13 @@ import org.bitrepository.integrityservice.cache.FileInfo; import org.bitrepository.integrityservice.cache.PillarCollectionMetric; import org.bitrepository.integrityservice.cache.PillarCollectionStat; +import org.bitrepository.integrityservice.checking.MaxChecksumAgeProvider; import org.bitrepository.integrityservice.statistics.StatisticsCollector; +import org.bitrepository.integrityservice.workflow.step.HandleObsoleteChecksumsStep; import org.bitrepository.service.database.DBConnector; import org.bitrepository.service.database.DatabaseUtils; +import org.bitrepository.settings.referencesettings.ObsoleteChecksumSettings; +import org.jetbrains.annotations.NotNull; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -79,12 +83,12 @@ public void close() { } /** - * Method to ensure that pillars found in RepositorySettings is present in the database + * Method to ensure that pillars found in RepositorySettings are present in the database */ protected abstract void initializePillars(); /** - * Method to ensure that collections found in RepositorySettings is present in the database + * Method to ensure that collections found in RepositorySettings are present in the database */ protected abstract void initializeCollections(); @@ -533,16 +537,8 @@ public List getLatestPillarStats(String collectionID) { String pillarHostname = Objects.requireNonNullElse(SettingsUtils.getHostname(pillarID), "N/A"); String pillarType = (SettingsUtils.getPillarType(pillarID) != null) ? Objects.requireNonNull(SettingsUtils.getPillarType(pillarID)).value() : "Unknown"; - String maxAgeForChecksums = SettingsUtils.getMaxAgeForChecksums(pillarID); - long oldestChecksumTimestamp = dbResult.getLong("oldest_checksum_timestamp"); - String ageOfOldestChecksum; - if (dbResult.wasNull()) { - ageOfOldestChecksum = "N/A"; - } else { - ZoneId zone = ZoneId.systemDefault(); - ZonedDateTime oldestChecksumZdt = Instant.ofEpochMilli(oldestChecksumTimestamp).atZone(zone); - ageOfOldestChecksum = TimeUtils.humanDifference(oldestChecksumZdt, ZonedDateTime.now(zone)); - } + String maxAgeForChecksums = getMaxAgeForChecksums(pillarID); + String ageOfOldestChecksum = getAgeOfOldestChecksum(dbResult); PillarCollectionStat p = new PillarCollectionStat(pillarID, collectionID, pillarHostname, pillarType, fileCount, dataSize, missingFiles, checksumErrors, missingChecksums, obsoleteChecksums, @@ -559,6 +555,31 @@ public List getLatestPillarStats(String collectionID) { return stats; } + @NotNull + private String getMaxAgeForChecksums(String pillarID) { + ObsoleteChecksumSettings obsoleteChecksumSettings = + SettingsUtils.getIntegrityServiceSettings().getObsoleteChecksumSettings(); + MaxChecksumAgeProvider maxChecksumAgeProvider = + new MaxChecksumAgeProvider(HandleObsoleteChecksumsStep.DEFAULT_MAX_CHECKSUM_AGE, + obsoleteChecksumSettings); + long maxAge = maxChecksumAgeProvider.getMaxChecksumAge(pillarID); + return maxAge == 0 ? "unlimited" : TimeUtils.millisecondsToHuman(maxAge); + } + + @NotNull + private String getAgeOfOldestChecksum(ResultSet dbResult) throws SQLException { + long oldestChecksumTimestamp = dbResult.getLong("oldest_checksum_timestamp"); + String ageOfOldestChecksum; + if (dbResult.wasNull()) { + ageOfOldestChecksum = "N/A"; + } else { + ZoneId zone = ZoneId.systemDefault(); + ZonedDateTime oldestChecksumZdt = Instant.ofEpochMilli(oldestChecksumTimestamp).atZone(zone); + ageOfOldestChecksum = TimeUtils.humanDifference(oldestChecksumZdt, ZonedDateTime.now(zone)); + } + return ageOfOldestChecksum; + } + /** * Method that should deliver the database specific SQL for getting the latest N collection statistics * From b0389c99a623201535e7f51dda5993f880becd3a Mon Sep 17 00:00:00 2001 From: "Ole V. Villumsen" Date: Fri, 19 Aug 2022 09:26:37 +0200 Subject: [PATCH 05/47] BITMAG-1142 Use real checksum timestamp from database --- .../integrityservice/cache/database/IntegrityDAO.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/database/IntegrityDAO.java b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/database/IntegrityDAO.java index fb7d49926..9e8d73840 100644 --- a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/database/IntegrityDAO.java +++ b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/database/IntegrityDAO.java @@ -517,9 +517,12 @@ public List getLatestPillarStats(String collectionID) { String latestPillarStatsSql = "SELECT pillarID, file_count, file_size, missing_files_count," + " checksum_errors_count, missing_checksums_count, obsolete_checksums_count," + - " 1000000000000 as oldest_checksum_timestamp" + // TODO Ole V. use new col + " oldest_checksum_timestamp" + " FROM pillarstats" + - " WHERE stat_key = (" + " SELECT MAX(stat_key) FROM stats" + " WHERE collectionID = ?)"; + " WHERE stat_key = (" + + " SELECT MAX(stat_key)" + + " FROM stats" + + " WHERE collectionID = ?)"; try (Connection conn = dbConnector.getConnection(); PreparedStatement ps = DatabaseUtils.createPreparedStatement(conn, latestPillarStatsSql, collectionID)) { From 9fb4d79d7ce0d65efb26ed15db001ae23d16999b Mon Sep 17 00:00:00 2001 From: "Ole V. Villumsen" Date: Wed, 24 Aug 2022 15:03:30 +0200 Subject: [PATCH 06/47] BITMAG-1142 Fix bugs and missing parts in display of age of oldest checksum --- .../bitrepository/common/utils/TimeUtils.java | 26 ++++++-------- .../common/utils/TimeUtilsTest.java | 17 ++++----- .../cache/PillarCollectionStat.java | 36 ++++++++++++++++--- .../cache/database/IntegrityDAO.java | 19 +++------- .../cache/database/StatisticsCreator.java | 33 +++++++++-------- .../web/RestIntegrityService.java | 7 ++-- .../step/CreateStatisticsEntryStep.java | 2 ++ 7 files changed, 79 insertions(+), 61 deletions(-) diff --git a/bitrepository-core/src/main/java/org/bitrepository/common/utils/TimeUtils.java b/bitrepository-core/src/main/java/org/bitrepository/common/utils/TimeUtils.java index 0fdf794f4..7e176e8bf 100644 --- a/bitrepository-core/src/main/java/org/bitrepository/common/utils/TimeUtils.java +++ b/bitrepository-core/src/main/java/org/bitrepository/common/utils/TimeUtils.java @@ -29,6 +29,7 @@ import java.time.Duration; import java.time.Period; import java.time.ZonedDateTime; +import java.time.temporal.ChronoUnit; import java.util.ArrayList; import java.util.Date; import java.util.List; @@ -140,16 +141,20 @@ public static String humanDifference(ZonedDateTime start, ZonedDateTime end) { periodBetween = Period.between(start.toLocalDate(), end.toLocalDate().minusDays(1)); afterPeriod = start.plus(periodBetween); } - Duration durationBetween = Duration.between(afterPeriod, end); + // Round duration to whole seconds + Duration durationBetween = Duration.between(afterPeriod, end) + .plusMillis(500) + .truncatedTo(ChronoUnit.SECONDS); if (periodBetween.isZero() && durationBetween.isZero()) { - return "0 ms"; + return "0s"; } - // The following gives an ambiguous string like "3m" - // in the very rare cases where only months or only minutes are non-zero. - // Since in practice the text is updated a few seconds later, it is not expected to be a problem for the user. - List elements = new ArrayList<>(7); + // The following gives an ambiguous string like "3m" or "3y 11m 5s" + // in the very rare cases where months *or* minutes are non-zero and days and hours are zero. + // Since in practice the text is updated a few seconds later and any minutes 1 minute later, + // it is not expected to be a problem for the user. + List elements = new ArrayList<>(6); if (periodBetween.getYears() != 0) { elements.add(periodBetween.getYears() + "y"); } @@ -168,15 +173,6 @@ public static String humanDifference(ZonedDateTime start, ZonedDateTime end) { if (durationBetween.toSecondsPart() != 0) { elements.add(durationBetween.toSecondsPart() + "s"); } - if (durationBetween.toNanosPart() != 0) { - int millis = durationBetween.toMillisPart(); - int nanos = durationBetween.toNanosPart(); - if (Duration.ofMillis(millis).toNanos() == nanos) { // millis give full precision; print them - elements.add(millis + " ms"); - } else { // print only nanos - elements.add(nanos + " ns"); - } - } return String.join(" ", elements); } diff --git a/bitrepository-core/src/test/java/org/bitrepository/common/utils/TimeUtilsTest.java b/bitrepository-core/src/test/java/org/bitrepository/common/utils/TimeUtilsTest.java index a9a437352..d0297d9de 100644 --- a/bitrepository-core/src/test/java/org/bitrepository/common/utils/TimeUtilsTest.java +++ b/bitrepository-core/src/test/java/org/bitrepository/common/utils/TimeUtilsTest.java @@ -92,20 +92,21 @@ public void differencesPrintHumanly() { addDescription("TimeUtils.humanDifference() should return" + " similar human readable strings to those from millisecondsToHuman()"); - addStep("Call humanDifference() with same time twice", "The output should be '0 ms'"); + addStep("Call humanDifference() with same time twice", "The output should be '0s'"); String zeroTimeString = TimeUtils.humanDifference(BASE, BASE); - assertEquals(zeroTimeString, "0 ms"); + assertEquals(zeroTimeString, "0s"); addStep("Call humanDifference() with a difference obtained from a Duration", "Expect corresponding readable output"); - testHumanDifference("1 ns", Duration.ofNanos(1)); - testHumanDifference("1 ms", Duration.ofMillis(1)); - // If there are nanos, don’t print millis - testHumanDifference("2000003 ns", Duration.ofNanos(2_000_003)); + // Don’t print fraction of second + testHumanDifference("0s", Duration.ofNanos(1)); + testHumanDifference("0s", Duration.ofMillis(1)); + testHumanDifference("0s", Duration.ofNanos(499_999_999)); + testHumanDifference("1s", Duration.ofNanos(500_000_000)); testHumanDifference("1s", Duration.ofSeconds(1)); testHumanDifference("1m", Duration.ofMinutes(1)); testHumanDifference("1h", Duration.ofHours(1)); - testHumanDifference("2h 3m 5s 7 ns", Duration.parse("PT2H3M5.000000007S")); + testHumanDifference("2h 3m 5s", Duration.parse("PT2H3M5.000000007S")); addStep("Call humanDifference() with a difference obtained from a Period", "Expect corresponding readable output"); @@ -116,7 +117,7 @@ public void differencesPrintHumanly() { addStep("Call humanDifference() with a difference obtained from a combo of a Period and a Duration", "Expect corresponding readable output"); - testHumanDifference("3y 5m 7d 11h 13m 17s 23 ms", + testHumanDifference("3y 5m 7d 11h 13m 17s", Period.of(3, 5, 7), Duration.parse("PT11H13M17.023S")); addStep("Call humanDifference()" + diff --git a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/PillarCollectionStat.java b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/PillarCollectionStat.java index ed337a643..a4621120c 100644 --- a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/PillarCollectionStat.java +++ b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/PillarCollectionStat.java @@ -21,6 +21,12 @@ */ package org.bitrepository.integrityservice.cache; +import org.bitrepository.common.utils.TimeUtils; +import org.jetbrains.annotations.NotNull; + +import java.time.Instant; +import java.time.ZoneId; +import java.time.ZonedDateTime; import java.util.Date; /** @@ -38,8 +44,7 @@ public class PillarCollectionStat { private Long missingChecksums = 0L; private Long checksumErrors = 0L; private String maxAgeForChecksums; - /** Human-readable age of the oldest checksum, for example " 3m 46s 299 ms" */ - private String ageOfOldestChecksum; + private Instant oldestChecksumTimestamp; private Date statsTime; private Date updateTime; @@ -53,7 +58,8 @@ public PillarCollectionStat(String pillarID, String collectionID, String pillarH public PillarCollectionStat(String pillarID, String collectionID, String pillarHostname, String pillarType, Long fileCount, Long dataSize, Long missingFiles, Long checksumErrors, Long missingChecksums, Long obsoleteChecksum, - String maxAgeForChecksums, String ageOfOldestChecksum, Date statsTime, Date updateTime) { + String maxAgeForChecksums, + Instant oldestChecksumTimestamp, Date statsTime, Date updateTime) { this.pillarID = pillarID; this.collectionID = collectionID; this.pillarHostname = pillarHostname; @@ -67,7 +73,7 @@ public PillarCollectionStat(String pillarID, String collectionID, String pillarH this.statsTime = statsTime; this.updateTime = updateTime; this.maxAgeForChecksums = maxAgeForChecksums; - this.ageOfOldestChecksum = ageOfOldestChecksum; + this.oldestChecksumTimestamp = oldestChecksumTimestamp; } public String getPillarID() { @@ -150,11 +156,31 @@ public void setMissingChecksums(Long missingChecksums) { this.missingChecksums = missingChecksums; } + /** @return Human-readable age of the oldest checksum, for example "3m 46s" */ + @NotNull public String getAgeOfOldestChecksum() { - return ageOfOldestChecksum; + if (oldestChecksumTimestamp == null) { + return "N/A"; + } + ZoneId zone = ZoneId.systemDefault(); + return TimeUtils.humanDifference(oldestChecksumTimestamp.atZone(zone), ZonedDateTime.now(zone)); + } + + public boolean hasOldestChecksumTimestamp() { + return oldestChecksumTimestamp != null; + } + + /** @throws NullPointerException if hasOldestChecksumTimestamp() does not return true */ + public long getOldestChecksumTimestampMillis() { + return oldestChecksumTimestamp.toEpochMilli(); + } + + public void setOldestChecksumTimestamp(Instant oldestChecksumTimestamp) { + this.oldestChecksumTimestamp = oldestChecksumTimestamp; } public String getMaxAgeForChecksums() { return maxAgeForChecksums; } + } diff --git a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/database/IntegrityDAO.java b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/database/IntegrityDAO.java index 9e8d73840..c75288b26 100644 --- a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/database/IntegrityDAO.java +++ b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/database/IntegrityDAO.java @@ -46,8 +46,6 @@ import java.sql.ResultSet; import java.sql.SQLException; import java.time.Instant; -import java.time.ZoneId; -import java.time.ZonedDateTime; import java.util.ArrayList; import java.util.Arrays; import java.util.Date; @@ -541,11 +539,11 @@ public List getLatestPillarStats(String collectionID) { String pillarType = (SettingsUtils.getPillarType(pillarID) != null) ? Objects.requireNonNull(SettingsUtils.getPillarType(pillarID)).value() : "Unknown"; String maxAgeForChecksums = getMaxAgeForChecksums(pillarID); - String ageOfOldestChecksum = getAgeOfOldestChecksum(dbResult); + Instant oldestChecksumTimestamp = getOldestChecksumTimestamp(dbResult); PillarCollectionStat p = new PillarCollectionStat(pillarID, collectionID, pillarHostname, pillarType, fileCount, dataSize, missingFiles, checksumErrors, missingChecksums, obsoleteChecksums, - maxAgeForChecksums, ageOfOldestChecksum, statsTime, updateTime); + maxAgeForChecksums, oldestChecksumTimestamp, statsTime, updateTime); stats.add(p); } } @@ -569,18 +567,9 @@ private String getMaxAgeForChecksums(String pillarID) { return maxAge == 0 ? "unlimited" : TimeUtils.millisecondsToHuman(maxAge); } - @NotNull - private String getAgeOfOldestChecksum(ResultSet dbResult) throws SQLException { + private Instant getOldestChecksumTimestamp(ResultSet dbResult) throws SQLException { long oldestChecksumTimestamp = dbResult.getLong("oldest_checksum_timestamp"); - String ageOfOldestChecksum; - if (dbResult.wasNull()) { - ageOfOldestChecksum = "N/A"; - } else { - ZoneId zone = ZoneId.systemDefault(); - ZonedDateTime oldestChecksumZdt = Instant.ofEpochMilli(oldestChecksumTimestamp).atZone(zone); - ageOfOldestChecksum = TimeUtils.humanDifference(oldestChecksumZdt, ZonedDateTime.now(zone)); - } - return ageOfOldestChecksum; + return dbResult.wasNull() ? null : Instant.ofEpochMilli(oldestChecksumTimestamp); } /** diff --git a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/database/StatisticsCreator.java b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/database/StatisticsCreator.java index cefb53bd9..c0409ee6d 100644 --- a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/database/StatisticsCreator.java +++ b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/database/StatisticsCreator.java @@ -28,9 +28,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.sql.Connection; -import java.sql.PreparedStatement; -import java.sql.SQLException; +import java.sql.*; import java.util.Date; import java.util.List; @@ -49,8 +47,8 @@ public class StatisticsCreator { private final String insertPillarStatEntrySql = "INSERT INTO pillarstats" + " (stat_key, pillarID, file_count, file_size, missing_files_count, " - + "checksum_errors_count, missing_checksums_count, obsolete_checksums_count)" - + " (SELECT MAX(stat_key), ?, ?, ?, ?, ?, ?, ? FROM stats WHERE collectionID = ?)"; + + "checksum_errors_count, missing_checksums_count, obsolete_checksums_count, oldest_checksum_timestamp)" + + " (SELECT MAX(stat_key), ?, ?, ?, ?, ?, ?, ?, ? FROM stats WHERE collectionID = ?)"; private final Logger log = LoggerFactory.getLogger(getClass()); @@ -112,19 +110,24 @@ private void addCollectionStatistics(CollectionStat cs) throws SQLException { insertCollectionStatPS.setString(5, cs.getCollectionID()); } - private void addPillarStat(PillarCollectionStat ps) throws SQLException { - insertPillarStatPS.setString(1, ps.getPillarID()); - insertPillarStatPS.setLong(2, ps.getFileCount()); - insertPillarStatPS.setLong(3, ps.getDataSize()); - insertPillarStatPS.setLong(4, ps.getMissingFiles()); - insertPillarStatPS.setLong(5, ps.getChecksumErrors()); - insertPillarStatPS.setLong(6, ps.getMissingChecksums()); - insertPillarStatPS.setLong(7, ps.getObsoleteChecksums()); - insertPillarStatPS.setString(8, ps.getCollectionID()); + private void addPillarStat(PillarCollectionStat pcStat) throws SQLException { + insertPillarStatPS.setString(1, pcStat.getPillarID()); + insertPillarStatPS.setLong(2, pcStat.getFileCount()); + insertPillarStatPS.setLong(3, pcStat.getDataSize()); + insertPillarStatPS.setLong(4, pcStat.getMissingFiles()); + insertPillarStatPS.setLong(5, pcStat.getChecksumErrors()); + insertPillarStatPS.setLong(6, pcStat.getMissingChecksums()); + insertPillarStatPS.setLong(7, pcStat.getObsoleteChecksums()); + if (pcStat.hasOldestChecksumTimestamp()) { + insertPillarStatPS.setLong(8, pcStat.getOldestChecksumTimestampMillis()); + } else { + insertPillarStatPS.setNull(8, Types.BIGINT); + } + + insertPillarStatPS.setString(9, pcStat.getCollectionID()); insertPillarStatPS.addBatch(); } - private void execute() throws SQLException { insertStatisticsEntryPS.execute(); insertCollectionStatPS.execute(); diff --git a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/web/RestIntegrityService.java b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/web/RestIntegrityService.java index 24968062f..da606a45b 100644 --- a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/web/RestIntegrityService.java +++ b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/web/RestIntegrityService.java @@ -249,8 +249,9 @@ public String getIntegrityStatus( String pillarHostname = Objects.requireNonNullElse(SettingsUtils.getHostname(pillar), "N/A"); PillarType pillarTypeObject = SettingsUtils.getPillarType(pillar); String pillarType = pillarTypeObject != null ? pillarTypeObject.value() : null; - PillarCollectionStat emptyStat = new PillarCollectionStat(pillar, collectionID, pillarHostname, pillarType, - 0L, 0L, 0L, 0L, 0L, 0L, "", "", new Date(0), new Date(0)); + PillarCollectionStat emptyStat = new PillarCollectionStat(pillar, collectionID, pillarHostname, + pillarType, 0L, 0L, 0L, 0L, 0L, + 0L, "", null, new Date(0), new Date(0)); stats.put(pillar, emptyStat); } } @@ -266,7 +267,7 @@ public String getIntegrityStatus( } /*** - * Get the current workflows setup as a JSON array + * Get the current workflow’s setup as a JSON array */ @GET @Path("/getWorkflowSetup/") diff --git a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/workflow/step/CreateStatisticsEntryStep.java b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/workflow/step/CreateStatisticsEntryStep.java index 40427b1f3..fab49ba3a 100644 --- a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/workflow/step/CreateStatisticsEntryStep.java +++ b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/workflow/step/CreateStatisticsEntryStep.java @@ -63,9 +63,11 @@ public synchronized void performStep() { if (metric == null) { sc.getPillarCollectionStat(pillar).setFileCount(0L); sc.getPillarCollectionStat(pillar).setDataSize(0L); + sc.getPillarCollectionStat(pillar).setOldestChecksumTimestamp(null); } else { sc.getPillarCollectionStat(pillar).setFileCount(metric.getPillarFileCount()); sc.getPillarCollectionStat(pillar).setDataSize(metric.getPillarCollectionSize()); + sc.getPillarCollectionStat(pillar).setOldestChecksumTimestamp(metric.getOldestChecksumTimestamp()); } } sc.getCollectionStat().setFileCount(store.getNumberOfFilesInCollection(collectionID)); From d203f711940b5170e4b12892810ffc43555b03dd Mon Sep 17 00:00:00 2001 From: "Ole V. Villumsen" Date: Thu, 25 Aug 2022 13:01:50 +0200 Subject: [PATCH 07/47] BITMAG-1142 Delete method that is no longer used after change --- .../common/utils/SettingsUtils.java | 24 ------------------- 1 file changed, 24 deletions(-) diff --git a/bitrepository-core/src/main/java/org/bitrepository/common/utils/SettingsUtils.java b/bitrepository-core/src/main/java/org/bitrepository/common/utils/SettingsUtils.java index 6c9fc1252..9449dd70b 100644 --- a/bitrepository-core/src/main/java/org/bitrepository/common/utils/SettingsUtils.java +++ b/bitrepository-core/src/main/java/org/bitrepository/common/utils/SettingsUtils.java @@ -119,30 +119,6 @@ public static String getHostname(String pillarID) { return null; } - // TODO Ole V. delete method -// /** -// * Get the configured max age for checksums for the given pillarID from the ReferenceSettings. -// * -// * @param pillarID The pillarID for which the max checksum age is wanted. -// * @return human-readable maximum age for checksums for the given pillar ID. -// */ -// public static String getMaxAgeForChecksums(String pillarID) { -// PillarSettings pillarSettings = settings.getReferenceSettings().getPillarSettings(); -// if (pillarSettings == null) { -// return "Not set (no pillar settings)"; -// } -// if (! pillarSettings.getPillarID().equals(pillarID)) { -// return "Unknown"; -// } -// BigInteger maxAge = pillarSettings.getMaxAgeForChecksums(); -// try { -// return TimeUtils.millisecondsToHuman(maxAge.longValueExact()); -// } -// catch (ArithmeticException ae) { -// return String.format(Locale.getDefault(Locale.Category.FORMAT), "Extremely long; %d ms", maxAge); -// } -// } - public static IntegrityServiceSettings getIntegrityServiceSettings() { return settings.getReferenceSettings().getIntegrityServiceSettings(); } From 10b99c1d81fecc0f5cb18d9038820dc11eb16323 Mon Sep 17 00:00:00 2001 From: "Ole V. Villumsen" Date: Tue, 26 Jul 2022 12:55:59 +0200 Subject: [PATCH 08/47] BITMAG-1142 Checksum age in GUI: underway --- .../common/utils/SettingsUtils.java | 20 ++++++++ .../cache/PillarCollectionMetric.java | 11 ++++- .../cache/PillarCollectionStat.java | 19 ++++++-- .../cache/database/IntegrityDAO.java | 46 ++++++++++++------- .../web/RestIntegrityService.java | 4 +- .../src/main/webapp/integrity-service.html | 23 ++++++++-- 6 files changed, 99 insertions(+), 24 deletions(-) diff --git a/bitrepository-core/src/main/java/org/bitrepository/common/utils/SettingsUtils.java b/bitrepository-core/src/main/java/org/bitrepository/common/utils/SettingsUtils.java index fea26d3ab..6fc9fbd66 100644 --- a/bitrepository-core/src/main/java/org/bitrepository/common/utils/SettingsUtils.java +++ b/bitrepository-core/src/main/java/org/bitrepository/common/utils/SettingsUtils.java @@ -118,6 +118,26 @@ public static String getPillarName(String pillarID) { return null; } + /** + * Get the configured max age for checksums for the given pillarID from the ReferenceSettings. + * + * @param pillarID The pillarID for which the hostname is wanted. + * @return human-readable maximum age for checksums for the given pillar ID. + */ + public static String getMaxAgeForChecksums(String pillarID) { + // TODO how to use pillarID?? + BigInteger maxAge = settings.getReferenceSettings().getPillarSettings().getMaxAgeForChecksums(); + if (maxAge == null) { + return null; + } + try { + return TimeUtils.millisecondsToHuman(maxAge.longValueExact()); + } + catch (ArithmeticException) { + return "Extremely long"; + } + } + /** * Get the {@link PillarType} for the given Pillar ID. * diff --git a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/PillarCollectionMetric.java b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/PillarCollectionMetric.java index 5157b58b2..43d3f2dd1 100644 --- a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/PillarCollectionMetric.java +++ b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/PillarCollectionMetric.java @@ -1,5 +1,7 @@ package org.bitrepository.integrityservice.cache; +import java.time.Instant; + /** * Class to carry information of collection specific pillar metrics. * The class exists as java is not able to handle simple tuples, @@ -19,9 +21,13 @@ public class PillarCollectionMetric { */ private final long pillarFileCount; - public PillarCollectionMetric(Long pillarCollectionSize, Long pillarFileCount) { + /** Timestamp of the oldest checksum on the pillar or null if no checksums yet */ + private final Instant oldestChecksumTimestamp; + + public PillarCollectionMetric(Long pillarCollectionSize, Long pillarFileCount, Instant oldestChecksumTimestamp) { this.pillarCollectionSize = pillarCollectionSize == null ? 0 : pillarCollectionSize; this.pillarFileCount = pillarFileCount == null ? 0 : pillarFileCount; + this.oldestChecksumTimestamp = oldestChecksumTimestamp; } public long getPillarCollectionSize() { @@ -32,4 +38,7 @@ public long getPillarFileCount() { return pillarFileCount; } + public Instant getOldestChecksumTimestamp() { + return oldestChecksumTimestamp; + } } diff --git a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/PillarCollectionStat.java b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/PillarCollectionStat.java index 611ce2361..f7e674bbd 100644 --- a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/PillarCollectionStat.java +++ b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/PillarCollectionStat.java @@ -37,6 +37,9 @@ public class PillarCollectionStat { private Long obsoleteChecksums = 0L; private Long missingChecksums = 0L; private Long checksumErrors = 0L; + private String maxAgeForChecksums; + /** Human-readable age of the oldest checksum, for example " 3m 46s 299 ms" */ + private String ageOfOldestChecksum; private Date statsTime; private Date updateTime; @@ -47,9 +50,10 @@ public PillarCollectionStat(String pillarID, String collectionID, String pillarN this.pillarType = pillarType; } - public PillarCollectionStat(String pillarID, String collectionID, String pillarName, String pillarType, Long fileCount, - Long dataSize, Long missingFiles, Long checksumErrors, Long missingChecksums, Long obsoleteChecksum, - Date statsTime, Date updateTime) { + public PillarCollectionStat(String pillarID, String collectionID, String pillarName, String pillarType, + Long fileCount, Long dataSize, + Long missingFiles, Long checksumErrors, Long missingChecksums, Long obsoleteChecksum, + String maxAgeForChecksums, String ageOfOldestChecksum, Date statsTime, Date updateTime) { this.pillarID = pillarID; this.collectionID = collectionID; this.pillarName = pillarName; @@ -62,6 +66,8 @@ public PillarCollectionStat(String pillarID, String collectionID, String pillarN this.obsoleteChecksums = obsoleteChecksum; this.statsTime = statsTime; this.updateTime = updateTime; + this.maxAgeForChecksums = maxAgeForChecksums; + this.ageOfOldestChecksum = ageOfOldestChecksum; } public String getPillarID() { @@ -144,4 +150,11 @@ public void setMissingChecksums(Long missingChecksums) { this.missingChecksums = missingChecksums; } + public String getAgeOfOldestChecksum() { + return ageOfOldestChecksum; + } + + public String getMaxAgeForChecksums() { + return maxAgeForChecksums; + } } diff --git a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/database/IntegrityDAO.java b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/database/IntegrityDAO.java index 72b40f378..24d7d51d3 100644 --- a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/database/IntegrityDAO.java +++ b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/database/IntegrityDAO.java @@ -26,6 +26,7 @@ import org.bitrepository.common.ArgumentValidator; import org.bitrepository.common.utils.CalendarUtils; import org.bitrepository.common.utils.SettingsUtils; +import org.bitrepository.common.utils.TimeUtils; import org.bitrepository.integrityservice.cache.CollectionStat; import org.bitrepository.integrityservice.cache.FileInfo; import org.bitrepository.integrityservice.cache.PillarCollectionMetric; @@ -40,14 +41,8 @@ import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Date; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Optional; +import java.time.Instant; +import java.util.*; /** * Common parts of the implementation of the access to the integrity db. @@ -439,7 +434,10 @@ public void createStatistics(String collectionID, StatisticsCollector statistics public Map getPillarCollectionMetrics(String collectionID) { Map metrics = new HashMap<>(); String selectSql = - "SELECT pillarID, COUNT(fileID) as filecount, SUM(filesize) as sizesum FROM fileinfo" + " WHERE collectionID = ?" + + "SELECT pillarID, COUNT(fileID) as filecount, SUM(filesize) as sizesum," + + " MIN(checksum_timestamp) as oldest_checksum_timestamp" + + " FROM fileinfo" + + " WHERE collectionID = ?" + " GROUP BY pillarID"; try (Connection conn = dbConnector.getConnection(); @@ -447,9 +445,13 @@ public Map getPillarCollectionMetrics(String col ResultSet dbResult = ps.executeQuery()) { while (dbResult.next()) { String pillarID = dbResult.getString("pillarID"); - Long fileCount = dbResult.getLong("filecount"); - Long fileSize = dbResult.getLong("sizesum"); - PillarCollectionMetric metric = new PillarCollectionMetric(fileSize, fileCount); + long fileCount = dbResult.getLong("filecount"); + // In case SUM(filesize) returned null, dbResult.getLong() will return 0, which is the sum we want + long fileSize = dbResult.getLong("sizesum"); + long oldestChecksumTimestampMillis = dbResult.getLong("oldest_checksum_timestamp"); + Instant oldestChecksumTimestamp = + dbResult.wasNull() ? null : Instant.ofEpochMilli(oldestChecksumTimestampMillis); + PillarCollectionMetric metric = new PillarCollectionMetric(fileSize, fileCount, oldestChecksumTimestamp); metrics.put(pillarID, metric); } } catch (SQLException e) { @@ -501,8 +503,10 @@ public List getLatestPillarStats(String collectionID) { List stats = new ArrayList<>(); String latestPillarStatsSql = "SELECT pillarID, file_count, file_size, missing_files_count," + - " checksum_errors_count, missing_checksums_count, obsolete_checksums_count" + " FROM pillarstats" + " WHERE stat_key = (" + - " SELECT MAX(stat_key) FROM stats" + " WHERE collectionID = ?)"; + " checksum_errors_count, missing_checksums_count, obsolete_checksums_count," + + " 1000000000000 as oldest_checksum_timestamp" + // TODO extend table with new col + " FROM pillarstats" + + " WHERE stat_key = (" + " SELECT MAX(stat_key) FROM stats" + " WHERE collectionID = ?)"; try (Connection conn = dbConnector.getConnection(); PreparedStatement ps = DatabaseUtils.createPreparedStatement(conn, latestPillarStatsSql, collectionID)) { @@ -520,8 +524,18 @@ public List getLatestPillarStats(String collectionID) { String pillarName = Objects.requireNonNullElse(SettingsUtils.getPillarName(pillarID), "N/A"); String pillarType = (SettingsUtils.getPillarType(pillarID) != null) ? Objects.requireNonNull(SettingsUtils.getPillarType(pillarID)).value() : "Unknown"; - PillarCollectionStat p = new PillarCollectionStat(pillarID, collectionID, pillarName, pillarType, fileCount, - dataSize, missingFiles, checksumErrors, missingChecksums, obsoleteChecksums, statsTime, updateTime); + long oldestChecksumTimestamp = dbResult.getLong("oldest_checksum_timestamp"); + String ageOfOldestChecksum; + if (dbResult.wasNull()) { + ageOfOldestChecksum = "N/A"; + } else { + ageOfOldestChecksum = TimeUtils.millisecondsToHuman( + System.currentTimeMillis() - oldestChecksumTimestamp); + } + PillarCollectionStat p = new PillarCollectionStat(pillarID, collectionID, + pillarName, pillarType, fileCount, dataSize, + missingFiles, checksumErrors, missingChecksums, obsoleteChecksums, + "TODO", ageOfOldestChecksum, statsTime, updateTime); // TODO Ole V. conflict not correctly resolved stats.add(p); } } diff --git a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/web/RestIntegrityService.java b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/web/RestIntegrityService.java index 4d06a3022..32a195da5 100644 --- a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/web/RestIntegrityService.java +++ b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/web/RestIntegrityService.java @@ -250,7 +250,7 @@ public String getIntegrityStatus( PillarType pillarTypeObject = SettingsUtils.getPillarType(pillar); String pillarType = pillarTypeObject != null ? pillarTypeObject.value() : null; PillarCollectionStat emptyStat = new PillarCollectionStat(pillar, collectionID, pillarName, pillarType, - 0L, 0L, 0L, 0L, 0L, 0L, new Date(0), new Date(0)); + 0L, 0L, 0L, 0L, 0L, 0L, "", "", new Date(0), new Date(0)); stats.put(pillar, emptyStat); } } @@ -422,6 +422,8 @@ private void writeIntegrityStatusObject(PillarCollectionStat stat, JsonGenerator jg.writeObjectField("checksumErrorCount", stat.getChecksumErrors()); jg.writeObjectField("obsoleteChecksumsCount", stat.getObsoleteChecksums()); jg.writeObjectField("missingChecksumsCount", stat.getMissingChecksums()); + jg.writeObjectField("maxAgeForChecksums", stat.getMaxAgeForChecksums()); + jg.writeObjectField("ageOfOldestChecksum", stat.getAgeOfOldestChecksum()); jg.writeEndObject(); } diff --git a/bitrepository-webclient/src/main/webapp/integrity-service.html b/bitrepository-webclient/src/main/webapp/integrity-service.html index 0401e7be1..6b121ae2b 100644 --- a/bitrepository-webclient/src/main/webapp/integrity-service.html +++ b/bitrepository-webclient/src/main/webapp/integrity-service.html @@ -85,6 +85,8 @@

Integrity service

Number of missing checksums Number of obsolete checksums Number of inconsistent checksums + Configured max age of checksums + Age of oldest checksum @@ -267,6 +269,14 @@

Modal header

context.title = type + " on " + id; context.member = "checksumErrorCount"; context.url += "getChecksumErrorFileIDs/"; + } else if(type == "Checksum age limit") { + context.element = id + "-maxAgeForChecksums"; + context.title = type + " on " + id; + context.member = "maxAgeForChecksums"; + } else if(type == "Oldest checksum age") { + context.element = id + "-ageOfOldestChecksum"; + context.title = type + " on " + id; + context.member = "ageOfOldestChecksum"; } context.url += "?pillarID=" + id + "&collectionID=" + getCollectionID(); return context; @@ -282,8 +292,11 @@

Modal header

html += ""; html += ""; html += ""; - html += ""; - return html; + html += ""; + html += ""; + html += ""; + html += ""; + return html; } function updateCells(pillarID) { @@ -294,6 +307,8 @@

Modal header

updateIntCell(pillarID, "Missing checksums", pillars[pillarID].missingChecksumsCount); updateIntCell(pillarID, "Obsolete checksums", pillars[pillarID].obsoleteChecksumsCount); updateIntCell(pillarID, "Inconsistent checksums", pillars[pillarID].checksumErrorCount); + updateStringCell(pillarID, "Checksum age limit", pillars[pillarID].maxAgeForChecksums); + updateStringCell(pillarID, "Oldest checksum age", pillars[pillarID].ageOfOldestChecksum); } function updateStringCell(id, type, cellValue) { @@ -331,7 +346,9 @@

Modal header

missingFilesCount : j[i].missingFilesCount, missingChecksumsCount : j[i].missingChecksumsCount, obsoleteChecksumsCount : j[i].obsoleteChecksumsCount, - checksumErrorCount : j[i].checksumErrorCount + checksumErrorCount : j[i].checksumErrorCount, + maxAgeForChecksums : j[i].maxAgeForChecksums, + ageOfOldestChecksum : j[i].ageOfOldestChecksum }; updateCells(j[i].pillarID); } From 9fcec79744b1dcf917a1669c8f1bc7959a92eba3 Mon Sep 17 00:00:00 2001 From: "Ole V. Villumsen" Date: Wed, 27 Jul 2022 12:16:38 +0200 Subject: [PATCH 09/47] Fix syntax error --- .../main/java/org/bitrepository/common/utils/SettingsUtils.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bitrepository-core/src/main/java/org/bitrepository/common/utils/SettingsUtils.java b/bitrepository-core/src/main/java/org/bitrepository/common/utils/SettingsUtils.java index 6fc9fbd66..5459104cb 100644 --- a/bitrepository-core/src/main/java/org/bitrepository/common/utils/SettingsUtils.java +++ b/bitrepository-core/src/main/java/org/bitrepository/common/utils/SettingsUtils.java @@ -133,7 +133,7 @@ public static String getMaxAgeForChecksums(String pillarID) { try { return TimeUtils.millisecondsToHuman(maxAge.longValueExact()); } - catch (ArithmeticException) { + catch (ArithmeticException ae) { return "Extremely long"; } } From dbbf3676a55be2d1a8c7420c9c5bb12ddc13be8b Mon Sep 17 00:00:00 2001 From: "Ole V. Villumsen" Date: Thu, 18 Aug 2022 11:04:42 +0200 Subject: [PATCH 10/47] BITMAG-1142 Show human-readable age of oldest checksum --- .../common/utils/SettingsUtils.java | 16 +++-- .../bitrepository/common/utils/TimeUtils.java | 59 ++++++++++++++++ .../common/utils/TimeUtilsTest.java | 67 ++++++++++++++++++- .../cache/IntegrityDatabaseMigrator.java | 16 ++++- .../cache/database/IntegrityDAO.java | 21 ++++-- .../sql/derby/integrityDB7to8migration.sql | 31 +++++++++ 6 files changed, 196 insertions(+), 14 deletions(-) create mode 100644 bitrepository-integrity-service/src/main/resources/sql/derby/integrityDB7to8migration.sql diff --git a/bitrepository-core/src/main/java/org/bitrepository/common/utils/SettingsUtils.java b/bitrepository-core/src/main/java/org/bitrepository/common/utils/SettingsUtils.java index 5459104cb..88c4b9335 100644 --- a/bitrepository-core/src/main/java/org/bitrepository/common/utils/SettingsUtils.java +++ b/bitrepository-core/src/main/java/org/bitrepository/common/utils/SettingsUtils.java @@ -23,6 +23,7 @@ import org.bitrepository.common.settings.Settings; import org.bitrepository.settings.referencesettings.PillarIntegrityDetails; +import org.bitrepository.settings.referencesettings.PillarSettings; import org.bitrepository.settings.referencesettings.PillarType; import org.bitrepository.settings.repositorysettings.Collection; @@ -121,20 +122,23 @@ public static String getPillarName(String pillarID) { /** * Get the configured max age for checksums for the given pillarID from the ReferenceSettings. * - * @param pillarID The pillarID for which the hostname is wanted. + * @param pillarID The pillarID for which the max checksum age is wanted. * @return human-readable maximum age for checksums for the given pillar ID. */ public static String getMaxAgeForChecksums(String pillarID) { - // TODO how to use pillarID?? - BigInteger maxAge = settings.getReferenceSettings().getPillarSettings().getMaxAgeForChecksums(); - if (maxAge == null) { - return null; + PillarSettings pillarSettings = settings.getReferenceSettings().getPillarSettings(); + if (pillarSettings == null) { + return "Not set (no pillar settings)"; } + if (! pillarSettings.getPillarID().equals(pillarID)) { + return "Unknown"; + } + BigInteger maxAge = pillarSettings.getMaxAgeForChecksums(); try { return TimeUtils.millisecondsToHuman(maxAge.longValueExact()); } catch (ArithmeticException ae) { - return "Extremely long"; + return String.format(Locale.getDefault(Locale.Category.FORMAT), "Extremely long; %d ms", maxAge); } } diff --git a/bitrepository-core/src/main/java/org/bitrepository/common/utils/TimeUtils.java b/bitrepository-core/src/main/java/org/bitrepository/common/utils/TimeUtils.java index feccec24f..0fdf794f4 100644 --- a/bitrepository-core/src/main/java/org/bitrepository/common/utils/TimeUtils.java +++ b/bitrepository-core/src/main/java/org/bitrepository/common/utils/TimeUtils.java @@ -21,10 +21,17 @@ */ package org.bitrepository.common.utils; +import org.bitrepository.common.ArgumentValidator; + import javax.xml.datatype.XMLGregorianCalendar; import java.text.DateFormat; import java.text.SimpleDateFormat; +import java.time.Duration; +import java.time.Period; +import java.time.ZonedDateTime; +import java.util.ArrayList; import java.util.Date; +import java.util.List; import java.util.Locale; /** @@ -123,6 +130,57 @@ public static String millisecondsToHuman(long ms) { return sb.toString(); } + public static String humanDifference(ZonedDateTime start, ZonedDateTime end) { + ArgumentValidator.checkTrue(! end.isBefore(start), start + " > " + end); + + Period periodBetween = Period.between(start.toLocalDate(), end.toLocalDate()); + ZonedDateTime afterPeriod = start.plus(periodBetween); + if (afterPeriod.isAfter(end)) { // Too far + // One day fewer + periodBetween = Period.between(start.toLocalDate(), end.toLocalDate().minusDays(1)); + afterPeriod = start.plus(periodBetween); + } + Duration durationBetween = Duration.between(afterPeriod, end); + + if (periodBetween.isZero() && durationBetween.isZero()) { + return "0 ms"; + } + + // The following gives an ambiguous string like "3m" + // in the very rare cases where only months or only minutes are non-zero. + // Since in practice the text is updated a few seconds later, it is not expected to be a problem for the user. + List elements = new ArrayList<>(7); + if (periodBetween.getYears() != 0) { + elements.add(periodBetween.getYears() + "y"); + } + if (periodBetween.getMonths() != 0) { + elements.add(periodBetween.getMonths() + "m"); + } + if (periodBetween.getDays() != 0) { + elements.add(periodBetween.getDays() + "d"); + } + if (durationBetween.toHours() != 0) { + elements.add(durationBetween.toHours() + "h"); + } + if (durationBetween.toMinutesPart() != 0) { + elements.add(durationBetween.toMinutesPart() + "m"); + } + if (durationBetween.toSecondsPart() != 0) { + elements.add(durationBetween.toSecondsPart() + "s"); + } + if (durationBetween.toNanosPart() != 0) { + int millis = durationBetween.toMillisPart(); + int nanos = durationBetween.toNanosPart(); + if (Duration.ofMillis(millis).toNanos() == nanos) { // millis give full precision; print them + elements.add(millis + " ms"); + } else { // print only nanos + elements.add(nanos + " ns"); + } + } + + return String.join(" ", elements); + } + public static String shortDate(Date date) { return formatter.format(date); } @@ -145,4 +203,5 @@ public static Date getMaxDate(Date currentMax, Date itemDate) { return currentMax; } } + } diff --git a/bitrepository-core/src/test/java/org/bitrepository/common/utils/TimeUtilsTest.java b/bitrepository-core/src/test/java/org/bitrepository/common/utils/TimeUtilsTest.java index 8bc2186f7..a9a437352 100644 --- a/bitrepository-core/src/test/java/org/bitrepository/common/utils/TimeUtilsTest.java +++ b/bitrepository-core/src/test/java/org/bitrepository/common/utils/TimeUtilsTest.java @@ -27,6 +27,13 @@ import java.text.DateFormat; import java.text.SimpleDateFormat; +import java.time.Duration; +import java.time.Instant; +import java.time.Period; +import java.time.ZoneId; +import java.time.ZoneOffset; +import java.time.ZonedDateTime; +import java.time.temporal.TemporalAmount; import java.util.Date; import java.util.Locale; @@ -34,6 +41,8 @@ import static org.testng.Assert.assertTrue; public class TimeUtilsTest extends ExtendedTestCase { + private static final ZonedDateTime BASE = Instant.EPOCH.atZone(ZoneOffset.UTC); + @Test(groups = {"regressiontest"}) public void timeTester() throws Exception { addDescription("Tests the TimeUtils. Pi days = 271433605 milliseconds"); @@ -73,11 +82,67 @@ public void timeTester() throws Exception { @Test(groups = {"regressiontest"}) public void zeroIntervalTest() throws Exception { addDescription("Verifies that a 0 ms interval is represented correctly"); - addStep("Call the millisecondsToHuman with 0 ms", "The output should be '0 ms'"); + addStep("Call millisecondsToHuman with 0 ms", "The output should be '0 ms'"); String zeroTimeString = TimeUtils.millisecondsToHuman(0); assertEquals(zeroTimeString, " 0 ms"); } + @Test(groups = {"regressiontest"}) + public void differencesPrintHumanly() { + addDescription("TimeUtils.humanDifference() should return" + + " similar human readable strings to those from millisecondsToHuman()"); + + addStep("Call humanDifference() with same time twice", "The output should be '0 ms'"); + String zeroTimeString = TimeUtils.humanDifference(BASE, BASE); + assertEquals(zeroTimeString, "0 ms"); + + addStep("Call humanDifference() with a difference obtained from a Duration", + "Expect corresponding readable output"); + testHumanDifference("1 ns", Duration.ofNanos(1)); + testHumanDifference("1 ms", Duration.ofMillis(1)); + // If there are nanos, don’t print millis + testHumanDifference("2000003 ns", Duration.ofNanos(2_000_003)); + testHumanDifference("1s", Duration.ofSeconds(1)); + testHumanDifference("1m", Duration.ofMinutes(1)); + testHumanDifference("1h", Duration.ofHours(1)); + testHumanDifference("2h 3m 5s 7 ns", Duration.parse("PT2H3M5.000000007S")); + + addStep("Call humanDifference() with a difference obtained from a Period", + "Expect corresponding readable output"); + testHumanDifference("1d", Period.ofDays(1)); + testHumanDifference("1m", Period.ofMonths(1)); + testHumanDifference("1y", Period.ofYears(1)); + testHumanDifference("2y 3m 5d", Period.of(2, 3, 5)); + + addStep("Call humanDifference() with a difference obtained from a combo of a Period and a Duration", + "Expect corresponding readable output"); + testHumanDifference("3y 5m 7d 11h 13m 17s 23 ms", + Period.of(3, 5, 7), Duration.parse("PT11H13M17.023S")); + + addStep("Call humanDifference()" + + " with dates that are 2 days apart but times that cause the diff to be less than 2 full days", + "Expect output 1d something"); + ZoneId testZoneId = ZoneId.of("Europe/Vienna"); + String oneDaySomethingString = TimeUtils.humanDifference( + ZonedDateTime.of(2021, 1, 31, + 12, 0, 0, 0, testZoneId), + ZonedDateTime.of(2021, 2, 2, + 11, 59, 59, 0, testZoneId)); + assertEquals("1d 23h 59m 59s", oneDaySomethingString); + } + + /** + * Note that the expected result comes first in the argument list + * so that we can use varargs to pass a number of amounts, for example both a Period and a Duration. + */ + private void testHumanDifference(String expected, TemporalAmount... amounts) { + ZonedDateTime end = BASE; + for (TemporalAmount amount: amounts) { + end = end.plus(amount); + } + String differenceString = TimeUtils.humanDifference(BASE, end); + assertEquals(differenceString, expected); + } /* * The test only ensures that the output format is fixed. Which timezone the the date is diff --git a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/IntegrityDatabaseMigrator.java b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/IntegrityDatabaseMigrator.java index f93b36edb..34f8b62a5 100644 --- a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/IntegrityDatabaseMigrator.java +++ b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/IntegrityDatabaseMigrator.java @@ -30,7 +30,7 @@ import java.util.Map; /** - * Migration class for the AuditTrailDatabase of the AuditTrailService. + * Migration class for the IntegrityDatabaseDatabase of the IntegrityDatabaseService. * Will only try to perform the migration on an embedded derby database. */ public class IntegrityDatabaseMigrator extends DatabaseMigrator { @@ -63,10 +63,13 @@ public class IntegrityDatabaseMigrator extends DatabaseMigrator { * The name of the update script for version 6 to 7. */ private static final String UPDATE_SCRIPT_VERSION_6_TO_7 = "sql/derby/integrityDB6to7migration.sql"; + + private static final String UPDATE_SCRIPT_VERSION_7_TO_8 = "sql/derby/integrityDB7to8migration.sql"; + /** * The current version of the database. */ - private final Integer currentVersion = 7; + private final int currentVersion = 8; /** * @param connector connection to the database. @@ -114,6 +117,15 @@ public void migrate() { log.warn("Migrating integrityDB from version 6 to 7"); migrateDerbyDatabase(UPDATE_SCRIPT_VERSION_6_TO_7); } + if (versions.get(DATABASE_VERSION_ENTRY) < 8) { + log.warn("Migrating integrityDB from version 7 to 8"); + migrateDerbyDatabase(UPDATE_SCRIPT_VERSION_7_TO_8); + } + + if (needsMigration()) { + log.error("Database still appears to need migration after it has been migrated. Expected version is {}.", + currentVersion); + } } @Override diff --git a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/database/IntegrityDAO.java b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/database/IntegrityDAO.java index 24d7d51d3..6effc2d7d 100644 --- a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/database/IntegrityDAO.java +++ b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/database/IntegrityDAO.java @@ -42,7 +42,16 @@ import java.sql.ResultSet; import java.sql.SQLException; import java.time.Instant; -import java.util.*; +import java.time.ZoneId; +import java.time.ZonedDateTime; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; /** * Common parts of the implementation of the access to the integrity db. @@ -504,7 +513,7 @@ public List getLatestPillarStats(String collectionID) { String latestPillarStatsSql = "SELECT pillarID, file_count, file_size, missing_files_count," + " checksum_errors_count, missing_checksums_count, obsolete_checksums_count," + - " 1000000000000 as oldest_checksum_timestamp" + // TODO extend table with new col + " 1000000000000 as oldest_checksum_timestamp" + // TODO Ole V. use new col " FROM pillarstats" + " WHERE stat_key = (" + " SELECT MAX(stat_key) FROM stats" + " WHERE collectionID = ?)"; @@ -524,18 +533,20 @@ public List getLatestPillarStats(String collectionID) { String pillarName = Objects.requireNonNullElse(SettingsUtils.getPillarName(pillarID), "N/A"); String pillarType = (SettingsUtils.getPillarType(pillarID) != null) ? Objects.requireNonNull(SettingsUtils.getPillarType(pillarID)).value() : "Unknown"; + String maxAgeForChecksums = SettingsUtils.getMaxAgeForChecksums(pillarID); long oldestChecksumTimestamp = dbResult.getLong("oldest_checksum_timestamp"); String ageOfOldestChecksum; if (dbResult.wasNull()) { ageOfOldestChecksum = "N/A"; } else { - ageOfOldestChecksum = TimeUtils.millisecondsToHuman( - System.currentTimeMillis() - oldestChecksumTimestamp); + ZoneId zone = ZoneId.systemDefault(); + ZonedDateTime oldestChecksumZdt = Instant.ofEpochMilli(oldestChecksumTimestamp).atZone(zone); + ageOfOldestChecksum = TimeUtils.humanDifference(oldestChecksumZdt, ZonedDateTime.now(zone)); } PillarCollectionStat p = new PillarCollectionStat(pillarID, collectionID, pillarName, pillarType, fileCount, dataSize, missingFiles, checksumErrors, missingChecksums, obsoleteChecksums, - "TODO", ageOfOldestChecksum, statsTime, updateTime); // TODO Ole V. conflict not correctly resolved + maxAgeForChecksums, ageOfOldestChecksum, statsTime, updateTime); stats.add(p); } } diff --git a/bitrepository-integrity-service/src/main/resources/sql/derby/integrityDB7to8migration.sql b/bitrepository-integrity-service/src/main/resources/sql/derby/integrityDB7to8migration.sql new file mode 100644 index 000000000..44fcde9bf --- /dev/null +++ b/bitrepository-integrity-service/src/main/resources/sql/derby/integrityDB7to8migration.sql @@ -0,0 +1,31 @@ +--- +-- #%L +-- Bitrepository Integrity Client +-- %% +-- Copyright (C) 2010 - 2022 Royal Danish Library and The State Archives, Denmark +-- %% +-- This program is free software: you can redistribute it and/or modify +-- it under the terms of the GNU Lesser General Public License as +-- published by the Free Software Foundation, either version 2.1 of the +-- License, or (at your option) any later version. +-- +-- This program is distributed in the hope that it will be useful, +-- but WITHOUT ANY WARRANTY; without even the implied warranty of +-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +-- GNU General Lesser Public License for more details. +-- +-- You should have received a copy of the GNU General Lesser Public +-- License along with this program. If not, see +-- . +-- #L% +--- + +connect 'jdbc:derby:integritydb'; + +-- database version +UPDATE tableversions SET version = 8 WHERE tablename = 'integritydb'; + +-- table version +UPDATE tableversions SET version = 3 WHERE tablename = 'pillarstats'; + +ALTER TABLE pillarstats ADD COLUMN oldest_checksum_timestamp BIGINT DEFAULT NULL; From 6c919753502551dd1b326484709d5534bf75ef64 Mon Sep 17 00:00:00 2001 From: "Ole V. Villumsen" Date: Thu, 18 Aug 2022 14:13:48 +0200 Subject: [PATCH 11/47] BITMAG-1142 Display configured max age from MaxChecksumAgeProvider --- .../common/utils/SettingsUtils.java | 50 +++++++++++-------- .../cache/database/IntegrityDAO.java | 45 ++++++++++++----- 2 files changed, 61 insertions(+), 34 deletions(-) diff --git a/bitrepository-core/src/main/java/org/bitrepository/common/utils/SettingsUtils.java b/bitrepository-core/src/main/java/org/bitrepository/common/utils/SettingsUtils.java index 88c4b9335..8a50b0d97 100644 --- a/bitrepository-core/src/main/java/org/bitrepository/common/utils/SettingsUtils.java +++ b/bitrepository-core/src/main/java/org/bitrepository/common/utils/SettingsUtils.java @@ -22,8 +22,8 @@ package org.bitrepository.common.utils; import org.bitrepository.common.settings.Settings; +import org.bitrepository.settings.referencesettings.IntegrityServiceSettings; import org.bitrepository.settings.referencesettings.PillarIntegrityDetails; -import org.bitrepository.settings.referencesettings.PillarSettings; import org.bitrepository.settings.referencesettings.PillarType; import org.bitrepository.settings.repositorysettings.Collection; @@ -119,27 +119,32 @@ public static String getPillarName(String pillarID) { return null; } - /** - * Get the configured max age for checksums for the given pillarID from the ReferenceSettings. - * - * @param pillarID The pillarID for which the max checksum age is wanted. - * @return human-readable maximum age for checksums for the given pillar ID. - */ - public static String getMaxAgeForChecksums(String pillarID) { - PillarSettings pillarSettings = settings.getReferenceSettings().getPillarSettings(); - if (pillarSettings == null) { - return "Not set (no pillar settings)"; - } - if (! pillarSettings.getPillarID().equals(pillarID)) { - return "Unknown"; - } - BigInteger maxAge = pillarSettings.getMaxAgeForChecksums(); - try { - return TimeUtils.millisecondsToHuman(maxAge.longValueExact()); - } - catch (ArithmeticException ae) { - return String.format(Locale.getDefault(Locale.Category.FORMAT), "Extremely long; %d ms", maxAge); - } + // TODO Ole V. delete method +// /** +// * Get the configured max age for checksums for the given pillarID from the ReferenceSettings. +// * +// * @param pillarID The pillarID for which the max checksum age is wanted. +// * @return human-readable maximum age for checksums for the given pillar ID. +// */ +// public static String getMaxAgeForChecksums(String pillarID) { +// PillarSettings pillarSettings = settings.getReferenceSettings().getPillarSettings(); +// if (pillarSettings == null) { +// return "Not set (no pillar settings)"; +// } +// if (! pillarSettings.getPillarID().equals(pillarID)) { +// return "Unknown"; +// } +// BigInteger maxAge = pillarSettings.getMaxAgeForChecksums(); +// try { +// return TimeUtils.millisecondsToHuman(maxAge.longValueExact()); +// } +// catch (ArithmeticException ae) { +// return String.format(Locale.getDefault(Locale.Category.FORMAT), "Extremely long; %d ms", maxAge); +// } +// } + + public static IntegrityServiceSettings getIntegrityServiceSettings() { + return settings.getReferenceSettings().getIntegrityServiceSettings(); } /** @@ -231,4 +236,5 @@ public static Set getStatusContributorsForCollection() { contributors.addAll(SettingsUtils.getAllPillarIDs()); return contributors; } + } diff --git a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/database/IntegrityDAO.java b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/database/IntegrityDAO.java index 6effc2d7d..f154466d3 100644 --- a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/database/IntegrityDAO.java +++ b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/database/IntegrityDAO.java @@ -31,9 +31,13 @@ import org.bitrepository.integrityservice.cache.FileInfo; import org.bitrepository.integrityservice.cache.PillarCollectionMetric; import org.bitrepository.integrityservice.cache.PillarCollectionStat; +import org.bitrepository.integrityservice.checking.MaxChecksumAgeProvider; import org.bitrepository.integrityservice.statistics.StatisticsCollector; +import org.bitrepository.integrityservice.workflow.step.HandleObsoleteChecksumsStep; import org.bitrepository.service.database.DBConnector; import org.bitrepository.service.database.DatabaseUtils; +import org.bitrepository.settings.referencesettings.ObsoleteChecksumSettings; +import org.jetbrains.annotations.NotNull; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -79,12 +83,12 @@ public void close() { } /** - * Method to ensure that pillars found in RepositorySettings is present in the database + * Method to ensure that pillars found in RepositorySettings are present in the database */ protected abstract void initializePillars(); /** - * Method to ensure that collections found in RepositorySettings is present in the database + * Method to ensure that collections found in RepositorySettings are present in the database */ protected abstract void initializeCollections(); @@ -533,16 +537,8 @@ public List getLatestPillarStats(String collectionID) { String pillarName = Objects.requireNonNullElse(SettingsUtils.getPillarName(pillarID), "N/A"); String pillarType = (SettingsUtils.getPillarType(pillarID) != null) ? Objects.requireNonNull(SettingsUtils.getPillarType(pillarID)).value() : "Unknown"; - String maxAgeForChecksums = SettingsUtils.getMaxAgeForChecksums(pillarID); - long oldestChecksumTimestamp = dbResult.getLong("oldest_checksum_timestamp"); - String ageOfOldestChecksum; - if (dbResult.wasNull()) { - ageOfOldestChecksum = "N/A"; - } else { - ZoneId zone = ZoneId.systemDefault(); - ZonedDateTime oldestChecksumZdt = Instant.ofEpochMilli(oldestChecksumTimestamp).atZone(zone); - ageOfOldestChecksum = TimeUtils.humanDifference(oldestChecksumZdt, ZonedDateTime.now(zone)); - } + String maxAgeForChecksums = getMaxAgeForChecksums(pillarID); + String ageOfOldestChecksum = getAgeOfOldestChecksum(dbResult); PillarCollectionStat p = new PillarCollectionStat(pillarID, collectionID, pillarName, pillarType, fileCount, dataSize, missingFiles, checksumErrors, missingChecksums, obsoleteChecksums, @@ -559,6 +555,31 @@ public List getLatestPillarStats(String collectionID) { return stats; } + @NotNull + private String getMaxAgeForChecksums(String pillarID) { + ObsoleteChecksumSettings obsoleteChecksumSettings = + SettingsUtils.getIntegrityServiceSettings().getObsoleteChecksumSettings(); + MaxChecksumAgeProvider maxChecksumAgeProvider = + new MaxChecksumAgeProvider(HandleObsoleteChecksumsStep.DEFAULT_MAX_CHECKSUM_AGE, + obsoleteChecksumSettings); + long maxAge = maxChecksumAgeProvider.getMaxChecksumAge(pillarID); + return maxAge == 0 ? "unlimited" : TimeUtils.millisecondsToHuman(maxAge); + } + + @NotNull + private String getAgeOfOldestChecksum(ResultSet dbResult) throws SQLException { + long oldestChecksumTimestamp = dbResult.getLong("oldest_checksum_timestamp"); + String ageOfOldestChecksum; + if (dbResult.wasNull()) { + ageOfOldestChecksum = "N/A"; + } else { + ZoneId zone = ZoneId.systemDefault(); + ZonedDateTime oldestChecksumZdt = Instant.ofEpochMilli(oldestChecksumTimestamp).atZone(zone); + ageOfOldestChecksum = TimeUtils.humanDifference(oldestChecksumZdt, ZonedDateTime.now(zone)); + } + return ageOfOldestChecksum; + } + /** * Method that should deliver the database specific SQL for getting the latest N collection statistics * From 7e0cc5d2cbbf2d00676cb9f388488ab8d6062403 Mon Sep 17 00:00:00 2001 From: "Ole V. Villumsen" Date: Fri, 19 Aug 2022 09:26:37 +0200 Subject: [PATCH 12/47] BITMAG-1142 Use real checksum timestamp from database --- .../integrityservice/cache/database/IntegrityDAO.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/database/IntegrityDAO.java b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/database/IntegrityDAO.java index f154466d3..85fddd7e6 100644 --- a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/database/IntegrityDAO.java +++ b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/database/IntegrityDAO.java @@ -517,9 +517,12 @@ public List getLatestPillarStats(String collectionID) { String latestPillarStatsSql = "SELECT pillarID, file_count, file_size, missing_files_count," + " checksum_errors_count, missing_checksums_count, obsolete_checksums_count," + - " 1000000000000 as oldest_checksum_timestamp" + // TODO Ole V. use new col + " oldest_checksum_timestamp" + " FROM pillarstats" + - " WHERE stat_key = (" + " SELECT MAX(stat_key) FROM stats" + " WHERE collectionID = ?)"; + " WHERE stat_key = (" + + " SELECT MAX(stat_key)" + + " FROM stats" + + " WHERE collectionID = ?)"; try (Connection conn = dbConnector.getConnection(); PreparedStatement ps = DatabaseUtils.createPreparedStatement(conn, latestPillarStatsSql, collectionID)) { From 325f7008c83f182315210f9cca2dc40ec505a264 Mon Sep 17 00:00:00 2001 From: "Ole V. Villumsen" Date: Wed, 24 Aug 2022 15:03:30 +0200 Subject: [PATCH 13/47] BITMAG-1142 Fix bugs and missing parts in display of age of oldest checksum --- .../bitrepository/common/utils/TimeUtils.java | 26 ++++++-------- .../common/utils/TimeUtilsTest.java | 17 ++++----- .../cache/PillarCollectionStat.java | 36 ++++++++++++++++--- .../cache/database/IntegrityDAO.java | 19 +++------- .../cache/database/StatisticsCreator.java | 33 +++++++++-------- .../web/RestIntegrityService.java | 7 ++-- .../step/CreateStatisticsEntryStep.java | 2 ++ 7 files changed, 79 insertions(+), 61 deletions(-) diff --git a/bitrepository-core/src/main/java/org/bitrepository/common/utils/TimeUtils.java b/bitrepository-core/src/main/java/org/bitrepository/common/utils/TimeUtils.java index 0fdf794f4..7e176e8bf 100644 --- a/bitrepository-core/src/main/java/org/bitrepository/common/utils/TimeUtils.java +++ b/bitrepository-core/src/main/java/org/bitrepository/common/utils/TimeUtils.java @@ -29,6 +29,7 @@ import java.time.Duration; import java.time.Period; import java.time.ZonedDateTime; +import java.time.temporal.ChronoUnit; import java.util.ArrayList; import java.util.Date; import java.util.List; @@ -140,16 +141,20 @@ public static String humanDifference(ZonedDateTime start, ZonedDateTime end) { periodBetween = Period.between(start.toLocalDate(), end.toLocalDate().minusDays(1)); afterPeriod = start.plus(periodBetween); } - Duration durationBetween = Duration.between(afterPeriod, end); + // Round duration to whole seconds + Duration durationBetween = Duration.between(afterPeriod, end) + .plusMillis(500) + .truncatedTo(ChronoUnit.SECONDS); if (periodBetween.isZero() && durationBetween.isZero()) { - return "0 ms"; + return "0s"; } - // The following gives an ambiguous string like "3m" - // in the very rare cases where only months or only minutes are non-zero. - // Since in practice the text is updated a few seconds later, it is not expected to be a problem for the user. - List elements = new ArrayList<>(7); + // The following gives an ambiguous string like "3m" or "3y 11m 5s" + // in the very rare cases where months *or* minutes are non-zero and days and hours are zero. + // Since in practice the text is updated a few seconds later and any minutes 1 minute later, + // it is not expected to be a problem for the user. + List elements = new ArrayList<>(6); if (periodBetween.getYears() != 0) { elements.add(periodBetween.getYears() + "y"); } @@ -168,15 +173,6 @@ public static String humanDifference(ZonedDateTime start, ZonedDateTime end) { if (durationBetween.toSecondsPart() != 0) { elements.add(durationBetween.toSecondsPart() + "s"); } - if (durationBetween.toNanosPart() != 0) { - int millis = durationBetween.toMillisPart(); - int nanos = durationBetween.toNanosPart(); - if (Duration.ofMillis(millis).toNanos() == nanos) { // millis give full precision; print them - elements.add(millis + " ms"); - } else { // print only nanos - elements.add(nanos + " ns"); - } - } return String.join(" ", elements); } diff --git a/bitrepository-core/src/test/java/org/bitrepository/common/utils/TimeUtilsTest.java b/bitrepository-core/src/test/java/org/bitrepository/common/utils/TimeUtilsTest.java index a9a437352..d0297d9de 100644 --- a/bitrepository-core/src/test/java/org/bitrepository/common/utils/TimeUtilsTest.java +++ b/bitrepository-core/src/test/java/org/bitrepository/common/utils/TimeUtilsTest.java @@ -92,20 +92,21 @@ public void differencesPrintHumanly() { addDescription("TimeUtils.humanDifference() should return" + " similar human readable strings to those from millisecondsToHuman()"); - addStep("Call humanDifference() with same time twice", "The output should be '0 ms'"); + addStep("Call humanDifference() with same time twice", "The output should be '0s'"); String zeroTimeString = TimeUtils.humanDifference(BASE, BASE); - assertEquals(zeroTimeString, "0 ms"); + assertEquals(zeroTimeString, "0s"); addStep("Call humanDifference() with a difference obtained from a Duration", "Expect corresponding readable output"); - testHumanDifference("1 ns", Duration.ofNanos(1)); - testHumanDifference("1 ms", Duration.ofMillis(1)); - // If there are nanos, don’t print millis - testHumanDifference("2000003 ns", Duration.ofNanos(2_000_003)); + // Don’t print fraction of second + testHumanDifference("0s", Duration.ofNanos(1)); + testHumanDifference("0s", Duration.ofMillis(1)); + testHumanDifference("0s", Duration.ofNanos(499_999_999)); + testHumanDifference("1s", Duration.ofNanos(500_000_000)); testHumanDifference("1s", Duration.ofSeconds(1)); testHumanDifference("1m", Duration.ofMinutes(1)); testHumanDifference("1h", Duration.ofHours(1)); - testHumanDifference("2h 3m 5s 7 ns", Duration.parse("PT2H3M5.000000007S")); + testHumanDifference("2h 3m 5s", Duration.parse("PT2H3M5.000000007S")); addStep("Call humanDifference() with a difference obtained from a Period", "Expect corresponding readable output"); @@ -116,7 +117,7 @@ public void differencesPrintHumanly() { addStep("Call humanDifference() with a difference obtained from a combo of a Period and a Duration", "Expect corresponding readable output"); - testHumanDifference("3y 5m 7d 11h 13m 17s 23 ms", + testHumanDifference("3y 5m 7d 11h 13m 17s", Period.of(3, 5, 7), Duration.parse("PT11H13M17.023S")); addStep("Call humanDifference()" + diff --git a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/PillarCollectionStat.java b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/PillarCollectionStat.java index f7e674bbd..0c006eeb2 100644 --- a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/PillarCollectionStat.java +++ b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/PillarCollectionStat.java @@ -21,6 +21,12 @@ */ package org.bitrepository.integrityservice.cache; +import org.bitrepository.common.utils.TimeUtils; +import org.jetbrains.annotations.NotNull; + +import java.time.Instant; +import java.time.ZoneId; +import java.time.ZonedDateTime; import java.util.Date; /** @@ -38,8 +44,7 @@ public class PillarCollectionStat { private Long missingChecksums = 0L; private Long checksumErrors = 0L; private String maxAgeForChecksums; - /** Human-readable age of the oldest checksum, for example " 3m 46s 299 ms" */ - private String ageOfOldestChecksum; + private Instant oldestChecksumTimestamp; private Date statsTime; private Date updateTime; @@ -53,7 +58,8 @@ public PillarCollectionStat(String pillarID, String collectionID, String pillarN public PillarCollectionStat(String pillarID, String collectionID, String pillarName, String pillarType, Long fileCount, Long dataSize, Long missingFiles, Long checksumErrors, Long missingChecksums, Long obsoleteChecksum, - String maxAgeForChecksums, String ageOfOldestChecksum, Date statsTime, Date updateTime) { + String maxAgeForChecksums, + Instant oldestChecksumTimestamp, Date statsTime, Date updateTime) { this.pillarID = pillarID; this.collectionID = collectionID; this.pillarName = pillarName; @@ -67,7 +73,7 @@ public PillarCollectionStat(String pillarID, String collectionID, String pillarN this.statsTime = statsTime; this.updateTime = updateTime; this.maxAgeForChecksums = maxAgeForChecksums; - this.ageOfOldestChecksum = ageOfOldestChecksum; + this.oldestChecksumTimestamp = oldestChecksumTimestamp; } public String getPillarID() { @@ -150,11 +156,31 @@ public void setMissingChecksums(Long missingChecksums) { this.missingChecksums = missingChecksums; } + /** @return Human-readable age of the oldest checksum, for example "3m 46s" */ + @NotNull public String getAgeOfOldestChecksum() { - return ageOfOldestChecksum; + if (oldestChecksumTimestamp == null) { + return "N/A"; + } + ZoneId zone = ZoneId.systemDefault(); + return TimeUtils.humanDifference(oldestChecksumTimestamp.atZone(zone), ZonedDateTime.now(zone)); + } + + public boolean hasOldestChecksumTimestamp() { + return oldestChecksumTimestamp != null; + } + + /** @throws NullPointerException if hasOldestChecksumTimestamp() does not return true */ + public long getOldestChecksumTimestampMillis() { + return oldestChecksumTimestamp.toEpochMilli(); + } + + public void setOldestChecksumTimestamp(Instant oldestChecksumTimestamp) { + this.oldestChecksumTimestamp = oldestChecksumTimestamp; } public String getMaxAgeForChecksums() { return maxAgeForChecksums; } + } diff --git a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/database/IntegrityDAO.java b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/database/IntegrityDAO.java index 85fddd7e6..c66acb116 100644 --- a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/database/IntegrityDAO.java +++ b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/database/IntegrityDAO.java @@ -46,8 +46,6 @@ import java.sql.ResultSet; import java.sql.SQLException; import java.time.Instant; -import java.time.ZoneId; -import java.time.ZonedDateTime; import java.util.ArrayList; import java.util.Arrays; import java.util.Date; @@ -541,11 +539,11 @@ public List getLatestPillarStats(String collectionID) { String pillarType = (SettingsUtils.getPillarType(pillarID) != null) ? Objects.requireNonNull(SettingsUtils.getPillarType(pillarID)).value() : "Unknown"; String maxAgeForChecksums = getMaxAgeForChecksums(pillarID); - String ageOfOldestChecksum = getAgeOfOldestChecksum(dbResult); + Instant oldestChecksumTimestamp = getOldestChecksumTimestamp(dbResult); PillarCollectionStat p = new PillarCollectionStat(pillarID, collectionID, pillarName, pillarType, fileCount, dataSize, missingFiles, checksumErrors, missingChecksums, obsoleteChecksums, - maxAgeForChecksums, ageOfOldestChecksum, statsTime, updateTime); + maxAgeForChecksums, oldestChecksumTimestamp, statsTime, updateTime); stats.add(p); } } @@ -569,18 +567,9 @@ private String getMaxAgeForChecksums(String pillarID) { return maxAge == 0 ? "unlimited" : TimeUtils.millisecondsToHuman(maxAge); } - @NotNull - private String getAgeOfOldestChecksum(ResultSet dbResult) throws SQLException { + private Instant getOldestChecksumTimestamp(ResultSet dbResult) throws SQLException { long oldestChecksumTimestamp = dbResult.getLong("oldest_checksum_timestamp"); - String ageOfOldestChecksum; - if (dbResult.wasNull()) { - ageOfOldestChecksum = "N/A"; - } else { - ZoneId zone = ZoneId.systemDefault(); - ZonedDateTime oldestChecksumZdt = Instant.ofEpochMilli(oldestChecksumTimestamp).atZone(zone); - ageOfOldestChecksum = TimeUtils.humanDifference(oldestChecksumZdt, ZonedDateTime.now(zone)); - } - return ageOfOldestChecksum; + return dbResult.wasNull() ? null : Instant.ofEpochMilli(oldestChecksumTimestamp); } /** diff --git a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/database/StatisticsCreator.java b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/database/StatisticsCreator.java index cefb53bd9..c0409ee6d 100644 --- a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/database/StatisticsCreator.java +++ b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/database/StatisticsCreator.java @@ -28,9 +28,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.sql.Connection; -import java.sql.PreparedStatement; -import java.sql.SQLException; +import java.sql.*; import java.util.Date; import java.util.List; @@ -49,8 +47,8 @@ public class StatisticsCreator { private final String insertPillarStatEntrySql = "INSERT INTO pillarstats" + " (stat_key, pillarID, file_count, file_size, missing_files_count, " - + "checksum_errors_count, missing_checksums_count, obsolete_checksums_count)" - + " (SELECT MAX(stat_key), ?, ?, ?, ?, ?, ?, ? FROM stats WHERE collectionID = ?)"; + + "checksum_errors_count, missing_checksums_count, obsolete_checksums_count, oldest_checksum_timestamp)" + + " (SELECT MAX(stat_key), ?, ?, ?, ?, ?, ?, ?, ? FROM stats WHERE collectionID = ?)"; private final Logger log = LoggerFactory.getLogger(getClass()); @@ -112,19 +110,24 @@ private void addCollectionStatistics(CollectionStat cs) throws SQLException { insertCollectionStatPS.setString(5, cs.getCollectionID()); } - private void addPillarStat(PillarCollectionStat ps) throws SQLException { - insertPillarStatPS.setString(1, ps.getPillarID()); - insertPillarStatPS.setLong(2, ps.getFileCount()); - insertPillarStatPS.setLong(3, ps.getDataSize()); - insertPillarStatPS.setLong(4, ps.getMissingFiles()); - insertPillarStatPS.setLong(5, ps.getChecksumErrors()); - insertPillarStatPS.setLong(6, ps.getMissingChecksums()); - insertPillarStatPS.setLong(7, ps.getObsoleteChecksums()); - insertPillarStatPS.setString(8, ps.getCollectionID()); + private void addPillarStat(PillarCollectionStat pcStat) throws SQLException { + insertPillarStatPS.setString(1, pcStat.getPillarID()); + insertPillarStatPS.setLong(2, pcStat.getFileCount()); + insertPillarStatPS.setLong(3, pcStat.getDataSize()); + insertPillarStatPS.setLong(4, pcStat.getMissingFiles()); + insertPillarStatPS.setLong(5, pcStat.getChecksumErrors()); + insertPillarStatPS.setLong(6, pcStat.getMissingChecksums()); + insertPillarStatPS.setLong(7, pcStat.getObsoleteChecksums()); + if (pcStat.hasOldestChecksumTimestamp()) { + insertPillarStatPS.setLong(8, pcStat.getOldestChecksumTimestampMillis()); + } else { + insertPillarStatPS.setNull(8, Types.BIGINT); + } + + insertPillarStatPS.setString(9, pcStat.getCollectionID()); insertPillarStatPS.addBatch(); } - private void execute() throws SQLException { insertStatisticsEntryPS.execute(); insertCollectionStatPS.execute(); diff --git a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/web/RestIntegrityService.java b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/web/RestIntegrityService.java index 32a195da5..356fbb932 100644 --- a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/web/RestIntegrityService.java +++ b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/web/RestIntegrityService.java @@ -249,8 +249,9 @@ public String getIntegrityStatus( String pillarName = Objects.requireNonNullElse(SettingsUtils.getPillarName(pillar), "N/A"); PillarType pillarTypeObject = SettingsUtils.getPillarType(pillar); String pillarType = pillarTypeObject != null ? pillarTypeObject.value() : null; - PillarCollectionStat emptyStat = new PillarCollectionStat(pillar, collectionID, pillarName, pillarType, - 0L, 0L, 0L, 0L, 0L, 0L, "", "", new Date(0), new Date(0)); + PillarCollectionStat emptyStat = new PillarCollectionStat(pillar, collectionID, pillarName, + pillarType, 0L, 0L, 0L, 0L, 0L, + 0L, "", null, new Date(0), new Date(0)); stats.put(pillar, emptyStat); } } @@ -266,7 +267,7 @@ public String getIntegrityStatus( } /*** - * Get the current workflows setup as a JSON array + * Get the current workflow’s setup as a JSON array */ @GET @Path("/getWorkflowSetup/") diff --git a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/workflow/step/CreateStatisticsEntryStep.java b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/workflow/step/CreateStatisticsEntryStep.java index 40427b1f3..fab49ba3a 100644 --- a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/workflow/step/CreateStatisticsEntryStep.java +++ b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/workflow/step/CreateStatisticsEntryStep.java @@ -63,9 +63,11 @@ public synchronized void performStep() { if (metric == null) { sc.getPillarCollectionStat(pillar).setFileCount(0L); sc.getPillarCollectionStat(pillar).setDataSize(0L); + sc.getPillarCollectionStat(pillar).setOldestChecksumTimestamp(null); } else { sc.getPillarCollectionStat(pillar).setFileCount(metric.getPillarFileCount()); sc.getPillarCollectionStat(pillar).setDataSize(metric.getPillarCollectionSize()); + sc.getPillarCollectionStat(pillar).setOldestChecksumTimestamp(metric.getOldestChecksumTimestamp()); } } sc.getCollectionStat().setFileCount(store.getNumberOfFilesInCollection(collectionID)); From 978d7022a7998faf41727a92d9637d539a87105c Mon Sep 17 00:00:00 2001 From: "Ole V. Villumsen" Date: Thu, 25 Aug 2022 13:01:50 +0200 Subject: [PATCH 14/47] BITMAG-1142 Delete method that is no longer used after change --- .../common/utils/SettingsUtils.java | 24 ------------------- 1 file changed, 24 deletions(-) diff --git a/bitrepository-core/src/main/java/org/bitrepository/common/utils/SettingsUtils.java b/bitrepository-core/src/main/java/org/bitrepository/common/utils/SettingsUtils.java index 8a50b0d97..c97eb183e 100644 --- a/bitrepository-core/src/main/java/org/bitrepository/common/utils/SettingsUtils.java +++ b/bitrepository-core/src/main/java/org/bitrepository/common/utils/SettingsUtils.java @@ -119,30 +119,6 @@ public static String getPillarName(String pillarID) { return null; } - // TODO Ole V. delete method -// /** -// * Get the configured max age for checksums for the given pillarID from the ReferenceSettings. -// * -// * @param pillarID The pillarID for which the max checksum age is wanted. -// * @return human-readable maximum age for checksums for the given pillar ID. -// */ -// public static String getMaxAgeForChecksums(String pillarID) { -// PillarSettings pillarSettings = settings.getReferenceSettings().getPillarSettings(); -// if (pillarSettings == null) { -// return "Not set (no pillar settings)"; -// } -// if (! pillarSettings.getPillarID().equals(pillarID)) { -// return "Unknown"; -// } -// BigInteger maxAge = pillarSettings.getMaxAgeForChecksums(); -// try { -// return TimeUtils.millisecondsToHuman(maxAge.longValueExact()); -// } -// catch (ArithmeticException ae) { -// return String.format(Locale.getDefault(Locale.Category.FORMAT), "Extremely long; %d ms", maxAge); -// } -// } - public static IntegrityServiceSettings getIntegrityServiceSettings() { return settings.getReferenceSettings().getIntegrityServiceSettings(); } From d1c4897319cbbcbf8a3448e209a9c2ce38ab9506 Mon Sep 17 00:00:00 2001 From: "Ole V. Villumsen" Date: Thu, 25 Aug 2022 13:24:09 +0200 Subject: [PATCH 15/47] Fix some of the errors from attempt to solve merge conflicts --- .../integrityservice/cache/PillarCollectionStat.java | 12 ++++++------ .../cache/database/IntegrityDAO.java | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/PillarCollectionStat.java b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/PillarCollectionStat.java index 0c006eeb2..1382346d1 100644 --- a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/PillarCollectionStat.java +++ b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/PillarCollectionStat.java @@ -35,7 +35,7 @@ public class PillarCollectionStat { private final String pillarID; private final String collectionID; - private final String pillarName; + private final String pillarHostname; private final String pillarType; private Long fileCount = 0L; private Long dataSize = 0L; @@ -48,10 +48,10 @@ public class PillarCollectionStat { private Date statsTime; private Date updateTime; - public PillarCollectionStat(String pillarID, String collectionID, String pillarName, String pillarType) { + public PillarCollectionStat(String pillarID, String collectionID, String pillarHostname, String pillarType) { this.pillarID = pillarID; this.collectionID = collectionID; - this.pillarName = pillarName; + this.pillarHostname = pillarHostname; this.pillarType = pillarType; } @@ -62,7 +62,7 @@ public PillarCollectionStat(String pillarID, String collectionID, String pillarN Instant oldestChecksumTimestamp, Date statsTime, Date updateTime) { this.pillarID = pillarID; this.collectionID = collectionID; - this.pillarName = pillarName; + this.pillarHostname = pillarHostname; this.pillarType = pillarType; this.fileCount = fileCount; this.dataSize = dataSize; @@ -84,8 +84,8 @@ public String getCollectionID() { return collectionID; } - public String getPillarName() { - return pillarName; + public String getPillarHostname() { + return pillarHostname; } public String getPillarType() { diff --git a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/database/IntegrityDAO.java b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/database/IntegrityDAO.java index c66acb116..1a02aaf34 100644 --- a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/database/IntegrityDAO.java +++ b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/database/IntegrityDAO.java @@ -535,7 +535,7 @@ public List getLatestPillarStats(String collectionID) { Long obsoleteChecksums = dbResult.getLong("obsolete_checksums_count"); Date statsTime = null; Date updateTime = null; - String pillarName = Objects.requireNonNullElse(SettingsUtils.getPillarName(pillarID), "N/A"); + String pillarHostname = Objects.requireNonNullElse(SettingsUtils.getPillarName(pillarID), "N/A"); String pillarType = (SettingsUtils.getPillarType(pillarID) != null) ? Objects.requireNonNull(SettingsUtils.getPillarType(pillarID)).value() : "Unknown"; String maxAgeForChecksums = getMaxAgeForChecksums(pillarID); From e83eb8c77847a7ab35e7fa4392af670d99aa0eb6 Mon Sep 17 00:00:00 2001 From: "Ole V. Villumsen" Date: Thu, 25 Aug 2022 14:06:39 +0200 Subject: [PATCH 16/47] Resolved further merge conflicts and compile errors from same --- .../integrityservice/cache/PillarCollectionStat.java | 12 ++++++------ .../cache/database/IntegrityDAO.java | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/PillarCollectionStat.java b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/PillarCollectionStat.java index 1382346d1..0c006eeb2 100644 --- a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/PillarCollectionStat.java +++ b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/PillarCollectionStat.java @@ -35,7 +35,7 @@ public class PillarCollectionStat { private final String pillarID; private final String collectionID; - private final String pillarHostname; + private final String pillarName; private final String pillarType; private Long fileCount = 0L; private Long dataSize = 0L; @@ -48,10 +48,10 @@ public class PillarCollectionStat { private Date statsTime; private Date updateTime; - public PillarCollectionStat(String pillarID, String collectionID, String pillarHostname, String pillarType) { + public PillarCollectionStat(String pillarID, String collectionID, String pillarName, String pillarType) { this.pillarID = pillarID; this.collectionID = collectionID; - this.pillarHostname = pillarHostname; + this.pillarName = pillarName; this.pillarType = pillarType; } @@ -62,7 +62,7 @@ public PillarCollectionStat(String pillarID, String collectionID, String pillarN Instant oldestChecksumTimestamp, Date statsTime, Date updateTime) { this.pillarID = pillarID; this.collectionID = collectionID; - this.pillarHostname = pillarHostname; + this.pillarName = pillarName; this.pillarType = pillarType; this.fileCount = fileCount; this.dataSize = dataSize; @@ -84,8 +84,8 @@ public String getCollectionID() { return collectionID; } - public String getPillarHostname() { - return pillarHostname; + public String getPillarName() { + return pillarName; } public String getPillarType() { diff --git a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/database/IntegrityDAO.java b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/database/IntegrityDAO.java index 1a02aaf34..c66acb116 100644 --- a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/database/IntegrityDAO.java +++ b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/database/IntegrityDAO.java @@ -535,7 +535,7 @@ public List getLatestPillarStats(String collectionID) { Long obsoleteChecksums = dbResult.getLong("obsolete_checksums_count"); Date statsTime = null; Date updateTime = null; - String pillarHostname = Objects.requireNonNullElse(SettingsUtils.getPillarName(pillarID), "N/A"); + String pillarName = Objects.requireNonNullElse(SettingsUtils.getPillarName(pillarID), "N/A"); String pillarType = (SettingsUtils.getPillarType(pillarID) != null) ? Objects.requireNonNull(SettingsUtils.getPillarType(pillarID)).value() : "Unknown"; String maxAgeForChecksums = getMaxAgeForChecksums(pillarID); From 19eb9cb5c20ae907f4a778d8fcea59e53b865169 Mon Sep 17 00:00:00 2001 From: "Ole V. Villumsen" Date: Thu, 25 Aug 2022 14:47:17 +0200 Subject: [PATCH 17/47] Cross your fingers that this is the last merge conflict to be resolved --- .../src/main/webapp/integrity-service.html | 4 ---- 1 file changed, 4 deletions(-) diff --git a/bitrepository-webclient/src/main/webapp/integrity-service.html b/bitrepository-webclient/src/main/webapp/integrity-service.html index e8efeaea7..90364a6eb 100644 --- a/bitrepository-webclient/src/main/webapp/integrity-service.html +++ b/bitrepository-webclient/src/main/webapp/integrity-service.html @@ -349,11 +349,7 @@

Modal header

checksumErrorCount : j[i].checksumErrorCount, maxAgeForChecksums : j[i].maxAgeForChecksums, ageOfOldestChecksum : j[i].ageOfOldestChecksum -<<<<<<< HEAD }; -======= - }; ->>>>>>> 2f286acfbec529ae49c96b80d928a0eb7c7db129 updateCells(j[i].pillarID); } }); From d5f02b62489ee0116ebb2b2a66e09d08f5f07695 Mon Sep 17 00:00:00 2001 From: "Ole V. Villumsen" Date: Thu, 18 Aug 2022 11:04:42 +0200 Subject: [PATCH 18/47] BITMAG-1142 Show human-readable age of oldest checksum --- .../common/utils/SettingsUtils.java | 1 + .../bitrepository/common/utils/TimeUtils.java | 16 ++++++++++++++++ .../common/utils/TimeUtilsTest.java | 1 + 3 files changed, 18 insertions(+) diff --git a/bitrepository-core/src/main/java/org/bitrepository/common/utils/SettingsUtils.java b/bitrepository-core/src/main/java/org/bitrepository/common/utils/SettingsUtils.java index c97eb183e..c8c367344 100644 --- a/bitrepository-core/src/main/java/org/bitrepository/common/utils/SettingsUtils.java +++ b/bitrepository-core/src/main/java/org/bitrepository/common/utils/SettingsUtils.java @@ -24,6 +24,7 @@ import org.bitrepository.common.settings.Settings; import org.bitrepository.settings.referencesettings.IntegrityServiceSettings; import org.bitrepository.settings.referencesettings.PillarIntegrityDetails; +import org.bitrepository.settings.referencesettings.PillarSettings; import org.bitrepository.settings.referencesettings.PillarType; import org.bitrepository.settings.repositorysettings.Collection; diff --git a/bitrepository-core/src/main/java/org/bitrepository/common/utils/TimeUtils.java b/bitrepository-core/src/main/java/org/bitrepository/common/utils/TimeUtils.java index 7e176e8bf..0204c64f2 100644 --- a/bitrepository-core/src/main/java/org/bitrepository/common/utils/TimeUtils.java +++ b/bitrepository-core/src/main/java/org/bitrepository/common/utils/TimeUtils.java @@ -29,7 +29,10 @@ import java.time.Duration; import java.time.Period; import java.time.ZonedDateTime; +<<<<<<< HEAD import java.time.temporal.ChronoUnit; +======= +>>>>>>> dbbf3676a (BITMAG-1142 Show human-readable age of oldest checksum) import java.util.ArrayList; import java.util.Date; import java.util.List; @@ -141,6 +144,7 @@ public static String humanDifference(ZonedDateTime start, ZonedDateTime end) { periodBetween = Period.between(start.toLocalDate(), end.toLocalDate().minusDays(1)); afterPeriod = start.plus(periodBetween); } +<<<<<<< HEAD // Round duration to whole seconds Duration durationBetween = Duration.between(afterPeriod, end) .plusMillis(500) @@ -155,6 +159,18 @@ public static String humanDifference(ZonedDateTime start, ZonedDateTime end) { // Since in practice the text is updated a few seconds later and any minutes 1 minute later, // it is not expected to be a problem for the user. List elements = new ArrayList<>(6); +======= + Duration durationBetween = Duration.between(afterPeriod, end); + + if (periodBetween.isZero() && durationBetween.isZero()) { + return "0 ms"; + } + + // The following gives an ambiguous string like "3m" + // in the very rare cases where only months or only minutes are non-zero. + // Since in practice the text is updated a few seconds later, it is not expected to be a problem for the user. + List elements = new ArrayList<>(7); +>>>>>>> dbbf3676a (BITMAG-1142 Show human-readable age of oldest checksum) if (periodBetween.getYears() != 0) { elements.add(periodBetween.getYears() + "y"); } diff --git a/bitrepository-core/src/test/java/org/bitrepository/common/utils/TimeUtilsTest.java b/bitrepository-core/src/test/java/org/bitrepository/common/utils/TimeUtilsTest.java index d0297d9de..1fc9d6d56 100644 --- a/bitrepository-core/src/test/java/org/bitrepository/common/utils/TimeUtilsTest.java +++ b/bitrepository-core/src/test/java/org/bitrepository/common/utils/TimeUtilsTest.java @@ -92,6 +92,7 @@ public void differencesPrintHumanly() { addDescription("TimeUtils.humanDifference() should return" + " similar human readable strings to those from millisecondsToHuman()"); +<<<<<<< HEAD addStep("Call humanDifference() with same time twice", "The output should be '0s'"); String zeroTimeString = TimeUtils.humanDifference(BASE, BASE); assertEquals(zeroTimeString, "0s"); From 91183359d868ad746de36dbc8366c67f9e45eb99 Mon Sep 17 00:00:00 2001 From: "Ole V. Villumsen" Date: Tue, 30 Aug 2022 14:46:23 +0200 Subject: [PATCH 19/47] BITMAG-1142 Complete rebase and conflict resolution --- .../common/utils/SettingsUtils.java | 28 ++++++++++++++++++- .../cache/database/IntegrityDAO.java | 19 +++++++++++++ 2 files changed, 46 insertions(+), 1 deletion(-) diff --git a/bitrepository-core/src/main/java/org/bitrepository/common/utils/SettingsUtils.java b/bitrepository-core/src/main/java/org/bitrepository/common/utils/SettingsUtils.java index c8c367344..bd97c1028 100644 --- a/bitrepository-core/src/main/java/org/bitrepository/common/utils/SettingsUtils.java +++ b/bitrepository-core/src/main/java/org/bitrepository/common/utils/SettingsUtils.java @@ -24,7 +24,6 @@ import org.bitrepository.common.settings.Settings; import org.bitrepository.settings.referencesettings.IntegrityServiceSettings; import org.bitrepository.settings.referencesettings.PillarIntegrityDetails; -import org.bitrepository.settings.referencesettings.PillarSettings; import org.bitrepository.settings.referencesettings.PillarType; import org.bitrepository.settings.repositorysettings.Collection; @@ -120,6 +119,33 @@ public static String getPillarName(String pillarID) { return null; } +<<<<<<< HEAD +======= + // TODO Ole V. delete method +// /** +// * Get the configured max age for checksums for the given pillarID from the ReferenceSettings. +// * +// * @param pillarID The pillarID for which the max checksum age is wanted. +// * @return human-readable maximum age for checksums for the given pillar ID. +// */ +// public static String getMaxAgeForChecksums(String pillarID) { +// PillarSettings pillarSettings = settings.getReferenceSettings().getPillarSettings(); +// if (pillarSettings == null) { +// return "Not set (no pillar settings)"; +// } +// if (! pillarSettings.getPillarID().equals(pillarID)) { +// return "Unknown"; +// } +// BigInteger maxAge = pillarSettings.getMaxAgeForChecksums(); +// try { +// return TimeUtils.millisecondsToHuman(maxAge.longValueExact()); +// } +// catch (ArithmeticException ae) { +// return String.format(Locale.getDefault(Locale.Category.FORMAT), "Extremely long; %d ms", maxAge); +// } +// } + +>>>>>>> 6c9197535 (BITMAG-1142 Display configured max age from MaxChecksumAgeProvider) public static IntegrityServiceSettings getIntegrityServiceSettings() { return settings.getReferenceSettings().getIntegrityServiceSettings(); } diff --git a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/database/IntegrityDAO.java b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/database/IntegrityDAO.java index c66acb116..399e9a21b 100644 --- a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/database/IntegrityDAO.java +++ b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/database/IntegrityDAO.java @@ -539,7 +539,11 @@ public List getLatestPillarStats(String collectionID) { String pillarType = (SettingsUtils.getPillarType(pillarID) != null) ? Objects.requireNonNull(SettingsUtils.getPillarType(pillarID)).value() : "Unknown"; String maxAgeForChecksums = getMaxAgeForChecksums(pillarID); +<<<<<<< HEAD Instant oldestChecksumTimestamp = getOldestChecksumTimestamp(dbResult); +======= + String ageOfOldestChecksum = getAgeOfOldestChecksum(dbResult); +>>>>>>> 6c9197535 (BITMAG-1142 Display configured max age from MaxChecksumAgeProvider) PillarCollectionStat p = new PillarCollectionStat(pillarID, collectionID, pillarName, pillarType, fileCount, dataSize, missingFiles, checksumErrors, missingChecksums, obsoleteChecksums, @@ -567,9 +571,24 @@ private String getMaxAgeForChecksums(String pillarID) { return maxAge == 0 ? "unlimited" : TimeUtils.millisecondsToHuman(maxAge); } +<<<<<<< HEAD private Instant getOldestChecksumTimestamp(ResultSet dbResult) throws SQLException { long oldestChecksumTimestamp = dbResult.getLong("oldest_checksum_timestamp"); return dbResult.wasNull() ? null : Instant.ofEpochMilli(oldestChecksumTimestamp); +======= + @NotNull + private String getAgeOfOldestChecksum(ResultSet dbResult) throws SQLException { + long oldestChecksumTimestamp = dbResult.getLong("oldest_checksum_timestamp"); + String ageOfOldestChecksum; + if (dbResult.wasNull()) { + ageOfOldestChecksum = "N/A"; + } else { + ZoneId zone = ZoneId.systemDefault(); + ZonedDateTime oldestChecksumZdt = Instant.ofEpochMilli(oldestChecksumTimestamp).atZone(zone); + ageOfOldestChecksum = TimeUtils.humanDifference(oldestChecksumZdt, ZonedDateTime.now(zone)); + } + return ageOfOldestChecksum; +>>>>>>> 6c9197535 (BITMAG-1142 Display configured max age from MaxChecksumAgeProvider) } /** From 133cd960113ce91c135ec8817bfd3a5b13db1d1a Mon Sep 17 00:00:00 2001 From: "Ole V. Villumsen" Date: Tue, 30 Aug 2022 15:02:53 +0200 Subject: [PATCH 20/47] BITMAG-1142 Solve more merge conflicts --- .../common/utils/SettingsUtils.java | 27 ------------------- .../bitrepository/common/utils/TimeUtils.java | 16 ----------- .../common/utils/TimeUtilsTest.java | 1 - .../cache/database/IntegrityDAO.java | 19 ------------- 4 files changed, 63 deletions(-) diff --git a/bitrepository-core/src/main/java/org/bitrepository/common/utils/SettingsUtils.java b/bitrepository-core/src/main/java/org/bitrepository/common/utils/SettingsUtils.java index bd97c1028..c97eb183e 100644 --- a/bitrepository-core/src/main/java/org/bitrepository/common/utils/SettingsUtils.java +++ b/bitrepository-core/src/main/java/org/bitrepository/common/utils/SettingsUtils.java @@ -119,33 +119,6 @@ public static String getPillarName(String pillarID) { return null; } -<<<<<<< HEAD -======= - // TODO Ole V. delete method -// /** -// * Get the configured max age for checksums for the given pillarID from the ReferenceSettings. -// * -// * @param pillarID The pillarID for which the max checksum age is wanted. -// * @return human-readable maximum age for checksums for the given pillar ID. -// */ -// public static String getMaxAgeForChecksums(String pillarID) { -// PillarSettings pillarSettings = settings.getReferenceSettings().getPillarSettings(); -// if (pillarSettings == null) { -// return "Not set (no pillar settings)"; -// } -// if (! pillarSettings.getPillarID().equals(pillarID)) { -// return "Unknown"; -// } -// BigInteger maxAge = pillarSettings.getMaxAgeForChecksums(); -// try { -// return TimeUtils.millisecondsToHuman(maxAge.longValueExact()); -// } -// catch (ArithmeticException ae) { -// return String.format(Locale.getDefault(Locale.Category.FORMAT), "Extremely long; %d ms", maxAge); -// } -// } - ->>>>>>> 6c9197535 (BITMAG-1142 Display configured max age from MaxChecksumAgeProvider) public static IntegrityServiceSettings getIntegrityServiceSettings() { return settings.getReferenceSettings().getIntegrityServiceSettings(); } diff --git a/bitrepository-core/src/main/java/org/bitrepository/common/utils/TimeUtils.java b/bitrepository-core/src/main/java/org/bitrepository/common/utils/TimeUtils.java index 0204c64f2..7e176e8bf 100644 --- a/bitrepository-core/src/main/java/org/bitrepository/common/utils/TimeUtils.java +++ b/bitrepository-core/src/main/java/org/bitrepository/common/utils/TimeUtils.java @@ -29,10 +29,7 @@ import java.time.Duration; import java.time.Period; import java.time.ZonedDateTime; -<<<<<<< HEAD import java.time.temporal.ChronoUnit; -======= ->>>>>>> dbbf3676a (BITMAG-1142 Show human-readable age of oldest checksum) import java.util.ArrayList; import java.util.Date; import java.util.List; @@ -144,7 +141,6 @@ public static String humanDifference(ZonedDateTime start, ZonedDateTime end) { periodBetween = Period.between(start.toLocalDate(), end.toLocalDate().minusDays(1)); afterPeriod = start.plus(periodBetween); } -<<<<<<< HEAD // Round duration to whole seconds Duration durationBetween = Duration.between(afterPeriod, end) .plusMillis(500) @@ -159,18 +155,6 @@ public static String humanDifference(ZonedDateTime start, ZonedDateTime end) { // Since in practice the text is updated a few seconds later and any minutes 1 minute later, // it is not expected to be a problem for the user. List elements = new ArrayList<>(6); -======= - Duration durationBetween = Duration.between(afterPeriod, end); - - if (periodBetween.isZero() && durationBetween.isZero()) { - return "0 ms"; - } - - // The following gives an ambiguous string like "3m" - // in the very rare cases where only months or only minutes are non-zero. - // Since in practice the text is updated a few seconds later, it is not expected to be a problem for the user. - List elements = new ArrayList<>(7); ->>>>>>> dbbf3676a (BITMAG-1142 Show human-readable age of oldest checksum) if (periodBetween.getYears() != 0) { elements.add(periodBetween.getYears() + "y"); } diff --git a/bitrepository-core/src/test/java/org/bitrepository/common/utils/TimeUtilsTest.java b/bitrepository-core/src/test/java/org/bitrepository/common/utils/TimeUtilsTest.java index 1fc9d6d56..d0297d9de 100644 --- a/bitrepository-core/src/test/java/org/bitrepository/common/utils/TimeUtilsTest.java +++ b/bitrepository-core/src/test/java/org/bitrepository/common/utils/TimeUtilsTest.java @@ -92,7 +92,6 @@ public void differencesPrintHumanly() { addDescription("TimeUtils.humanDifference() should return" + " similar human readable strings to those from millisecondsToHuman()"); -<<<<<<< HEAD addStep("Call humanDifference() with same time twice", "The output should be '0s'"); String zeroTimeString = TimeUtils.humanDifference(BASE, BASE); assertEquals(zeroTimeString, "0s"); diff --git a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/database/IntegrityDAO.java b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/database/IntegrityDAO.java index 399e9a21b..c66acb116 100644 --- a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/database/IntegrityDAO.java +++ b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/database/IntegrityDAO.java @@ -539,11 +539,7 @@ public List getLatestPillarStats(String collectionID) { String pillarType = (SettingsUtils.getPillarType(pillarID) != null) ? Objects.requireNonNull(SettingsUtils.getPillarType(pillarID)).value() : "Unknown"; String maxAgeForChecksums = getMaxAgeForChecksums(pillarID); -<<<<<<< HEAD Instant oldestChecksumTimestamp = getOldestChecksumTimestamp(dbResult); -======= - String ageOfOldestChecksum = getAgeOfOldestChecksum(dbResult); ->>>>>>> 6c9197535 (BITMAG-1142 Display configured max age from MaxChecksumAgeProvider) PillarCollectionStat p = new PillarCollectionStat(pillarID, collectionID, pillarName, pillarType, fileCount, dataSize, missingFiles, checksumErrors, missingChecksums, obsoleteChecksums, @@ -571,24 +567,9 @@ private String getMaxAgeForChecksums(String pillarID) { return maxAge == 0 ? "unlimited" : TimeUtils.millisecondsToHuman(maxAge); } -<<<<<<< HEAD private Instant getOldestChecksumTimestamp(ResultSet dbResult) throws SQLException { long oldestChecksumTimestamp = dbResult.getLong("oldest_checksum_timestamp"); return dbResult.wasNull() ? null : Instant.ofEpochMilli(oldestChecksumTimestamp); -======= - @NotNull - private String getAgeOfOldestChecksum(ResultSet dbResult) throws SQLException { - long oldestChecksumTimestamp = dbResult.getLong("oldest_checksum_timestamp"); - String ageOfOldestChecksum; - if (dbResult.wasNull()) { - ageOfOldestChecksum = "N/A"; - } else { - ZoneId zone = ZoneId.systemDefault(); - ZonedDateTime oldestChecksumZdt = Instant.ofEpochMilli(oldestChecksumTimestamp).atZone(zone); - ageOfOldestChecksum = TimeUtils.humanDifference(oldestChecksumZdt, ZonedDateTime.now(zone)); - } - return ageOfOldestChecksum; ->>>>>>> 6c9197535 (BITMAG-1142 Display configured max age from MaxChecksumAgeProvider) } /** From 7c9e362e3a00932e59c7a057924f11553649e94b Mon Sep 17 00:00:00 2001 From: "Ole V. Villumsen" Date: Tue, 30 Aug 2022 16:00:16 +0200 Subject: [PATCH 21/47] =?UTF-8?q?BITMAG-1142=20Don=E2=80=99t=20print=20ove?= =?UTF-8?q?rly=20precise=20checksum=20ages?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../bitrepository/common/utils/TimeUtils.java | 34 +++++++----- .../common/utils/TimeUtilsTest.java | 52 ++++++++++++++----- 2 files changed, 60 insertions(+), 26 deletions(-) diff --git a/bitrepository-core/src/main/java/org/bitrepository/common/utils/TimeUtils.java b/bitrepository-core/src/main/java/org/bitrepository/common/utils/TimeUtils.java index 7e176e8bf..ff5b28ab1 100644 --- a/bitrepository-core/src/main/java/org/bitrepository/common/utils/TimeUtils.java +++ b/bitrepository-core/src/main/java/org/bitrepository/common/utils/TimeUtils.java @@ -131,6 +131,13 @@ public static String millisecondsToHuman(long ms) { return sb.toString(); } + /** + * Generate a human-readable difference between start and end like "5y 2m 23d" or "7d 23m". + * + * Include years, months and days if they are non-zero. Include hours if months are 6 or less. + * Include minutes if days are 8 or less. Never include seconds. + * This generally gives the user a precision of 0.5 % of the difference or finer. + */ public static String humanDifference(ZonedDateTime start, ZonedDateTime end) { ArgumentValidator.checkTrue(! end.isBefore(start), start + " > " + end); @@ -141,19 +148,23 @@ public static String humanDifference(ZonedDateTime start, ZonedDateTime end) { periodBetween = Period.between(start.toLocalDate(), end.toLocalDate().minusDays(1)); afterPeriod = start.plus(periodBetween); } - // Round duration to whole seconds + // Round duration to whole minutes Duration durationBetween = Duration.between(afterPeriod, end) - .plusMillis(500) - .truncatedTo(ChronoUnit.SECONDS); + .plusSeconds(30) + .truncatedTo(ChronoUnit.MINUTES); if (periodBetween.isZero() && durationBetween.isZero()) { - return "0s"; + return "0m"; } - // The following gives an ambiguous string like "3m" or "3y 11m 5s" - // in the very rare cases where months *or* minutes are non-zero and days and hours are zero. - // Since in practice the text is updated a few seconds later and any minutes 1 minute later, - // it is not expected to be a problem for the user. + boolean includeHours = periodBetween.getYears() == 0 && periodBetween.getMonths() <= 6; + boolean includeMinutes = periodBetween.getYears() == 0 + && periodBetween.getMonths() == 0 + && periodBetween.getDays() <= 8; + + // The following gives an ambiguous string like "3m" + // in the very rare cases where months or minutes are non-zero and days and hours are zero. + // It is not expected to be a problem for the user in practice. List elements = new ArrayList<>(6); if (periodBetween.getYears() != 0) { elements.add(periodBetween.getYears() + "y"); @@ -164,15 +175,12 @@ public static String humanDifference(ZonedDateTime start, ZonedDateTime end) { if (periodBetween.getDays() != 0) { elements.add(periodBetween.getDays() + "d"); } - if (durationBetween.toHours() != 0) { + if (includeHours && durationBetween.toHours() != 0) { elements.add(durationBetween.toHours() + "h"); } - if (durationBetween.toMinutesPart() != 0) { + if (includeMinutes && durationBetween.toMinutesPart() != 0) { elements.add(durationBetween.toMinutesPart() + "m"); } - if (durationBetween.toSecondsPart() != 0) { - elements.add(durationBetween.toSecondsPart() + "s"); - } return String.join(" ", elements); } diff --git a/bitrepository-core/src/test/java/org/bitrepository/common/utils/TimeUtilsTest.java b/bitrepository-core/src/test/java/org/bitrepository/common/utils/TimeUtilsTest.java index d0297d9de..4fe2a4617 100644 --- a/bitrepository-core/src/test/java/org/bitrepository/common/utils/TimeUtilsTest.java +++ b/bitrepository-core/src/test/java/org/bitrepository/common/utils/TimeUtilsTest.java @@ -92,21 +92,17 @@ public void differencesPrintHumanly() { addDescription("TimeUtils.humanDifference() should return" + " similar human readable strings to those from millisecondsToHuman()"); - addStep("Call humanDifference() with same time twice", "The output should be '0s'"); + addStep("Call humanDifference() with same time twice", "The output should be '0m'"); String zeroTimeString = TimeUtils.humanDifference(BASE, BASE); - assertEquals(zeroTimeString, "0s"); + assertEquals(zeroTimeString, "0m"); addStep("Call humanDifference() with a difference obtained from a Duration", "Expect corresponding readable output"); - // Don’t print fraction of second - testHumanDifference("0s", Duration.ofNanos(1)); - testHumanDifference("0s", Duration.ofMillis(1)); - testHumanDifference("0s", Duration.ofNanos(499_999_999)); - testHumanDifference("1s", Duration.ofNanos(500_000_000)); - testHumanDifference("1s", Duration.ofSeconds(1)); + // Don’t print seconds + testHumanDifference("0m", Duration.ofSeconds(1)); testHumanDifference("1m", Duration.ofMinutes(1)); testHumanDifference("1h", Duration.ofHours(1)); - testHumanDifference("2h 3m 5s", Duration.parse("PT2H3M5.000000007S")); + testHumanDifference("2h 3m", Duration.parse("PT2H3M5.000000007S")); addStep("Call humanDifference() with a difference obtained from a Period", "Expect corresponding readable output"); @@ -117,8 +113,11 @@ public void differencesPrintHumanly() { addStep("Call humanDifference() with a difference obtained from a combo of a Period and a Duration", "Expect corresponding readable output"); - testHumanDifference("3y 5m 7d 11h 13m 17s", + testHumanDifference("3y 5m 7d", Period.of(3, 5, 7), Duration.parse("PT11H13M17.023S")); + testHumanDifference("2m 7d 11h", + Period.of(0, 2, 7), Duration.parse("PT11H13M17.023S")); + testHumanDifference("1d 11h 13m", Period.ofDays(1), Duration.parse("PT11H13M17.023S")); addStep("Call humanDifference()" + " with dates that are 2 days apart but times that cause the diff to be less than 2 full days", @@ -128,8 +127,35 @@ public void differencesPrintHumanly() { ZonedDateTime.of(2021, 1, 31, 12, 0, 0, 0, testZoneId), ZonedDateTime.of(2021, 2, 2, - 11, 59, 59, 0, testZoneId)); - assertEquals("1d 23h 59m 59s", oneDaySomethingString); + 11, 59, 29, 0, testZoneId)); + assertEquals(oneDaySomethingString, "1d 23h 59m"); + } + + @Test(groups = {"regressiontest"}) + public void differencesPrintsWithAppropriatePrecision() { + // Include hours if months are 6 or less. + testHumanDifference("11m", Period.ofMonths(11), Duration.ofHours(23)); + testHumanDifference("1y 1d", Period.of(1, 0, 1), Duration.ofHours(23)); + testHumanDifference("2m 1h", Period.ofMonths(2), Duration.ofHours(1)); + // Include minutes if days are 8 or less. + testHumanDifference("1y", Period.ofYears(1), Duration.ofMinutes(23)); + testHumanDifference("1m", Period.ofMonths(1), Duration.ofMinutes(23)); + testHumanDifference("27d", Period.ofDays(27), Duration.ofMinutes(23)); + testHumanDifference("2d 3m", Period.ofDays(2), Duration.ofMinutes(3)); + // Round to whole minutes + testHumanDifference("2d 3m", Period.ofDays(2), Duration.ofMinutes(2).plusSeconds(30)); + testHumanDifference("2d 3m", Period.ofDays(2), Duration.ofMinutes(3).plusSeconds(29)); + // Never include seconds. + testHumanDifference("1y", Period.ofYears(1), Duration.ofSeconds(55)); + testHumanDifference("1m", Period.ofMonths(1), Duration.ofSeconds(55)); + testHumanDifference("1d", Period.ofDays(1), Duration.ofSeconds(29)); + testHumanDifference("22h", Duration.ofHours(22).plusSeconds(29)); + testHumanDifference("4m", Duration.ofMinutes(4).plusSeconds(29)); + testHumanDifference("0m", Duration.ofSeconds(2).plusMillis(1)); + testHumanDifference("0m", Duration.ofNanos(500_000_000)); + testHumanDifference("0m", Duration.ofNanos(499_999_999)); + testHumanDifference("0m", Duration.ofMillis(1)); + testHumanDifference("0m", Duration.ofNanos(1)); } /** @@ -146,7 +172,7 @@ private void testHumanDifference(String expected, TemporalAmount... amounts) { } /* - * The test only ensures that the output format is fixed. Which timezone the the date is + * The test only ensures that the output format is fixed. Which timezone the date is * formatted to depends on the default/system timezone. At some time the use of the old java Date * api should be discontinued and the new Java Time api used instead. */ From c7e762af86deb2ee6d29ecbd0705ef184a99d16d Mon Sep 17 00:00:00 2001 From: "Ole V. Villumsen" Date: Tue, 26 Jul 2022 12:55:59 +0200 Subject: [PATCH 22/47] BITMAG-1142 Checksum age in GUI: underway --- .../common/utils/SettingsUtils.java | 20 ++++++++ .../cache/PillarCollectionMetric.java | 11 ++++- .../cache/PillarCollectionStat.java | 19 ++++++-- .../cache/database/IntegrityDAO.java | 46 ++++++++++++------- .../web/RestIntegrityService.java | 6 ++- .../src/main/webapp/integrity-service.html | 28 ++++++++++- 6 files changed, 106 insertions(+), 24 deletions(-) diff --git a/bitrepository-core/src/main/java/org/bitrepository/common/utils/SettingsUtils.java b/bitrepository-core/src/main/java/org/bitrepository/common/utils/SettingsUtils.java index fea26d3ab..6fc9fbd66 100644 --- a/bitrepository-core/src/main/java/org/bitrepository/common/utils/SettingsUtils.java +++ b/bitrepository-core/src/main/java/org/bitrepository/common/utils/SettingsUtils.java @@ -118,6 +118,26 @@ public static String getPillarName(String pillarID) { return null; } + /** + * Get the configured max age for checksums for the given pillarID from the ReferenceSettings. + * + * @param pillarID The pillarID for which the hostname is wanted. + * @return human-readable maximum age for checksums for the given pillar ID. + */ + public static String getMaxAgeForChecksums(String pillarID) { + // TODO how to use pillarID?? + BigInteger maxAge = settings.getReferenceSettings().getPillarSettings().getMaxAgeForChecksums(); + if (maxAge == null) { + return null; + } + try { + return TimeUtils.millisecondsToHuman(maxAge.longValueExact()); + } + catch (ArithmeticException) { + return "Extremely long"; + } + } + /** * Get the {@link PillarType} for the given Pillar ID. * diff --git a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/PillarCollectionMetric.java b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/PillarCollectionMetric.java index 5157b58b2..43d3f2dd1 100644 --- a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/PillarCollectionMetric.java +++ b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/PillarCollectionMetric.java @@ -1,5 +1,7 @@ package org.bitrepository.integrityservice.cache; +import java.time.Instant; + /** * Class to carry information of collection specific pillar metrics. * The class exists as java is not able to handle simple tuples, @@ -19,9 +21,13 @@ public class PillarCollectionMetric { */ private final long pillarFileCount; - public PillarCollectionMetric(Long pillarCollectionSize, Long pillarFileCount) { + /** Timestamp of the oldest checksum on the pillar or null if no checksums yet */ + private final Instant oldestChecksumTimestamp; + + public PillarCollectionMetric(Long pillarCollectionSize, Long pillarFileCount, Instant oldestChecksumTimestamp) { this.pillarCollectionSize = pillarCollectionSize == null ? 0 : pillarCollectionSize; this.pillarFileCount = pillarFileCount == null ? 0 : pillarFileCount; + this.oldestChecksumTimestamp = oldestChecksumTimestamp; } public long getPillarCollectionSize() { @@ -32,4 +38,7 @@ public long getPillarFileCount() { return pillarFileCount; } + public Instant getOldestChecksumTimestamp() { + return oldestChecksumTimestamp; + } } diff --git a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/PillarCollectionStat.java b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/PillarCollectionStat.java index 611ce2361..f7e674bbd 100644 --- a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/PillarCollectionStat.java +++ b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/PillarCollectionStat.java @@ -37,6 +37,9 @@ public class PillarCollectionStat { private Long obsoleteChecksums = 0L; private Long missingChecksums = 0L; private Long checksumErrors = 0L; + private String maxAgeForChecksums; + /** Human-readable age of the oldest checksum, for example " 3m 46s 299 ms" */ + private String ageOfOldestChecksum; private Date statsTime; private Date updateTime; @@ -47,9 +50,10 @@ public PillarCollectionStat(String pillarID, String collectionID, String pillarN this.pillarType = pillarType; } - public PillarCollectionStat(String pillarID, String collectionID, String pillarName, String pillarType, Long fileCount, - Long dataSize, Long missingFiles, Long checksumErrors, Long missingChecksums, Long obsoleteChecksum, - Date statsTime, Date updateTime) { + public PillarCollectionStat(String pillarID, String collectionID, String pillarName, String pillarType, + Long fileCount, Long dataSize, + Long missingFiles, Long checksumErrors, Long missingChecksums, Long obsoleteChecksum, + String maxAgeForChecksums, String ageOfOldestChecksum, Date statsTime, Date updateTime) { this.pillarID = pillarID; this.collectionID = collectionID; this.pillarName = pillarName; @@ -62,6 +66,8 @@ public PillarCollectionStat(String pillarID, String collectionID, String pillarN this.obsoleteChecksums = obsoleteChecksum; this.statsTime = statsTime; this.updateTime = updateTime; + this.maxAgeForChecksums = maxAgeForChecksums; + this.ageOfOldestChecksum = ageOfOldestChecksum; } public String getPillarID() { @@ -144,4 +150,11 @@ public void setMissingChecksums(Long missingChecksums) { this.missingChecksums = missingChecksums; } + public String getAgeOfOldestChecksum() { + return ageOfOldestChecksum; + } + + public String getMaxAgeForChecksums() { + return maxAgeForChecksums; + } } diff --git a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/database/IntegrityDAO.java b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/database/IntegrityDAO.java index 72b40f378..24d7d51d3 100644 --- a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/database/IntegrityDAO.java +++ b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/database/IntegrityDAO.java @@ -26,6 +26,7 @@ import org.bitrepository.common.ArgumentValidator; import org.bitrepository.common.utils.CalendarUtils; import org.bitrepository.common.utils.SettingsUtils; +import org.bitrepository.common.utils.TimeUtils; import org.bitrepository.integrityservice.cache.CollectionStat; import org.bitrepository.integrityservice.cache.FileInfo; import org.bitrepository.integrityservice.cache.PillarCollectionMetric; @@ -40,14 +41,8 @@ import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Date; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Optional; +import java.time.Instant; +import java.util.*; /** * Common parts of the implementation of the access to the integrity db. @@ -439,7 +434,10 @@ public void createStatistics(String collectionID, StatisticsCollector statistics public Map getPillarCollectionMetrics(String collectionID) { Map metrics = new HashMap<>(); String selectSql = - "SELECT pillarID, COUNT(fileID) as filecount, SUM(filesize) as sizesum FROM fileinfo" + " WHERE collectionID = ?" + + "SELECT pillarID, COUNT(fileID) as filecount, SUM(filesize) as sizesum," + + " MIN(checksum_timestamp) as oldest_checksum_timestamp" + + " FROM fileinfo" + + " WHERE collectionID = ?" + " GROUP BY pillarID"; try (Connection conn = dbConnector.getConnection(); @@ -447,9 +445,13 @@ public Map getPillarCollectionMetrics(String col ResultSet dbResult = ps.executeQuery()) { while (dbResult.next()) { String pillarID = dbResult.getString("pillarID"); - Long fileCount = dbResult.getLong("filecount"); - Long fileSize = dbResult.getLong("sizesum"); - PillarCollectionMetric metric = new PillarCollectionMetric(fileSize, fileCount); + long fileCount = dbResult.getLong("filecount"); + // In case SUM(filesize) returned null, dbResult.getLong() will return 0, which is the sum we want + long fileSize = dbResult.getLong("sizesum"); + long oldestChecksumTimestampMillis = dbResult.getLong("oldest_checksum_timestamp"); + Instant oldestChecksumTimestamp = + dbResult.wasNull() ? null : Instant.ofEpochMilli(oldestChecksumTimestampMillis); + PillarCollectionMetric metric = new PillarCollectionMetric(fileSize, fileCount, oldestChecksumTimestamp); metrics.put(pillarID, metric); } } catch (SQLException e) { @@ -501,8 +503,10 @@ public List getLatestPillarStats(String collectionID) { List stats = new ArrayList<>(); String latestPillarStatsSql = "SELECT pillarID, file_count, file_size, missing_files_count," + - " checksum_errors_count, missing_checksums_count, obsolete_checksums_count" + " FROM pillarstats" + " WHERE stat_key = (" + - " SELECT MAX(stat_key) FROM stats" + " WHERE collectionID = ?)"; + " checksum_errors_count, missing_checksums_count, obsolete_checksums_count," + + " 1000000000000 as oldest_checksum_timestamp" + // TODO extend table with new col + " FROM pillarstats" + + " WHERE stat_key = (" + " SELECT MAX(stat_key) FROM stats" + " WHERE collectionID = ?)"; try (Connection conn = dbConnector.getConnection(); PreparedStatement ps = DatabaseUtils.createPreparedStatement(conn, latestPillarStatsSql, collectionID)) { @@ -520,8 +524,18 @@ public List getLatestPillarStats(String collectionID) { String pillarName = Objects.requireNonNullElse(SettingsUtils.getPillarName(pillarID), "N/A"); String pillarType = (SettingsUtils.getPillarType(pillarID) != null) ? Objects.requireNonNull(SettingsUtils.getPillarType(pillarID)).value() : "Unknown"; - PillarCollectionStat p = new PillarCollectionStat(pillarID, collectionID, pillarName, pillarType, fileCount, - dataSize, missingFiles, checksumErrors, missingChecksums, obsoleteChecksums, statsTime, updateTime); + long oldestChecksumTimestamp = dbResult.getLong("oldest_checksum_timestamp"); + String ageOfOldestChecksum; + if (dbResult.wasNull()) { + ageOfOldestChecksum = "N/A"; + } else { + ageOfOldestChecksum = TimeUtils.millisecondsToHuman( + System.currentTimeMillis() - oldestChecksumTimestamp); + } + PillarCollectionStat p = new PillarCollectionStat(pillarID, collectionID, + pillarName, pillarType, fileCount, dataSize, + missingFiles, checksumErrors, missingChecksums, obsoleteChecksums, + "TODO", ageOfOldestChecksum, statsTime, updateTime); // TODO Ole V. conflict not correctly resolved stats.add(p); } } diff --git a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/web/RestIntegrityService.java b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/web/RestIntegrityService.java index 5eb7b31eb..e292858d6 100644 --- a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/web/RestIntegrityService.java +++ b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/web/RestIntegrityService.java @@ -264,8 +264,8 @@ public String getIntegrityStatus( String pillarName = Objects.requireNonNullElse(SettingsUtils.getPillarName(pillar), "N/A"); PillarType pillarTypeObject = SettingsUtils.getPillarType(pillar); String pillarType = pillarTypeObject != null ? pillarTypeObject.value() : null; - PillarCollectionStat emptyStat = new PillarCollectionStat(pillar, collectionID, pillarName, pillarType, 0L, 0L, 0L, 0L, 0L, - 0L, new Date(0), new Date(0)); + PillarCollectionStat emptyStat = new PillarCollectionStat(pillar, collectionID, pillarName, pillarType, + 0L, 0L, 0L, 0L, 0L, 0L, "", "", new Date(0), new Date(0)); stats.put(pillar, emptyStat); } } @@ -514,6 +514,8 @@ private void writeIntegrityStatusObject(PillarCollectionStat stat, JsonGenerator jg.writeObjectField("checksumErrorCount", stat.getChecksumErrors()); jg.writeObjectField("obsoleteChecksumsCount", stat.getObsoleteChecksums()); jg.writeObjectField("missingChecksumsCount", stat.getMissingChecksums()); + jg.writeObjectField("maxAgeForChecksums", stat.getMaxAgeForChecksums()); + jg.writeObjectField("ageOfOldestChecksum", stat.getAgeOfOldestChecksum()); jg.writeEndObject(); } diff --git a/bitrepository-webclient/src/main/webapp/integrity-service.html b/bitrepository-webclient/src/main/webapp/integrity-service.html index 1887a489f..21fa1f213 100644 --- a/bitrepository-webclient/src/main/webapp/integrity-service.html +++ b/bitrepository-webclient/src/main/webapp/integrity-service.html @@ -82,10 +82,19 @@

Integrity service

Pillar Name Pillar Type Total number of files +<<<<<<< HEAD Number of missing files Number of missing checksums Number of obsolete checksums Number of inconsistent checksums +======= + Number of missing files + Number of missing checksums + Number of obsolete checksums + Number of inconsistent checksums + Configured max age of checksums + Age of oldest checksum +>>>>>>> 10b99c1d8 (BITMAG-1142 Checksum age in GUI: underway) @@ -273,6 +282,16 @@

Modal header

context.title = type + " on " + id.toUpperCase(); context.propertyName = "checksumErrorCount"; context.url += "getChecksumErrorFileIDs"; + } else if(type == "Checksum age limit") { + context.element = id + "-maxAgeForChecksums"; + context.title = type + " on " + id.toUpperCase(); + context.propertyName = "maxAgeForChecksums"; + context.url += "getMaxAgeForChecksums"; + } else if(type == "Oldest checksum age") { + context.element = id + "-ageOfOldestChecksum"; + context.title = type + " on " + id.toUpperCase(); + context.propertyName = "ageOfOldestChecksum"; + context.url += "getAgeOfOldestChecksum"; } context.url += "?pillarID=" + id + "&collectionID=" + getCollectionID(); return context; @@ -306,8 +325,11 @@

Modal header

html += ""; html += ""; html += ""; - html += ""; - return html; + html += ""; + html += ""; + html += ""; + html += ""; + return html; } function updateTableHeader() { @@ -315,6 +337,8 @@

Modal header

setHeader("Number of missing checksums"); setHeader("Number of obsolete checksums"); setHeader("Number of inconsistent checksums"); + setHeader("Configured max age of checksums"); + setHeader("Age of oldest checksum"); } function setHeader(type) { From 690e4e806e94d438edb365b1d3975e391af74fe8 Mon Sep 17 00:00:00 2001 From: "Ole V. Villumsen" Date: Wed, 27 Jul 2022 12:16:38 +0200 Subject: [PATCH 23/47] Fix syntax error --- .../main/java/org/bitrepository/common/utils/SettingsUtils.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bitrepository-core/src/main/java/org/bitrepository/common/utils/SettingsUtils.java b/bitrepository-core/src/main/java/org/bitrepository/common/utils/SettingsUtils.java index 6fc9fbd66..5459104cb 100644 --- a/bitrepository-core/src/main/java/org/bitrepository/common/utils/SettingsUtils.java +++ b/bitrepository-core/src/main/java/org/bitrepository/common/utils/SettingsUtils.java @@ -133,7 +133,7 @@ public static String getMaxAgeForChecksums(String pillarID) { try { return TimeUtils.millisecondsToHuman(maxAge.longValueExact()); } - catch (ArithmeticException) { + catch (ArithmeticException ae) { return "Extremely long"; } } From 1de2b4b9511438f9802c71e233404782ce168d34 Mon Sep 17 00:00:00 2001 From: "Ole V. Villumsen" Date: Thu, 18 Aug 2022 11:04:42 +0200 Subject: [PATCH 24/47] BITMAG-1142 Show human-readable age of oldest checksum --- .../common/utils/SettingsUtils.java | 16 +++-- .../bitrepository/common/utils/TimeUtils.java | 59 ++++++++++++++++ .../common/utils/TimeUtilsTest.java | 67 ++++++++++++++++++- .../cache/IntegrityDatabaseMigrator.java | 16 ++++- .../cache/database/IntegrityDAO.java | 21 ++++-- .../sql/derby/integrityDB7to8migration.sql | 31 +++++++++ 6 files changed, 196 insertions(+), 14 deletions(-) create mode 100644 bitrepository-integrity-service/src/main/resources/sql/derby/integrityDB7to8migration.sql diff --git a/bitrepository-core/src/main/java/org/bitrepository/common/utils/SettingsUtils.java b/bitrepository-core/src/main/java/org/bitrepository/common/utils/SettingsUtils.java index 5459104cb..88c4b9335 100644 --- a/bitrepository-core/src/main/java/org/bitrepository/common/utils/SettingsUtils.java +++ b/bitrepository-core/src/main/java/org/bitrepository/common/utils/SettingsUtils.java @@ -23,6 +23,7 @@ import org.bitrepository.common.settings.Settings; import org.bitrepository.settings.referencesettings.PillarIntegrityDetails; +import org.bitrepository.settings.referencesettings.PillarSettings; import org.bitrepository.settings.referencesettings.PillarType; import org.bitrepository.settings.repositorysettings.Collection; @@ -121,20 +122,23 @@ public static String getPillarName(String pillarID) { /** * Get the configured max age for checksums for the given pillarID from the ReferenceSettings. * - * @param pillarID The pillarID for which the hostname is wanted. + * @param pillarID The pillarID for which the max checksum age is wanted. * @return human-readable maximum age for checksums for the given pillar ID. */ public static String getMaxAgeForChecksums(String pillarID) { - // TODO how to use pillarID?? - BigInteger maxAge = settings.getReferenceSettings().getPillarSettings().getMaxAgeForChecksums(); - if (maxAge == null) { - return null; + PillarSettings pillarSettings = settings.getReferenceSettings().getPillarSettings(); + if (pillarSettings == null) { + return "Not set (no pillar settings)"; } + if (! pillarSettings.getPillarID().equals(pillarID)) { + return "Unknown"; + } + BigInteger maxAge = pillarSettings.getMaxAgeForChecksums(); try { return TimeUtils.millisecondsToHuman(maxAge.longValueExact()); } catch (ArithmeticException ae) { - return "Extremely long"; + return String.format(Locale.getDefault(Locale.Category.FORMAT), "Extremely long; %d ms", maxAge); } } diff --git a/bitrepository-core/src/main/java/org/bitrepository/common/utils/TimeUtils.java b/bitrepository-core/src/main/java/org/bitrepository/common/utils/TimeUtils.java index feccec24f..0fdf794f4 100644 --- a/bitrepository-core/src/main/java/org/bitrepository/common/utils/TimeUtils.java +++ b/bitrepository-core/src/main/java/org/bitrepository/common/utils/TimeUtils.java @@ -21,10 +21,17 @@ */ package org.bitrepository.common.utils; +import org.bitrepository.common.ArgumentValidator; + import javax.xml.datatype.XMLGregorianCalendar; import java.text.DateFormat; import java.text.SimpleDateFormat; +import java.time.Duration; +import java.time.Period; +import java.time.ZonedDateTime; +import java.util.ArrayList; import java.util.Date; +import java.util.List; import java.util.Locale; /** @@ -123,6 +130,57 @@ public static String millisecondsToHuman(long ms) { return sb.toString(); } + public static String humanDifference(ZonedDateTime start, ZonedDateTime end) { + ArgumentValidator.checkTrue(! end.isBefore(start), start + " > " + end); + + Period periodBetween = Period.between(start.toLocalDate(), end.toLocalDate()); + ZonedDateTime afterPeriod = start.plus(periodBetween); + if (afterPeriod.isAfter(end)) { // Too far + // One day fewer + periodBetween = Period.between(start.toLocalDate(), end.toLocalDate().minusDays(1)); + afterPeriod = start.plus(periodBetween); + } + Duration durationBetween = Duration.between(afterPeriod, end); + + if (periodBetween.isZero() && durationBetween.isZero()) { + return "0 ms"; + } + + // The following gives an ambiguous string like "3m" + // in the very rare cases where only months or only minutes are non-zero. + // Since in practice the text is updated a few seconds later, it is not expected to be a problem for the user. + List elements = new ArrayList<>(7); + if (periodBetween.getYears() != 0) { + elements.add(periodBetween.getYears() + "y"); + } + if (periodBetween.getMonths() != 0) { + elements.add(periodBetween.getMonths() + "m"); + } + if (periodBetween.getDays() != 0) { + elements.add(periodBetween.getDays() + "d"); + } + if (durationBetween.toHours() != 0) { + elements.add(durationBetween.toHours() + "h"); + } + if (durationBetween.toMinutesPart() != 0) { + elements.add(durationBetween.toMinutesPart() + "m"); + } + if (durationBetween.toSecondsPart() != 0) { + elements.add(durationBetween.toSecondsPart() + "s"); + } + if (durationBetween.toNanosPart() != 0) { + int millis = durationBetween.toMillisPart(); + int nanos = durationBetween.toNanosPart(); + if (Duration.ofMillis(millis).toNanos() == nanos) { // millis give full precision; print them + elements.add(millis + " ms"); + } else { // print only nanos + elements.add(nanos + " ns"); + } + } + + return String.join(" ", elements); + } + public static String shortDate(Date date) { return formatter.format(date); } @@ -145,4 +203,5 @@ public static Date getMaxDate(Date currentMax, Date itemDate) { return currentMax; } } + } diff --git a/bitrepository-core/src/test/java/org/bitrepository/common/utils/TimeUtilsTest.java b/bitrepository-core/src/test/java/org/bitrepository/common/utils/TimeUtilsTest.java index 8bc2186f7..a9a437352 100644 --- a/bitrepository-core/src/test/java/org/bitrepository/common/utils/TimeUtilsTest.java +++ b/bitrepository-core/src/test/java/org/bitrepository/common/utils/TimeUtilsTest.java @@ -27,6 +27,13 @@ import java.text.DateFormat; import java.text.SimpleDateFormat; +import java.time.Duration; +import java.time.Instant; +import java.time.Period; +import java.time.ZoneId; +import java.time.ZoneOffset; +import java.time.ZonedDateTime; +import java.time.temporal.TemporalAmount; import java.util.Date; import java.util.Locale; @@ -34,6 +41,8 @@ import static org.testng.Assert.assertTrue; public class TimeUtilsTest extends ExtendedTestCase { + private static final ZonedDateTime BASE = Instant.EPOCH.atZone(ZoneOffset.UTC); + @Test(groups = {"regressiontest"}) public void timeTester() throws Exception { addDescription("Tests the TimeUtils. Pi days = 271433605 milliseconds"); @@ -73,11 +82,67 @@ public void timeTester() throws Exception { @Test(groups = {"regressiontest"}) public void zeroIntervalTest() throws Exception { addDescription("Verifies that a 0 ms interval is represented correctly"); - addStep("Call the millisecondsToHuman with 0 ms", "The output should be '0 ms'"); + addStep("Call millisecondsToHuman with 0 ms", "The output should be '0 ms'"); String zeroTimeString = TimeUtils.millisecondsToHuman(0); assertEquals(zeroTimeString, " 0 ms"); } + @Test(groups = {"regressiontest"}) + public void differencesPrintHumanly() { + addDescription("TimeUtils.humanDifference() should return" + + " similar human readable strings to those from millisecondsToHuman()"); + + addStep("Call humanDifference() with same time twice", "The output should be '0 ms'"); + String zeroTimeString = TimeUtils.humanDifference(BASE, BASE); + assertEquals(zeroTimeString, "0 ms"); + + addStep("Call humanDifference() with a difference obtained from a Duration", + "Expect corresponding readable output"); + testHumanDifference("1 ns", Duration.ofNanos(1)); + testHumanDifference("1 ms", Duration.ofMillis(1)); + // If there are nanos, don’t print millis + testHumanDifference("2000003 ns", Duration.ofNanos(2_000_003)); + testHumanDifference("1s", Duration.ofSeconds(1)); + testHumanDifference("1m", Duration.ofMinutes(1)); + testHumanDifference("1h", Duration.ofHours(1)); + testHumanDifference("2h 3m 5s 7 ns", Duration.parse("PT2H3M5.000000007S")); + + addStep("Call humanDifference() with a difference obtained from a Period", + "Expect corresponding readable output"); + testHumanDifference("1d", Period.ofDays(1)); + testHumanDifference("1m", Period.ofMonths(1)); + testHumanDifference("1y", Period.ofYears(1)); + testHumanDifference("2y 3m 5d", Period.of(2, 3, 5)); + + addStep("Call humanDifference() with a difference obtained from a combo of a Period and a Duration", + "Expect corresponding readable output"); + testHumanDifference("3y 5m 7d 11h 13m 17s 23 ms", + Period.of(3, 5, 7), Duration.parse("PT11H13M17.023S")); + + addStep("Call humanDifference()" + + " with dates that are 2 days apart but times that cause the diff to be less than 2 full days", + "Expect output 1d something"); + ZoneId testZoneId = ZoneId.of("Europe/Vienna"); + String oneDaySomethingString = TimeUtils.humanDifference( + ZonedDateTime.of(2021, 1, 31, + 12, 0, 0, 0, testZoneId), + ZonedDateTime.of(2021, 2, 2, + 11, 59, 59, 0, testZoneId)); + assertEquals("1d 23h 59m 59s", oneDaySomethingString); + } + + /** + * Note that the expected result comes first in the argument list + * so that we can use varargs to pass a number of amounts, for example both a Period and a Duration. + */ + private void testHumanDifference(String expected, TemporalAmount... amounts) { + ZonedDateTime end = BASE; + for (TemporalAmount amount: amounts) { + end = end.plus(amount); + } + String differenceString = TimeUtils.humanDifference(BASE, end); + assertEquals(differenceString, expected); + } /* * The test only ensures that the output format is fixed. Which timezone the the date is diff --git a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/IntegrityDatabaseMigrator.java b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/IntegrityDatabaseMigrator.java index f93b36edb..34f8b62a5 100644 --- a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/IntegrityDatabaseMigrator.java +++ b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/IntegrityDatabaseMigrator.java @@ -30,7 +30,7 @@ import java.util.Map; /** - * Migration class for the AuditTrailDatabase of the AuditTrailService. + * Migration class for the IntegrityDatabaseDatabase of the IntegrityDatabaseService. * Will only try to perform the migration on an embedded derby database. */ public class IntegrityDatabaseMigrator extends DatabaseMigrator { @@ -63,10 +63,13 @@ public class IntegrityDatabaseMigrator extends DatabaseMigrator { * The name of the update script for version 6 to 7. */ private static final String UPDATE_SCRIPT_VERSION_6_TO_7 = "sql/derby/integrityDB6to7migration.sql"; + + private static final String UPDATE_SCRIPT_VERSION_7_TO_8 = "sql/derby/integrityDB7to8migration.sql"; + /** * The current version of the database. */ - private final Integer currentVersion = 7; + private final int currentVersion = 8; /** * @param connector connection to the database. @@ -114,6 +117,15 @@ public void migrate() { log.warn("Migrating integrityDB from version 6 to 7"); migrateDerbyDatabase(UPDATE_SCRIPT_VERSION_6_TO_7); } + if (versions.get(DATABASE_VERSION_ENTRY) < 8) { + log.warn("Migrating integrityDB from version 7 to 8"); + migrateDerbyDatabase(UPDATE_SCRIPT_VERSION_7_TO_8); + } + + if (needsMigration()) { + log.error("Database still appears to need migration after it has been migrated. Expected version is {}.", + currentVersion); + } } @Override diff --git a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/database/IntegrityDAO.java b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/database/IntegrityDAO.java index 24d7d51d3..6effc2d7d 100644 --- a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/database/IntegrityDAO.java +++ b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/database/IntegrityDAO.java @@ -42,7 +42,16 @@ import java.sql.ResultSet; import java.sql.SQLException; import java.time.Instant; -import java.util.*; +import java.time.ZoneId; +import java.time.ZonedDateTime; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; /** * Common parts of the implementation of the access to the integrity db. @@ -504,7 +513,7 @@ public List getLatestPillarStats(String collectionID) { String latestPillarStatsSql = "SELECT pillarID, file_count, file_size, missing_files_count," + " checksum_errors_count, missing_checksums_count, obsolete_checksums_count," + - " 1000000000000 as oldest_checksum_timestamp" + // TODO extend table with new col + " 1000000000000 as oldest_checksum_timestamp" + // TODO Ole V. use new col " FROM pillarstats" + " WHERE stat_key = (" + " SELECT MAX(stat_key) FROM stats" + " WHERE collectionID = ?)"; @@ -524,18 +533,20 @@ public List getLatestPillarStats(String collectionID) { String pillarName = Objects.requireNonNullElse(SettingsUtils.getPillarName(pillarID), "N/A"); String pillarType = (SettingsUtils.getPillarType(pillarID) != null) ? Objects.requireNonNull(SettingsUtils.getPillarType(pillarID)).value() : "Unknown"; + String maxAgeForChecksums = SettingsUtils.getMaxAgeForChecksums(pillarID); long oldestChecksumTimestamp = dbResult.getLong("oldest_checksum_timestamp"); String ageOfOldestChecksum; if (dbResult.wasNull()) { ageOfOldestChecksum = "N/A"; } else { - ageOfOldestChecksum = TimeUtils.millisecondsToHuman( - System.currentTimeMillis() - oldestChecksumTimestamp); + ZoneId zone = ZoneId.systemDefault(); + ZonedDateTime oldestChecksumZdt = Instant.ofEpochMilli(oldestChecksumTimestamp).atZone(zone); + ageOfOldestChecksum = TimeUtils.humanDifference(oldestChecksumZdt, ZonedDateTime.now(zone)); } PillarCollectionStat p = new PillarCollectionStat(pillarID, collectionID, pillarName, pillarType, fileCount, dataSize, missingFiles, checksumErrors, missingChecksums, obsoleteChecksums, - "TODO", ageOfOldestChecksum, statsTime, updateTime); // TODO Ole V. conflict not correctly resolved + maxAgeForChecksums, ageOfOldestChecksum, statsTime, updateTime); stats.add(p); } } diff --git a/bitrepository-integrity-service/src/main/resources/sql/derby/integrityDB7to8migration.sql b/bitrepository-integrity-service/src/main/resources/sql/derby/integrityDB7to8migration.sql new file mode 100644 index 000000000..44fcde9bf --- /dev/null +++ b/bitrepository-integrity-service/src/main/resources/sql/derby/integrityDB7to8migration.sql @@ -0,0 +1,31 @@ +--- +-- #%L +-- Bitrepository Integrity Client +-- %% +-- Copyright (C) 2010 - 2022 Royal Danish Library and The State Archives, Denmark +-- %% +-- This program is free software: you can redistribute it and/or modify +-- it under the terms of the GNU Lesser General Public License as +-- published by the Free Software Foundation, either version 2.1 of the +-- License, or (at your option) any later version. +-- +-- This program is distributed in the hope that it will be useful, +-- but WITHOUT ANY WARRANTY; without even the implied warranty of +-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +-- GNU General Lesser Public License for more details. +-- +-- You should have received a copy of the GNU General Lesser Public +-- License along with this program. If not, see +-- . +-- #L% +--- + +connect 'jdbc:derby:integritydb'; + +-- database version +UPDATE tableversions SET version = 8 WHERE tablename = 'integritydb'; + +-- table version +UPDATE tableversions SET version = 3 WHERE tablename = 'pillarstats'; + +ALTER TABLE pillarstats ADD COLUMN oldest_checksum_timestamp BIGINT DEFAULT NULL; From 5b94f931a7eb85ecffc8c51173cd39d3324d2978 Mon Sep 17 00:00:00 2001 From: "Ole V. Villumsen" Date: Thu, 18 Aug 2022 14:13:48 +0200 Subject: [PATCH 25/47] BITMAG-1142 Display configured max age from MaxChecksumAgeProvider --- .../common/utils/SettingsUtils.java | 50 +++++++++++-------- .../cache/database/IntegrityDAO.java | 45 ++++++++++++----- 2 files changed, 61 insertions(+), 34 deletions(-) diff --git a/bitrepository-core/src/main/java/org/bitrepository/common/utils/SettingsUtils.java b/bitrepository-core/src/main/java/org/bitrepository/common/utils/SettingsUtils.java index 88c4b9335..8a50b0d97 100644 --- a/bitrepository-core/src/main/java/org/bitrepository/common/utils/SettingsUtils.java +++ b/bitrepository-core/src/main/java/org/bitrepository/common/utils/SettingsUtils.java @@ -22,8 +22,8 @@ package org.bitrepository.common.utils; import org.bitrepository.common.settings.Settings; +import org.bitrepository.settings.referencesettings.IntegrityServiceSettings; import org.bitrepository.settings.referencesettings.PillarIntegrityDetails; -import org.bitrepository.settings.referencesettings.PillarSettings; import org.bitrepository.settings.referencesettings.PillarType; import org.bitrepository.settings.repositorysettings.Collection; @@ -119,27 +119,32 @@ public static String getPillarName(String pillarID) { return null; } - /** - * Get the configured max age for checksums for the given pillarID from the ReferenceSettings. - * - * @param pillarID The pillarID for which the max checksum age is wanted. - * @return human-readable maximum age for checksums for the given pillar ID. - */ - public static String getMaxAgeForChecksums(String pillarID) { - PillarSettings pillarSettings = settings.getReferenceSettings().getPillarSettings(); - if (pillarSettings == null) { - return "Not set (no pillar settings)"; - } - if (! pillarSettings.getPillarID().equals(pillarID)) { - return "Unknown"; - } - BigInteger maxAge = pillarSettings.getMaxAgeForChecksums(); - try { - return TimeUtils.millisecondsToHuman(maxAge.longValueExact()); - } - catch (ArithmeticException ae) { - return String.format(Locale.getDefault(Locale.Category.FORMAT), "Extremely long; %d ms", maxAge); - } + // TODO Ole V. delete method +// /** +// * Get the configured max age for checksums for the given pillarID from the ReferenceSettings. +// * +// * @param pillarID The pillarID for which the max checksum age is wanted. +// * @return human-readable maximum age for checksums for the given pillar ID. +// */ +// public static String getMaxAgeForChecksums(String pillarID) { +// PillarSettings pillarSettings = settings.getReferenceSettings().getPillarSettings(); +// if (pillarSettings == null) { +// return "Not set (no pillar settings)"; +// } +// if (! pillarSettings.getPillarID().equals(pillarID)) { +// return "Unknown"; +// } +// BigInteger maxAge = pillarSettings.getMaxAgeForChecksums(); +// try { +// return TimeUtils.millisecondsToHuman(maxAge.longValueExact()); +// } +// catch (ArithmeticException ae) { +// return String.format(Locale.getDefault(Locale.Category.FORMAT), "Extremely long; %d ms", maxAge); +// } +// } + + public static IntegrityServiceSettings getIntegrityServiceSettings() { + return settings.getReferenceSettings().getIntegrityServiceSettings(); } /** @@ -231,4 +236,5 @@ public static Set getStatusContributorsForCollection() { contributors.addAll(SettingsUtils.getAllPillarIDs()); return contributors; } + } diff --git a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/database/IntegrityDAO.java b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/database/IntegrityDAO.java index 6effc2d7d..f154466d3 100644 --- a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/database/IntegrityDAO.java +++ b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/database/IntegrityDAO.java @@ -31,9 +31,13 @@ import org.bitrepository.integrityservice.cache.FileInfo; import org.bitrepository.integrityservice.cache.PillarCollectionMetric; import org.bitrepository.integrityservice.cache.PillarCollectionStat; +import org.bitrepository.integrityservice.checking.MaxChecksumAgeProvider; import org.bitrepository.integrityservice.statistics.StatisticsCollector; +import org.bitrepository.integrityservice.workflow.step.HandleObsoleteChecksumsStep; import org.bitrepository.service.database.DBConnector; import org.bitrepository.service.database.DatabaseUtils; +import org.bitrepository.settings.referencesettings.ObsoleteChecksumSettings; +import org.jetbrains.annotations.NotNull; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -79,12 +83,12 @@ public void close() { } /** - * Method to ensure that pillars found in RepositorySettings is present in the database + * Method to ensure that pillars found in RepositorySettings are present in the database */ protected abstract void initializePillars(); /** - * Method to ensure that collections found in RepositorySettings is present in the database + * Method to ensure that collections found in RepositorySettings are present in the database */ protected abstract void initializeCollections(); @@ -533,16 +537,8 @@ public List getLatestPillarStats(String collectionID) { String pillarName = Objects.requireNonNullElse(SettingsUtils.getPillarName(pillarID), "N/A"); String pillarType = (SettingsUtils.getPillarType(pillarID) != null) ? Objects.requireNonNull(SettingsUtils.getPillarType(pillarID)).value() : "Unknown"; - String maxAgeForChecksums = SettingsUtils.getMaxAgeForChecksums(pillarID); - long oldestChecksumTimestamp = dbResult.getLong("oldest_checksum_timestamp"); - String ageOfOldestChecksum; - if (dbResult.wasNull()) { - ageOfOldestChecksum = "N/A"; - } else { - ZoneId zone = ZoneId.systemDefault(); - ZonedDateTime oldestChecksumZdt = Instant.ofEpochMilli(oldestChecksumTimestamp).atZone(zone); - ageOfOldestChecksum = TimeUtils.humanDifference(oldestChecksumZdt, ZonedDateTime.now(zone)); - } + String maxAgeForChecksums = getMaxAgeForChecksums(pillarID); + String ageOfOldestChecksum = getAgeOfOldestChecksum(dbResult); PillarCollectionStat p = new PillarCollectionStat(pillarID, collectionID, pillarName, pillarType, fileCount, dataSize, missingFiles, checksumErrors, missingChecksums, obsoleteChecksums, @@ -559,6 +555,31 @@ public List getLatestPillarStats(String collectionID) { return stats; } + @NotNull + private String getMaxAgeForChecksums(String pillarID) { + ObsoleteChecksumSettings obsoleteChecksumSettings = + SettingsUtils.getIntegrityServiceSettings().getObsoleteChecksumSettings(); + MaxChecksumAgeProvider maxChecksumAgeProvider = + new MaxChecksumAgeProvider(HandleObsoleteChecksumsStep.DEFAULT_MAX_CHECKSUM_AGE, + obsoleteChecksumSettings); + long maxAge = maxChecksumAgeProvider.getMaxChecksumAge(pillarID); + return maxAge == 0 ? "unlimited" : TimeUtils.millisecondsToHuman(maxAge); + } + + @NotNull + private String getAgeOfOldestChecksum(ResultSet dbResult) throws SQLException { + long oldestChecksumTimestamp = dbResult.getLong("oldest_checksum_timestamp"); + String ageOfOldestChecksum; + if (dbResult.wasNull()) { + ageOfOldestChecksum = "N/A"; + } else { + ZoneId zone = ZoneId.systemDefault(); + ZonedDateTime oldestChecksumZdt = Instant.ofEpochMilli(oldestChecksumTimestamp).atZone(zone); + ageOfOldestChecksum = TimeUtils.humanDifference(oldestChecksumZdt, ZonedDateTime.now(zone)); + } + return ageOfOldestChecksum; + } + /** * Method that should deliver the database specific SQL for getting the latest N collection statistics * From 121e14d75a325f1d3243e8e4b976ccdf881eb925 Mon Sep 17 00:00:00 2001 From: "Ole V. Villumsen" Date: Fri, 19 Aug 2022 09:26:37 +0200 Subject: [PATCH 26/47] BITMAG-1142 Use real checksum timestamp from database --- .../integrityservice/cache/database/IntegrityDAO.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/database/IntegrityDAO.java b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/database/IntegrityDAO.java index f154466d3..85fddd7e6 100644 --- a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/database/IntegrityDAO.java +++ b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/database/IntegrityDAO.java @@ -517,9 +517,12 @@ public List getLatestPillarStats(String collectionID) { String latestPillarStatsSql = "SELECT pillarID, file_count, file_size, missing_files_count," + " checksum_errors_count, missing_checksums_count, obsolete_checksums_count," + - " 1000000000000 as oldest_checksum_timestamp" + // TODO Ole V. use new col + " oldest_checksum_timestamp" + " FROM pillarstats" + - " WHERE stat_key = (" + " SELECT MAX(stat_key) FROM stats" + " WHERE collectionID = ?)"; + " WHERE stat_key = (" + + " SELECT MAX(stat_key)" + + " FROM stats" + + " WHERE collectionID = ?)"; try (Connection conn = dbConnector.getConnection(); PreparedStatement ps = DatabaseUtils.createPreparedStatement(conn, latestPillarStatsSql, collectionID)) { From 36a32eeeaebdba30dcbbb30f903af01391a3f4c1 Mon Sep 17 00:00:00 2001 From: "Ole V. Villumsen" Date: Wed, 24 Aug 2022 15:03:30 +0200 Subject: [PATCH 27/47] BITMAG-1142 Fix bugs and missing parts in display of age of oldest checksum --- .../bitrepository/common/utils/TimeUtils.java | 26 ++++++-------- .../common/utils/TimeUtilsTest.java | 17 ++++----- .../cache/PillarCollectionStat.java | 36 ++++++++++++++++--- .../cache/database/IntegrityDAO.java | 19 +++------- .../cache/database/StatisticsCreator.java | 33 +++++++++-------- .../web/RestIntegrityService.java | 5 +-- .../step/CreateStatisticsEntryStep.java | 2 ++ 7 files changed, 78 insertions(+), 60 deletions(-) diff --git a/bitrepository-core/src/main/java/org/bitrepository/common/utils/TimeUtils.java b/bitrepository-core/src/main/java/org/bitrepository/common/utils/TimeUtils.java index 0fdf794f4..7e176e8bf 100644 --- a/bitrepository-core/src/main/java/org/bitrepository/common/utils/TimeUtils.java +++ b/bitrepository-core/src/main/java/org/bitrepository/common/utils/TimeUtils.java @@ -29,6 +29,7 @@ import java.time.Duration; import java.time.Period; import java.time.ZonedDateTime; +import java.time.temporal.ChronoUnit; import java.util.ArrayList; import java.util.Date; import java.util.List; @@ -140,16 +141,20 @@ public static String humanDifference(ZonedDateTime start, ZonedDateTime end) { periodBetween = Period.between(start.toLocalDate(), end.toLocalDate().minusDays(1)); afterPeriod = start.plus(periodBetween); } - Duration durationBetween = Duration.between(afterPeriod, end); + // Round duration to whole seconds + Duration durationBetween = Duration.between(afterPeriod, end) + .plusMillis(500) + .truncatedTo(ChronoUnit.SECONDS); if (periodBetween.isZero() && durationBetween.isZero()) { - return "0 ms"; + return "0s"; } - // The following gives an ambiguous string like "3m" - // in the very rare cases where only months or only minutes are non-zero. - // Since in practice the text is updated a few seconds later, it is not expected to be a problem for the user. - List elements = new ArrayList<>(7); + // The following gives an ambiguous string like "3m" or "3y 11m 5s" + // in the very rare cases where months *or* minutes are non-zero and days and hours are zero. + // Since in practice the text is updated a few seconds later and any minutes 1 minute later, + // it is not expected to be a problem for the user. + List elements = new ArrayList<>(6); if (periodBetween.getYears() != 0) { elements.add(periodBetween.getYears() + "y"); } @@ -168,15 +173,6 @@ public static String humanDifference(ZonedDateTime start, ZonedDateTime end) { if (durationBetween.toSecondsPart() != 0) { elements.add(durationBetween.toSecondsPart() + "s"); } - if (durationBetween.toNanosPart() != 0) { - int millis = durationBetween.toMillisPart(); - int nanos = durationBetween.toNanosPart(); - if (Duration.ofMillis(millis).toNanos() == nanos) { // millis give full precision; print them - elements.add(millis + " ms"); - } else { // print only nanos - elements.add(nanos + " ns"); - } - } return String.join(" ", elements); } diff --git a/bitrepository-core/src/test/java/org/bitrepository/common/utils/TimeUtilsTest.java b/bitrepository-core/src/test/java/org/bitrepository/common/utils/TimeUtilsTest.java index a9a437352..d0297d9de 100644 --- a/bitrepository-core/src/test/java/org/bitrepository/common/utils/TimeUtilsTest.java +++ b/bitrepository-core/src/test/java/org/bitrepository/common/utils/TimeUtilsTest.java @@ -92,20 +92,21 @@ public void differencesPrintHumanly() { addDescription("TimeUtils.humanDifference() should return" + " similar human readable strings to those from millisecondsToHuman()"); - addStep("Call humanDifference() with same time twice", "The output should be '0 ms'"); + addStep("Call humanDifference() with same time twice", "The output should be '0s'"); String zeroTimeString = TimeUtils.humanDifference(BASE, BASE); - assertEquals(zeroTimeString, "0 ms"); + assertEquals(zeroTimeString, "0s"); addStep("Call humanDifference() with a difference obtained from a Duration", "Expect corresponding readable output"); - testHumanDifference("1 ns", Duration.ofNanos(1)); - testHumanDifference("1 ms", Duration.ofMillis(1)); - // If there are nanos, don’t print millis - testHumanDifference("2000003 ns", Duration.ofNanos(2_000_003)); + // Don’t print fraction of second + testHumanDifference("0s", Duration.ofNanos(1)); + testHumanDifference("0s", Duration.ofMillis(1)); + testHumanDifference("0s", Duration.ofNanos(499_999_999)); + testHumanDifference("1s", Duration.ofNanos(500_000_000)); testHumanDifference("1s", Duration.ofSeconds(1)); testHumanDifference("1m", Duration.ofMinutes(1)); testHumanDifference("1h", Duration.ofHours(1)); - testHumanDifference("2h 3m 5s 7 ns", Duration.parse("PT2H3M5.000000007S")); + testHumanDifference("2h 3m 5s", Duration.parse("PT2H3M5.000000007S")); addStep("Call humanDifference() with a difference obtained from a Period", "Expect corresponding readable output"); @@ -116,7 +117,7 @@ public void differencesPrintHumanly() { addStep("Call humanDifference() with a difference obtained from a combo of a Period and a Duration", "Expect corresponding readable output"); - testHumanDifference("3y 5m 7d 11h 13m 17s 23 ms", + testHumanDifference("3y 5m 7d 11h 13m 17s", Period.of(3, 5, 7), Duration.parse("PT11H13M17.023S")); addStep("Call humanDifference()" + diff --git a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/PillarCollectionStat.java b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/PillarCollectionStat.java index f7e674bbd..0c006eeb2 100644 --- a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/PillarCollectionStat.java +++ b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/PillarCollectionStat.java @@ -21,6 +21,12 @@ */ package org.bitrepository.integrityservice.cache; +import org.bitrepository.common.utils.TimeUtils; +import org.jetbrains.annotations.NotNull; + +import java.time.Instant; +import java.time.ZoneId; +import java.time.ZonedDateTime; import java.util.Date; /** @@ -38,8 +44,7 @@ public class PillarCollectionStat { private Long missingChecksums = 0L; private Long checksumErrors = 0L; private String maxAgeForChecksums; - /** Human-readable age of the oldest checksum, for example " 3m 46s 299 ms" */ - private String ageOfOldestChecksum; + private Instant oldestChecksumTimestamp; private Date statsTime; private Date updateTime; @@ -53,7 +58,8 @@ public PillarCollectionStat(String pillarID, String collectionID, String pillarN public PillarCollectionStat(String pillarID, String collectionID, String pillarName, String pillarType, Long fileCount, Long dataSize, Long missingFiles, Long checksumErrors, Long missingChecksums, Long obsoleteChecksum, - String maxAgeForChecksums, String ageOfOldestChecksum, Date statsTime, Date updateTime) { + String maxAgeForChecksums, + Instant oldestChecksumTimestamp, Date statsTime, Date updateTime) { this.pillarID = pillarID; this.collectionID = collectionID; this.pillarName = pillarName; @@ -67,7 +73,7 @@ public PillarCollectionStat(String pillarID, String collectionID, String pillarN this.statsTime = statsTime; this.updateTime = updateTime; this.maxAgeForChecksums = maxAgeForChecksums; - this.ageOfOldestChecksum = ageOfOldestChecksum; + this.oldestChecksumTimestamp = oldestChecksumTimestamp; } public String getPillarID() { @@ -150,11 +156,31 @@ public void setMissingChecksums(Long missingChecksums) { this.missingChecksums = missingChecksums; } + /** @return Human-readable age of the oldest checksum, for example "3m 46s" */ + @NotNull public String getAgeOfOldestChecksum() { - return ageOfOldestChecksum; + if (oldestChecksumTimestamp == null) { + return "N/A"; + } + ZoneId zone = ZoneId.systemDefault(); + return TimeUtils.humanDifference(oldestChecksumTimestamp.atZone(zone), ZonedDateTime.now(zone)); + } + + public boolean hasOldestChecksumTimestamp() { + return oldestChecksumTimestamp != null; + } + + /** @throws NullPointerException if hasOldestChecksumTimestamp() does not return true */ + public long getOldestChecksumTimestampMillis() { + return oldestChecksumTimestamp.toEpochMilli(); + } + + public void setOldestChecksumTimestamp(Instant oldestChecksumTimestamp) { + this.oldestChecksumTimestamp = oldestChecksumTimestamp; } public String getMaxAgeForChecksums() { return maxAgeForChecksums; } + } diff --git a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/database/IntegrityDAO.java b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/database/IntegrityDAO.java index 85fddd7e6..c66acb116 100644 --- a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/database/IntegrityDAO.java +++ b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/database/IntegrityDAO.java @@ -46,8 +46,6 @@ import java.sql.ResultSet; import java.sql.SQLException; import java.time.Instant; -import java.time.ZoneId; -import java.time.ZonedDateTime; import java.util.ArrayList; import java.util.Arrays; import java.util.Date; @@ -541,11 +539,11 @@ public List getLatestPillarStats(String collectionID) { String pillarType = (SettingsUtils.getPillarType(pillarID) != null) ? Objects.requireNonNull(SettingsUtils.getPillarType(pillarID)).value() : "Unknown"; String maxAgeForChecksums = getMaxAgeForChecksums(pillarID); - String ageOfOldestChecksum = getAgeOfOldestChecksum(dbResult); + Instant oldestChecksumTimestamp = getOldestChecksumTimestamp(dbResult); PillarCollectionStat p = new PillarCollectionStat(pillarID, collectionID, pillarName, pillarType, fileCount, dataSize, missingFiles, checksumErrors, missingChecksums, obsoleteChecksums, - maxAgeForChecksums, ageOfOldestChecksum, statsTime, updateTime); + maxAgeForChecksums, oldestChecksumTimestamp, statsTime, updateTime); stats.add(p); } } @@ -569,18 +567,9 @@ private String getMaxAgeForChecksums(String pillarID) { return maxAge == 0 ? "unlimited" : TimeUtils.millisecondsToHuman(maxAge); } - @NotNull - private String getAgeOfOldestChecksum(ResultSet dbResult) throws SQLException { + private Instant getOldestChecksumTimestamp(ResultSet dbResult) throws SQLException { long oldestChecksumTimestamp = dbResult.getLong("oldest_checksum_timestamp"); - String ageOfOldestChecksum; - if (dbResult.wasNull()) { - ageOfOldestChecksum = "N/A"; - } else { - ZoneId zone = ZoneId.systemDefault(); - ZonedDateTime oldestChecksumZdt = Instant.ofEpochMilli(oldestChecksumTimestamp).atZone(zone); - ageOfOldestChecksum = TimeUtils.humanDifference(oldestChecksumZdt, ZonedDateTime.now(zone)); - } - return ageOfOldestChecksum; + return dbResult.wasNull() ? null : Instant.ofEpochMilli(oldestChecksumTimestamp); } /** diff --git a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/database/StatisticsCreator.java b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/database/StatisticsCreator.java index cefb53bd9..c0409ee6d 100644 --- a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/database/StatisticsCreator.java +++ b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/database/StatisticsCreator.java @@ -28,9 +28,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.sql.Connection; -import java.sql.PreparedStatement; -import java.sql.SQLException; +import java.sql.*; import java.util.Date; import java.util.List; @@ -49,8 +47,8 @@ public class StatisticsCreator { private final String insertPillarStatEntrySql = "INSERT INTO pillarstats" + " (stat_key, pillarID, file_count, file_size, missing_files_count, " - + "checksum_errors_count, missing_checksums_count, obsolete_checksums_count)" - + " (SELECT MAX(stat_key), ?, ?, ?, ?, ?, ?, ? FROM stats WHERE collectionID = ?)"; + + "checksum_errors_count, missing_checksums_count, obsolete_checksums_count, oldest_checksum_timestamp)" + + " (SELECT MAX(stat_key), ?, ?, ?, ?, ?, ?, ?, ? FROM stats WHERE collectionID = ?)"; private final Logger log = LoggerFactory.getLogger(getClass()); @@ -112,19 +110,24 @@ private void addCollectionStatistics(CollectionStat cs) throws SQLException { insertCollectionStatPS.setString(5, cs.getCollectionID()); } - private void addPillarStat(PillarCollectionStat ps) throws SQLException { - insertPillarStatPS.setString(1, ps.getPillarID()); - insertPillarStatPS.setLong(2, ps.getFileCount()); - insertPillarStatPS.setLong(3, ps.getDataSize()); - insertPillarStatPS.setLong(4, ps.getMissingFiles()); - insertPillarStatPS.setLong(5, ps.getChecksumErrors()); - insertPillarStatPS.setLong(6, ps.getMissingChecksums()); - insertPillarStatPS.setLong(7, ps.getObsoleteChecksums()); - insertPillarStatPS.setString(8, ps.getCollectionID()); + private void addPillarStat(PillarCollectionStat pcStat) throws SQLException { + insertPillarStatPS.setString(1, pcStat.getPillarID()); + insertPillarStatPS.setLong(2, pcStat.getFileCount()); + insertPillarStatPS.setLong(3, pcStat.getDataSize()); + insertPillarStatPS.setLong(4, pcStat.getMissingFiles()); + insertPillarStatPS.setLong(5, pcStat.getChecksumErrors()); + insertPillarStatPS.setLong(6, pcStat.getMissingChecksums()); + insertPillarStatPS.setLong(7, pcStat.getObsoleteChecksums()); + if (pcStat.hasOldestChecksumTimestamp()) { + insertPillarStatPS.setLong(8, pcStat.getOldestChecksumTimestampMillis()); + } else { + insertPillarStatPS.setNull(8, Types.BIGINT); + } + + insertPillarStatPS.setString(9, pcStat.getCollectionID()); insertPillarStatPS.addBatch(); } - private void execute() throws SQLException { insertStatisticsEntryPS.execute(); insertCollectionStatPS.execute(); diff --git a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/web/RestIntegrityService.java b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/web/RestIntegrityService.java index e292858d6..4d7853450 100644 --- a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/web/RestIntegrityService.java +++ b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/web/RestIntegrityService.java @@ -264,8 +264,9 @@ public String getIntegrityStatus( String pillarName = Objects.requireNonNullElse(SettingsUtils.getPillarName(pillar), "N/A"); PillarType pillarTypeObject = SettingsUtils.getPillarType(pillar); String pillarType = pillarTypeObject != null ? pillarTypeObject.value() : null; - PillarCollectionStat emptyStat = new PillarCollectionStat(pillar, collectionID, pillarName, pillarType, - 0L, 0L, 0L, 0L, 0L, 0L, "", "", new Date(0), new Date(0)); + PillarCollectionStat emptyStat = new PillarCollectionStat(pillar, collectionID, pillarName, + pillarType, 0L, 0L, 0L, 0L, 0L, + 0L, "", null, new Date(0), new Date(0)); stats.put(pillar, emptyStat); } } diff --git a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/workflow/step/CreateStatisticsEntryStep.java b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/workflow/step/CreateStatisticsEntryStep.java index 40427b1f3..fab49ba3a 100644 --- a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/workflow/step/CreateStatisticsEntryStep.java +++ b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/workflow/step/CreateStatisticsEntryStep.java @@ -63,9 +63,11 @@ public synchronized void performStep() { if (metric == null) { sc.getPillarCollectionStat(pillar).setFileCount(0L); sc.getPillarCollectionStat(pillar).setDataSize(0L); + sc.getPillarCollectionStat(pillar).setOldestChecksumTimestamp(null); } else { sc.getPillarCollectionStat(pillar).setFileCount(metric.getPillarFileCount()); sc.getPillarCollectionStat(pillar).setDataSize(metric.getPillarCollectionSize()); + sc.getPillarCollectionStat(pillar).setOldestChecksumTimestamp(metric.getOldestChecksumTimestamp()); } } sc.getCollectionStat().setFileCount(store.getNumberOfFilesInCollection(collectionID)); From 81c729105bcb51909ca6c1bdd2d2a98c7ddf3432 Mon Sep 17 00:00:00 2001 From: "Ole V. Villumsen" Date: Thu, 25 Aug 2022 13:01:50 +0200 Subject: [PATCH 28/47] BITMAG-1142 Delete method that is no longer used after change --- .../common/utils/SettingsUtils.java | 24 ------------------- 1 file changed, 24 deletions(-) diff --git a/bitrepository-core/src/main/java/org/bitrepository/common/utils/SettingsUtils.java b/bitrepository-core/src/main/java/org/bitrepository/common/utils/SettingsUtils.java index 8a50b0d97..c97eb183e 100644 --- a/bitrepository-core/src/main/java/org/bitrepository/common/utils/SettingsUtils.java +++ b/bitrepository-core/src/main/java/org/bitrepository/common/utils/SettingsUtils.java @@ -119,30 +119,6 @@ public static String getPillarName(String pillarID) { return null; } - // TODO Ole V. delete method -// /** -// * Get the configured max age for checksums for the given pillarID from the ReferenceSettings. -// * -// * @param pillarID The pillarID for which the max checksum age is wanted. -// * @return human-readable maximum age for checksums for the given pillar ID. -// */ -// public static String getMaxAgeForChecksums(String pillarID) { -// PillarSettings pillarSettings = settings.getReferenceSettings().getPillarSettings(); -// if (pillarSettings == null) { -// return "Not set (no pillar settings)"; -// } -// if (! pillarSettings.getPillarID().equals(pillarID)) { -// return "Unknown"; -// } -// BigInteger maxAge = pillarSettings.getMaxAgeForChecksums(); -// try { -// return TimeUtils.millisecondsToHuman(maxAge.longValueExact()); -// } -// catch (ArithmeticException ae) { -// return String.format(Locale.getDefault(Locale.Category.FORMAT), "Extremely long; %d ms", maxAge); -// } -// } - public static IntegrityServiceSettings getIntegrityServiceSettings() { return settings.getReferenceSettings().getIntegrityServiceSettings(); } From 0087c14915a442ca5a3cbbceebfb0d69c3074d70 Mon Sep 17 00:00:00 2001 From: "Ole V. Villumsen" Date: Thu, 25 Aug 2022 13:24:09 +0200 Subject: [PATCH 29/47] Fix some of the errors from attempt to solve merge conflicts --- .../integrityservice/cache/PillarCollectionStat.java | 12 ++++++------ .../cache/database/IntegrityDAO.java | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/PillarCollectionStat.java b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/PillarCollectionStat.java index 0c006eeb2..1382346d1 100644 --- a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/PillarCollectionStat.java +++ b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/PillarCollectionStat.java @@ -35,7 +35,7 @@ public class PillarCollectionStat { private final String pillarID; private final String collectionID; - private final String pillarName; + private final String pillarHostname; private final String pillarType; private Long fileCount = 0L; private Long dataSize = 0L; @@ -48,10 +48,10 @@ public class PillarCollectionStat { private Date statsTime; private Date updateTime; - public PillarCollectionStat(String pillarID, String collectionID, String pillarName, String pillarType) { + public PillarCollectionStat(String pillarID, String collectionID, String pillarHostname, String pillarType) { this.pillarID = pillarID; this.collectionID = collectionID; - this.pillarName = pillarName; + this.pillarHostname = pillarHostname; this.pillarType = pillarType; } @@ -62,7 +62,7 @@ public PillarCollectionStat(String pillarID, String collectionID, String pillarN Instant oldestChecksumTimestamp, Date statsTime, Date updateTime) { this.pillarID = pillarID; this.collectionID = collectionID; - this.pillarName = pillarName; + this.pillarHostname = pillarHostname; this.pillarType = pillarType; this.fileCount = fileCount; this.dataSize = dataSize; @@ -84,8 +84,8 @@ public String getCollectionID() { return collectionID; } - public String getPillarName() { - return pillarName; + public String getPillarHostname() { + return pillarHostname; } public String getPillarType() { diff --git a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/database/IntegrityDAO.java b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/database/IntegrityDAO.java index c66acb116..1a02aaf34 100644 --- a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/database/IntegrityDAO.java +++ b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/database/IntegrityDAO.java @@ -535,7 +535,7 @@ public List getLatestPillarStats(String collectionID) { Long obsoleteChecksums = dbResult.getLong("obsolete_checksums_count"); Date statsTime = null; Date updateTime = null; - String pillarName = Objects.requireNonNullElse(SettingsUtils.getPillarName(pillarID), "N/A"); + String pillarHostname = Objects.requireNonNullElse(SettingsUtils.getPillarName(pillarID), "N/A"); String pillarType = (SettingsUtils.getPillarType(pillarID) != null) ? Objects.requireNonNull(SettingsUtils.getPillarType(pillarID)).value() : "Unknown"; String maxAgeForChecksums = getMaxAgeForChecksums(pillarID); From 4b7f5c28463ba85892d36176a444e524e820887a Mon Sep 17 00:00:00 2001 From: "Ole V. Villumsen" Date: Tue, 26 Jul 2022 12:55:59 +0200 Subject: [PATCH 30/47] BITMAG-1142 Checksum age in GUI: underway --- .../common/utils/SettingsUtils.java | 20 +++ .../cache/database/IntegrityDAO.java | 25 +++ .../web/RestIntegrityService.java | 5 + .../src/main/webapp/integrity-service.html | 149 ++++++++++++++++++ 4 files changed, 199 insertions(+) diff --git a/bitrepository-core/src/main/java/org/bitrepository/common/utils/SettingsUtils.java b/bitrepository-core/src/main/java/org/bitrepository/common/utils/SettingsUtils.java index c97eb183e..8c6de3993 100644 --- a/bitrepository-core/src/main/java/org/bitrepository/common/utils/SettingsUtils.java +++ b/bitrepository-core/src/main/java/org/bitrepository/common/utils/SettingsUtils.java @@ -123,6 +123,26 @@ public static IntegrityServiceSettings getIntegrityServiceSettings() { return settings.getReferenceSettings().getIntegrityServiceSettings(); } + /** + * Get the configured max age for checksums for the given pillarID from the ReferenceSettings. + * + * @param pillarID The pillarID for which the hostname is wanted. + * @return human-readable maximum age for checksums for the given pillar ID. + */ + public static String getMaxAgeForChecksums(String pillarID) { + // TODO how to use pillarID?? + BigInteger maxAge = settings.getReferenceSettings().getPillarSettings().getMaxAgeForChecksums(); + if (maxAge == null) { + return null; + } + try { + return TimeUtils.millisecondsToHuman(maxAge.longValueExact()); + } + catch (ArithmeticException) { + return "Extremely long"; + } + } + /** * Get the {@link PillarType} for the given Pillar ID. * diff --git a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/database/IntegrityDAO.java b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/database/IntegrityDAO.java index 1a02aaf34..67e1eda9d 100644 --- a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/database/IntegrityDAO.java +++ b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/database/IntegrityDAO.java @@ -46,6 +46,7 @@ import java.sql.ResultSet; import java.sql.SQLException; import java.time.Instant; +<<<<<<< HEAD import java.util.ArrayList; import java.util.Arrays; import java.util.Date; @@ -54,6 +55,9 @@ import java.util.Map; import java.util.Objects; import java.util.Optional; +======= +import java.util.*; +>>>>>>> ac4d1581c (BITMAG-1142 Checksum age in GUI: underway) /** * Common parts of the implementation of the access to the integrity db. @@ -515,12 +519,18 @@ public List getLatestPillarStats(String collectionID) { String latestPillarStatsSql = "SELECT pillarID, file_count, file_size, missing_files_count," + " checksum_errors_count, missing_checksums_count, obsolete_checksums_count," + +<<<<<<< HEAD " oldest_checksum_timestamp" + " FROM pillarstats" + " WHERE stat_key = (" + " SELECT MAX(stat_key)" + " FROM stats" + " WHERE collectionID = ?)"; +======= + " 1000000000000 as oldest_checksum_timestamp" + // TODO extend table with new col + " FROM pillarstats" + + " WHERE stat_key = (" + " SELECT MAX(stat_key) FROM stats" + " WHERE collectionID = ?)"; +>>>>>>> ac4d1581c (BITMAG-1142 Checksum age in GUI: underway) try (Connection conn = dbConnector.getConnection(); PreparedStatement ps = DatabaseUtils.createPreparedStatement(conn, latestPillarStatsSql, collectionID)) { @@ -538,12 +548,27 @@ public List getLatestPillarStats(String collectionID) { String pillarHostname = Objects.requireNonNullElse(SettingsUtils.getPillarName(pillarID), "N/A"); String pillarType = (SettingsUtils.getPillarType(pillarID) != null) ? Objects.requireNonNull(SettingsUtils.getPillarType(pillarID)).value() : "Unknown"; +<<<<<<< HEAD String maxAgeForChecksums = getMaxAgeForChecksums(pillarID); Instant oldestChecksumTimestamp = getOldestChecksumTimestamp(dbResult); PillarCollectionStat p = new PillarCollectionStat(pillarID, collectionID, pillarName, pillarType, fileCount, dataSize, missingFiles, checksumErrors, missingChecksums, obsoleteChecksums, maxAgeForChecksums, oldestChecksumTimestamp, statsTime, updateTime); +======= + long oldestChecksumTimestamp = dbResult.getLong("oldest_checksum_timestamp"); + String ageOfOldestChecksum; + if (dbResult.wasNull()) { + ageOfOldestChecksum = "N/A"; + } else { + ageOfOldestChecksum = TimeUtils.millisecondsToHuman( + System.currentTimeMillis() - oldestChecksumTimestamp); + } + PillarCollectionStat p = new PillarCollectionStat(pillarID, collectionID, + pillarHostname, pillarType, fileCount, dataSize, + missingFiles, checksumErrors, missingChecksums, obsoleteChecksums, + "TODO", ageOfOldestChecksum, statsTime, updateTime); +>>>>>>> ac4d1581c (BITMAG-1142 Checksum age in GUI: underway) stats.add(p); } } diff --git a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/web/RestIntegrityService.java b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/web/RestIntegrityService.java index 4d7853450..b00d95242 100644 --- a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/web/RestIntegrityService.java +++ b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/web/RestIntegrityService.java @@ -264,9 +264,14 @@ public String getIntegrityStatus( String pillarName = Objects.requireNonNullElse(SettingsUtils.getPillarName(pillar), "N/A"); PillarType pillarTypeObject = SettingsUtils.getPillarType(pillar); String pillarType = pillarTypeObject != null ? pillarTypeObject.value() : null; +<<<<<<< HEAD PillarCollectionStat emptyStat = new PillarCollectionStat(pillar, collectionID, pillarName, pillarType, 0L, 0L, 0L, 0L, 0L, 0L, "", null, new Date(0), new Date(0)); +======= + PillarCollectionStat emptyStat = new PillarCollectionStat(pillar, collectionID, pillarHostname, pillarType, + 0L, 0L, 0L, 0L, 0L, 0L, "", "", new Date(0), new Date(0)); +>>>>>>> ac4d1581c (BITMAG-1142 Checksum age in GUI: underway) stats.put(pillar, emptyStat); } } diff --git a/bitrepository-webclient/src/main/webapp/integrity-service.html b/bitrepository-webclient/src/main/webapp/integrity-service.html index 21fa1f213..27ffd5ffb 100644 --- a/bitrepository-webclient/src/main/webapp/integrity-service.html +++ b/bitrepository-webclient/src/main/webapp/integrity-service.html @@ -94,7 +94,10 @@

Integrity service

Number of inconsistent checksums Configured max age of checksums Age of oldest checksum +<<<<<<< HEAD >>>>>>> 10b99c1d8 (BITMAG-1142 Checksum age in GUI: underway) +======= +>>>>>>> ac4d1581c (BITMAG-1142 Checksum age in GUI: underway) @@ -211,6 +214,152 @@

Modal header

}); } +<<<<<<< HEAD +======= + function getPagingLimit(id, member) { + var myID = id; + var myMember = member; + return function() { + return pillars[myID][myMember]; + } + } + + function showModalPager(id, member, title, url) { + var myTitle = title; + var myUrl = url; + var myID = id; + var myMember = member; + return function() { + pager = new Pager(getPagingLimit(myID, myMember), 20, myUrl, "#modalPager", "#modalPagerBody"); + $("#modalPagerLabel").html(myTitle); + $("#modalPagerBody").html("

Loading

"); + pager.getPage(1)(); + $("#modalPagerDialog").modal('show'); + } + } + + function getCellContext(id, type) { + var context = {}; + context.url = integrityServiceUrl + "/integrity/IntegrityService/"; + if(type == "Pillar Hostname") { + context.element = id + "-pillarHostname"; + context.title = type + " on " + id; + context.member = "pillarHostname"; + } else if(type == "Pillar Type") { + context.element = id + "-pillarType"; + context.title = type + " on " + id; + context.member = "pillarType"; + } else if(type == "Total files") { + context.element = id + "-totalFileCount"; + context.title = type + " on " + id; + context.member = "totalFileCount"; + context.url += "getAllFileIDs/"; + } else if(type == "Missing files") { + context.element = id + "-missingFiles"; + context.title = type + " on " + id; + context.member = "missingFilesCount"; + context.url += "getMissingFileIDs/"; + } else if(type == "Missing checksums") { + context.element = id + "-missingChecksums"; + context.title = type + " on " + id; + context.member = "missingChecksumsCount"; + context.url += "getMissingChecksumsFileIDs/"; + } else if(type == "Obsolete checksums") { + context.element = id + "-obsoleteChecksums"; + context.title = type + " on " + id; + context.member = "obsoleteChecksumsCount"; + context.url += "getObsoleteChecksumsFileIDs/"; + } else if(type == "Inconsistent checksums") { + context.element = id + "-checksumErrors"; + context.title = type + " on " + id; + context.member = "checksumErrorCount"; + context.url += "getChecksumErrorFileIDs/"; + } else if(type == "Checksum age limit") { + context.element = id + "-maxAgeForChecksums"; + context.title = type + " on " + id; + context.member = "maxAgeForChecksums"; + } else if(type == "Oldest checksum age") { + context.element = id + "-ageOfOldestChecksum"; + context.title = type + " on " + id; + context.member = "ageOfOldestChecksum"; + } + context.url += "?pillarID=" + id + "&collectionID=" + getCollectionID(); + return context; + } + + function makePillarRow(id) { + var html = ""; + html += ""; + html += "
" + id + "
"; + html += ""; + html += ""; + html += ""; + html += ""; + html += ""; + html += ""; + html += ""; + html += ""; + html += ""; + html += ""; + return html; + } + + function updateCells(pillarID) { + updateStringCell(pillarID, "Pillar Hostname", pillars[pillarID].pillarHostname); + updateStringCell(pillarID, "Pillar Type", pillars[pillarID].pillarType); + updateIntCell(pillarID, "Total files", pillars[pillarID].totalFileCount); + updateIntCell(pillarID, "Missing files", pillars[pillarID].missingFilesCount); + updateIntCell(pillarID, "Missing checksums", pillars[pillarID].missingChecksumsCount); + updateIntCell(pillarID, "Obsolete checksums", pillars[pillarID].obsoleteChecksumsCount); + updateIntCell(pillarID, "Inconsistent checksums", pillars[pillarID].checksumErrorCount); + updateStringCell(pillarID, "Checksum age limit", pillars[pillarID].maxAgeForChecksums); + updateStringCell(pillarID, "Oldest checksum age", pillars[pillarID].ageOfOldestChecksum); + } + + function updateStringCell(id, type, cellValue) { + var context = getCellContext(id, type); + var html = "

" + cellValue + "

"; + $("#" + context.element).html(html); + } + + function updateIntCell(id, type, cellValue) { + var context = getCellContext(id, type); + if(cellValue == 0) { + var html = ""; + $("#" + context.element).html(html); + } else { + var innerElement = context.element + "-a"; + var html = "" + formatInt(cellValue) + ""; + $("#" + context.element).html(html); + $("#" + innerElement).click(showModalPager(id, context.member, context.title, context.url)); + } + } + + function getIntegrityStatus() { + var url = integrityServiceUrl + "/integrity/IntegrityService/getIntegrityStatus/?collectionID=" + + getCollectionID(); + $.getJSON(url, {}, function(j){ + var htmlTable; + for (var i = 0; i < j.length; i++) { + if(pillars[j[i].pillarID] == null) { + $("#integrity-status-table-body").append(makePillarRow(j[i].pillarID)); + } + pillars[j[i].pillarID] = { + pillarHostname : j[i].pillarHostname, + pillarType : j[i].pillarType, + totalFileCount : j[i].totalFileCount, + missingFilesCount : j[i].missingFilesCount, + missingChecksumsCount : j[i].missingChecksumsCount, + obsoleteChecksumsCount : j[i].obsoleteChecksumsCount, + checksumErrorCount : j[i].checksumErrorCount, + maxAgeForChecksums : j[i].maxAgeForChecksums, + ageOfOldestChecksum : j[i].ageOfOldestChecksum}; + updateCells(j[i].pillarID); + } + }); + } + +>>>>>>> ac4d1581c (BITMAG-1142 Checksum age in GUI: underway) function startWorkflow() { let ID = $("#workflowSelector option:selected").val(); let url = integrityServiceUrl + '/integrity/IntegrityService/startWorkflow'; From 774da523fb090187bc94b247dd79f9adf1395d1b Mon Sep 17 00:00:00 2001 From: "Ole V. Villumsen" Date: Wed, 27 Jul 2022 12:16:38 +0200 Subject: [PATCH 31/47] Fix syntax error --- .../main/java/org/bitrepository/common/utils/SettingsUtils.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bitrepository-core/src/main/java/org/bitrepository/common/utils/SettingsUtils.java b/bitrepository-core/src/main/java/org/bitrepository/common/utils/SettingsUtils.java index 8c6de3993..cbe5ded8d 100644 --- a/bitrepository-core/src/main/java/org/bitrepository/common/utils/SettingsUtils.java +++ b/bitrepository-core/src/main/java/org/bitrepository/common/utils/SettingsUtils.java @@ -138,7 +138,7 @@ public static String getMaxAgeForChecksums(String pillarID) { try { return TimeUtils.millisecondsToHuman(maxAge.longValueExact()); } - catch (ArithmeticException) { + catch (ArithmeticException ae) { return "Extremely long"; } } From 32ca97dd0671a3840a5afb0e265206523a0ed314 Mon Sep 17 00:00:00 2001 From: "Ole V. Villumsen" Date: Thu, 18 Aug 2022 11:04:42 +0200 Subject: [PATCH 32/47] BITMAG-1142 Show human-readable age of oldest checksum --- .../common/utils/SettingsUtils.java | 16 +++++++----- .../bitrepository/common/utils/TimeUtils.java | 16 ++++++++++++ .../cache/database/IntegrityDAO.java | 26 ++----------------- 3 files changed, 28 insertions(+), 30 deletions(-) diff --git a/bitrepository-core/src/main/java/org/bitrepository/common/utils/SettingsUtils.java b/bitrepository-core/src/main/java/org/bitrepository/common/utils/SettingsUtils.java index cbe5ded8d..52d8180ad 100644 --- a/bitrepository-core/src/main/java/org/bitrepository/common/utils/SettingsUtils.java +++ b/bitrepository-core/src/main/java/org/bitrepository/common/utils/SettingsUtils.java @@ -24,6 +24,7 @@ import org.bitrepository.common.settings.Settings; import org.bitrepository.settings.referencesettings.IntegrityServiceSettings; import org.bitrepository.settings.referencesettings.PillarIntegrityDetails; +import org.bitrepository.settings.referencesettings.PillarSettings; import org.bitrepository.settings.referencesettings.PillarType; import org.bitrepository.settings.repositorysettings.Collection; @@ -126,20 +127,23 @@ public static IntegrityServiceSettings getIntegrityServiceSettings() { /** * Get the configured max age for checksums for the given pillarID from the ReferenceSettings. * - * @param pillarID The pillarID for which the hostname is wanted. + * @param pillarID The pillarID for which the max checksum age is wanted. * @return human-readable maximum age for checksums for the given pillar ID. */ public static String getMaxAgeForChecksums(String pillarID) { - // TODO how to use pillarID?? - BigInteger maxAge = settings.getReferenceSettings().getPillarSettings().getMaxAgeForChecksums(); - if (maxAge == null) { - return null; + PillarSettings pillarSettings = settings.getReferenceSettings().getPillarSettings(); + if (pillarSettings == null) { + return "Not set (no pillar settings)"; } + if (! pillarSettings.getPillarID().equals(pillarID)) { + return "Unknown"; + } + BigInteger maxAge = pillarSettings.getMaxAgeForChecksums(); try { return TimeUtils.millisecondsToHuman(maxAge.longValueExact()); } catch (ArithmeticException ae) { - return "Extremely long"; + return String.format(Locale.getDefault(Locale.Category.FORMAT), "Extremely long; %d ms", maxAge); } } diff --git a/bitrepository-core/src/main/java/org/bitrepository/common/utils/TimeUtils.java b/bitrepository-core/src/main/java/org/bitrepository/common/utils/TimeUtils.java index 7e176e8bf..574fd8aae 100644 --- a/bitrepository-core/src/main/java/org/bitrepository/common/utils/TimeUtils.java +++ b/bitrepository-core/src/main/java/org/bitrepository/common/utils/TimeUtils.java @@ -29,7 +29,10 @@ import java.time.Duration; import java.time.Period; import java.time.ZonedDateTime; +<<<<<<< HEAD import java.time.temporal.ChronoUnit; +======= +>>>>>>> 747f84e93 (BITMAG-1142 Show human-readable age of oldest checksum) import java.util.ArrayList; import java.util.Date; import java.util.List; @@ -141,6 +144,7 @@ public static String humanDifference(ZonedDateTime start, ZonedDateTime end) { periodBetween = Period.between(start.toLocalDate(), end.toLocalDate().minusDays(1)); afterPeriod = start.plus(periodBetween); } +<<<<<<< HEAD // Round duration to whole seconds Duration durationBetween = Duration.between(afterPeriod, end) .plusMillis(500) @@ -155,6 +159,18 @@ public static String humanDifference(ZonedDateTime start, ZonedDateTime end) { // Since in practice the text is updated a few seconds later and any minutes 1 minute later, // it is not expected to be a problem for the user. List elements = new ArrayList<>(6); +======= + Duration durationBetween = Duration.between(afterPeriod, end); + + if (periodBetween.isZero() && durationBetween.isZero()) { + return "0 ms"; + } + + // The following gives an ambiguous string like "3m" + // in the very rare cases where only months or only minutes are non-zero. + // Since in practice the text is updated a few seconds later, it is not expected to be a problem for the user. + List elements = new ArrayList<>(7); +>>>>>>> 747f84e93 (BITMAG-1142 Show human-readable age of oldest checksum) if (periodBetween.getYears() != 0) { elements.add(periodBetween.getYears() + "y"); } diff --git a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/database/IntegrityDAO.java b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/database/IntegrityDAO.java index 67e1eda9d..bcb27cf50 100644 --- a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/database/IntegrityDAO.java +++ b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/database/IntegrityDAO.java @@ -46,7 +46,8 @@ import java.sql.ResultSet; import java.sql.SQLException; import java.time.Instant; -<<<<<<< HEAD +import java.time.ZoneId; +import java.time.ZonedDateTime; import java.util.ArrayList; import java.util.Arrays; import java.util.Date; @@ -55,9 +56,7 @@ import java.util.Map; import java.util.Objects; import java.util.Optional; -======= import java.util.*; ->>>>>>> ac4d1581c (BITMAG-1142 Checksum age in GUI: underway) /** * Common parts of the implementation of the access to the integrity db. @@ -519,18 +518,12 @@ public List getLatestPillarStats(String collectionID) { String latestPillarStatsSql = "SELECT pillarID, file_count, file_size, missing_files_count," + " checksum_errors_count, missing_checksums_count, obsolete_checksums_count," + -<<<<<<< HEAD " oldest_checksum_timestamp" + " FROM pillarstats" + " WHERE stat_key = (" + " SELECT MAX(stat_key)" + " FROM stats" + " WHERE collectionID = ?)"; -======= - " 1000000000000 as oldest_checksum_timestamp" + // TODO extend table with new col - " FROM pillarstats" + - " WHERE stat_key = (" + " SELECT MAX(stat_key) FROM stats" + " WHERE collectionID = ?)"; ->>>>>>> ac4d1581c (BITMAG-1142 Checksum age in GUI: underway) try (Connection conn = dbConnector.getConnection(); PreparedStatement ps = DatabaseUtils.createPreparedStatement(conn, latestPillarStatsSql, collectionID)) { @@ -548,27 +541,12 @@ >>>>>>> ac4d1581c (BITMAG-1142 Checksum age in GUI: underway) String pillarHostname = Objects.requireNonNullElse(SettingsUtils.getPillarName(pillarID), "N/A"); String pillarType = (SettingsUtils.getPillarType(pillarID) != null) ? Objects.requireNonNull(SettingsUtils.getPillarType(pillarID)).value() : "Unknown"; -<<<<<<< HEAD String maxAgeForChecksums = getMaxAgeForChecksums(pillarID); Instant oldestChecksumTimestamp = getOldestChecksumTimestamp(dbResult); PillarCollectionStat p = new PillarCollectionStat(pillarID, collectionID, pillarName, pillarType, fileCount, dataSize, missingFiles, checksumErrors, missingChecksums, obsoleteChecksums, maxAgeForChecksums, oldestChecksumTimestamp, statsTime, updateTime); -======= - long oldestChecksumTimestamp = dbResult.getLong("oldest_checksum_timestamp"); - String ageOfOldestChecksum; - if (dbResult.wasNull()) { - ageOfOldestChecksum = "N/A"; - } else { - ageOfOldestChecksum = TimeUtils.millisecondsToHuman( - System.currentTimeMillis() - oldestChecksumTimestamp); - } - PillarCollectionStat p = new PillarCollectionStat(pillarID, collectionID, - pillarHostname, pillarType, fileCount, dataSize, - missingFiles, checksumErrors, missingChecksums, obsoleteChecksums, - "TODO", ageOfOldestChecksum, statsTime, updateTime); ->>>>>>> ac4d1581c (BITMAG-1142 Checksum age in GUI: underway) stats.add(p); } } From 7405c850f8fc488feca5ef46e81cc68155b7122a Mon Sep 17 00:00:00 2001 From: "Ole V. Villumsen" Date: Thu, 18 Aug 2022 14:13:48 +0200 Subject: [PATCH 33/47] BITMAG-1142 Display configured max age from MaxChecksumAgeProvider --- .../common/utils/SettingsUtils.java | 48 +++++++++---------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/bitrepository-core/src/main/java/org/bitrepository/common/utils/SettingsUtils.java b/bitrepository-core/src/main/java/org/bitrepository/common/utils/SettingsUtils.java index 52d8180ad..8a50b0d97 100644 --- a/bitrepository-core/src/main/java/org/bitrepository/common/utils/SettingsUtils.java +++ b/bitrepository-core/src/main/java/org/bitrepository/common/utils/SettingsUtils.java @@ -24,7 +24,6 @@ import org.bitrepository.common.settings.Settings; import org.bitrepository.settings.referencesettings.IntegrityServiceSettings; import org.bitrepository.settings.referencesettings.PillarIntegrityDetails; -import org.bitrepository.settings.referencesettings.PillarSettings; import org.bitrepository.settings.referencesettings.PillarType; import org.bitrepository.settings.repositorysettings.Collection; @@ -120,33 +119,34 @@ public static String getPillarName(String pillarID) { return null; } + // TODO Ole V. delete method +// /** +// * Get the configured max age for checksums for the given pillarID from the ReferenceSettings. +// * +// * @param pillarID The pillarID for which the max checksum age is wanted. +// * @return human-readable maximum age for checksums for the given pillar ID. +// */ +// public static String getMaxAgeForChecksums(String pillarID) { +// PillarSettings pillarSettings = settings.getReferenceSettings().getPillarSettings(); +// if (pillarSettings == null) { +// return "Not set (no pillar settings)"; +// } +// if (! pillarSettings.getPillarID().equals(pillarID)) { +// return "Unknown"; +// } +// BigInteger maxAge = pillarSettings.getMaxAgeForChecksums(); +// try { +// return TimeUtils.millisecondsToHuman(maxAge.longValueExact()); +// } +// catch (ArithmeticException ae) { +// return String.format(Locale.getDefault(Locale.Category.FORMAT), "Extremely long; %d ms", maxAge); +// } +// } + public static IntegrityServiceSettings getIntegrityServiceSettings() { return settings.getReferenceSettings().getIntegrityServiceSettings(); } - /** - * Get the configured max age for checksums for the given pillarID from the ReferenceSettings. - * - * @param pillarID The pillarID for which the max checksum age is wanted. - * @return human-readable maximum age for checksums for the given pillar ID. - */ - public static String getMaxAgeForChecksums(String pillarID) { - PillarSettings pillarSettings = settings.getReferenceSettings().getPillarSettings(); - if (pillarSettings == null) { - return "Not set (no pillar settings)"; - } - if (! pillarSettings.getPillarID().equals(pillarID)) { - return "Unknown"; - } - BigInteger maxAge = pillarSettings.getMaxAgeForChecksums(); - try { - return TimeUtils.millisecondsToHuman(maxAge.longValueExact()); - } - catch (ArithmeticException ae) { - return String.format(Locale.getDefault(Locale.Category.FORMAT), "Extremely long; %d ms", maxAge); - } - } - /** * Get the {@link PillarType} for the given Pillar ID. * From c6dc7b9b69503d4dc063237fd6d569d33bc0a194 Mon Sep 17 00:00:00 2001 From: "Ole V. Villumsen" Date: Wed, 24 Aug 2022 15:03:30 +0200 Subject: [PATCH 34/47] BITMAG-1142 Fix bugs and missing parts in display of age of oldest checksum --- .../org/bitrepository/common/utils/TimeUtils.java | 15 --------------- .../cache/database/IntegrityDAO.java | 2 -- .../web/RestIntegrityService.java | 5 ----- 3 files changed, 22 deletions(-) diff --git a/bitrepository-core/src/main/java/org/bitrepository/common/utils/TimeUtils.java b/bitrepository-core/src/main/java/org/bitrepository/common/utils/TimeUtils.java index 574fd8aae..a786a5e65 100644 --- a/bitrepository-core/src/main/java/org/bitrepository/common/utils/TimeUtils.java +++ b/bitrepository-core/src/main/java/org/bitrepository/common/utils/TimeUtils.java @@ -29,10 +29,7 @@ import java.time.Duration; import java.time.Period; import java.time.ZonedDateTime; -<<<<<<< HEAD import java.time.temporal.ChronoUnit; -======= ->>>>>>> 747f84e93 (BITMAG-1142 Show human-readable age of oldest checksum) import java.util.ArrayList; import java.util.Date; import java.util.List; @@ -144,7 +141,6 @@ public static String humanDifference(ZonedDateTime start, ZonedDateTime end) { periodBetween = Period.between(start.toLocalDate(), end.toLocalDate().minusDays(1)); afterPeriod = start.plus(periodBetween); } -<<<<<<< HEAD // Round duration to whole seconds Duration durationBetween = Duration.between(afterPeriod, end) .plusMillis(500) @@ -159,18 +155,7 @@ public static String humanDifference(ZonedDateTime start, ZonedDateTime end) { // Since in practice the text is updated a few seconds later and any minutes 1 minute later, // it is not expected to be a problem for the user. List elements = new ArrayList<>(6); -======= - Duration durationBetween = Duration.between(afterPeriod, end); - if (periodBetween.isZero() && durationBetween.isZero()) { - return "0 ms"; - } - - // The following gives an ambiguous string like "3m" - // in the very rare cases where only months or only minutes are non-zero. - // Since in practice the text is updated a few seconds later, it is not expected to be a problem for the user. - List elements = new ArrayList<>(7); ->>>>>>> 747f84e93 (BITMAG-1142 Show human-readable age of oldest checksum) if (periodBetween.getYears() != 0) { elements.add(periodBetween.getYears() + "y"); } diff --git a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/database/IntegrityDAO.java b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/database/IntegrityDAO.java index bcb27cf50..e97d23ef0 100644 --- a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/database/IntegrityDAO.java +++ b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/database/IntegrityDAO.java @@ -46,8 +46,6 @@ import java.sql.ResultSet; import java.sql.SQLException; import java.time.Instant; -import java.time.ZoneId; -import java.time.ZonedDateTime; import java.util.ArrayList; import java.util.Arrays; import java.util.Date; diff --git a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/web/RestIntegrityService.java b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/web/RestIntegrityService.java index b00d95242..4d7853450 100644 --- a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/web/RestIntegrityService.java +++ b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/web/RestIntegrityService.java @@ -264,14 +264,9 @@ public String getIntegrityStatus( String pillarName = Objects.requireNonNullElse(SettingsUtils.getPillarName(pillar), "N/A"); PillarType pillarTypeObject = SettingsUtils.getPillarType(pillar); String pillarType = pillarTypeObject != null ? pillarTypeObject.value() : null; -<<<<<<< HEAD PillarCollectionStat emptyStat = new PillarCollectionStat(pillar, collectionID, pillarName, pillarType, 0L, 0L, 0L, 0L, 0L, 0L, "", null, new Date(0), new Date(0)); -======= - PillarCollectionStat emptyStat = new PillarCollectionStat(pillar, collectionID, pillarHostname, pillarType, - 0L, 0L, 0L, 0L, 0L, 0L, "", "", new Date(0), new Date(0)); ->>>>>>> ac4d1581c (BITMAG-1142 Checksum age in GUI: underway) stats.put(pillar, emptyStat); } } From f24acd377be573e79a0aaf47117c34d96d40bb7a Mon Sep 17 00:00:00 2001 From: "Ole V. Villumsen" Date: Thu, 25 Aug 2022 13:01:50 +0200 Subject: [PATCH 35/47] BITMAG-1142 Delete method that is no longer used after change --- .../common/utils/SettingsUtils.java | 24 ------------------- 1 file changed, 24 deletions(-) diff --git a/bitrepository-core/src/main/java/org/bitrepository/common/utils/SettingsUtils.java b/bitrepository-core/src/main/java/org/bitrepository/common/utils/SettingsUtils.java index 8a50b0d97..c97eb183e 100644 --- a/bitrepository-core/src/main/java/org/bitrepository/common/utils/SettingsUtils.java +++ b/bitrepository-core/src/main/java/org/bitrepository/common/utils/SettingsUtils.java @@ -119,30 +119,6 @@ public static String getPillarName(String pillarID) { return null; } - // TODO Ole V. delete method -// /** -// * Get the configured max age for checksums for the given pillarID from the ReferenceSettings. -// * -// * @param pillarID The pillarID for which the max checksum age is wanted. -// * @return human-readable maximum age for checksums for the given pillar ID. -// */ -// public static String getMaxAgeForChecksums(String pillarID) { -// PillarSettings pillarSettings = settings.getReferenceSettings().getPillarSettings(); -// if (pillarSettings == null) { -// return "Not set (no pillar settings)"; -// } -// if (! pillarSettings.getPillarID().equals(pillarID)) { -// return "Unknown"; -// } -// BigInteger maxAge = pillarSettings.getMaxAgeForChecksums(); -// try { -// return TimeUtils.millisecondsToHuman(maxAge.longValueExact()); -// } -// catch (ArithmeticException ae) { -// return String.format(Locale.getDefault(Locale.Category.FORMAT), "Extremely long; %d ms", maxAge); -// } -// } - public static IntegrityServiceSettings getIntegrityServiceSettings() { return settings.getReferenceSettings().getIntegrityServiceSettings(); } From 4e1fa9e9a111c2873b79f59eb6fdb14b0ad774e8 Mon Sep 17 00:00:00 2001 From: "Ole V. Villumsen" Date: Thu, 25 Aug 2022 14:06:39 +0200 Subject: [PATCH 36/47] Resolved further merge conflicts and compile errors from same --- .../integrityservice/cache/PillarCollectionStat.java | 12 ++++++------ .../cache/database/IntegrityDAO.java | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/PillarCollectionStat.java b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/PillarCollectionStat.java index 1382346d1..0c006eeb2 100644 --- a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/PillarCollectionStat.java +++ b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/PillarCollectionStat.java @@ -35,7 +35,7 @@ public class PillarCollectionStat { private final String pillarID; private final String collectionID; - private final String pillarHostname; + private final String pillarName; private final String pillarType; private Long fileCount = 0L; private Long dataSize = 0L; @@ -48,10 +48,10 @@ public class PillarCollectionStat { private Date statsTime; private Date updateTime; - public PillarCollectionStat(String pillarID, String collectionID, String pillarHostname, String pillarType) { + public PillarCollectionStat(String pillarID, String collectionID, String pillarName, String pillarType) { this.pillarID = pillarID; this.collectionID = collectionID; - this.pillarHostname = pillarHostname; + this.pillarName = pillarName; this.pillarType = pillarType; } @@ -62,7 +62,7 @@ public PillarCollectionStat(String pillarID, String collectionID, String pillarN Instant oldestChecksumTimestamp, Date statsTime, Date updateTime) { this.pillarID = pillarID; this.collectionID = collectionID; - this.pillarHostname = pillarHostname; + this.pillarName = pillarName; this.pillarType = pillarType; this.fileCount = fileCount; this.dataSize = dataSize; @@ -84,8 +84,8 @@ public String getCollectionID() { return collectionID; } - public String getPillarHostname() { - return pillarHostname; + public String getPillarName() { + return pillarName; } public String getPillarType() { diff --git a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/database/IntegrityDAO.java b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/database/IntegrityDAO.java index e97d23ef0..436d78170 100644 --- a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/database/IntegrityDAO.java +++ b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/database/IntegrityDAO.java @@ -536,7 +536,7 @@ public List getLatestPillarStats(String collectionID) { Long obsoleteChecksums = dbResult.getLong("obsolete_checksums_count"); Date statsTime = null; Date updateTime = null; - String pillarHostname = Objects.requireNonNullElse(SettingsUtils.getPillarName(pillarID), "N/A"); + String pillarName = Objects.requireNonNullElse(SettingsUtils.getPillarName(pillarID), "N/A"); String pillarType = (SettingsUtils.getPillarType(pillarID) != null) ? Objects.requireNonNull(SettingsUtils.getPillarType(pillarID)).value() : "Unknown"; String maxAgeForChecksums = getMaxAgeForChecksums(pillarID); From b578b927aa6ffe5fe19dbd84db25c70a9d1cdd53 Mon Sep 17 00:00:00 2001 From: "Ole V. Villumsen" Date: Thu, 25 Aug 2022 14:47:17 +0200 Subject: [PATCH 37/47] Cross your fingers that this is the last merge conflict to be resolved --- .../src/main/webapp/integrity-service.html | 20 ++++--------------- 1 file changed, 4 insertions(+), 16 deletions(-) diff --git a/bitrepository-webclient/src/main/webapp/integrity-service.html b/bitrepository-webclient/src/main/webapp/integrity-service.html index 27ffd5ffb..9f4e4897e 100644 --- a/bitrepository-webclient/src/main/webapp/integrity-service.html +++ b/bitrepository-webclient/src/main/webapp/integrity-service.html @@ -82,22 +82,12 @@

Integrity service

Pillar Name Pillar Type Total number of files -<<<<<<< HEAD Number of missing files Number of missing checksums Number of obsolete checksums Number of inconsistent checksums -======= - Number of missing files - Number of missing checksums - Number of obsolete checksums - Number of inconsistent checksums - Configured max age of checksums - Age of oldest checksum -<<<<<<< HEAD ->>>>>>> 10b99c1d8 (BITMAG-1142 Checksum age in GUI: underway) -======= ->>>>>>> ac4d1581c (BITMAG-1142 Checksum age in GUI: underway) + Configured max age of checksums + Age of oldest checksum @@ -214,8 +204,6 @@

Modal header

}); } -<<<<<<< HEAD -======= function getPagingLimit(id, member) { var myID = id; var myMember = member; @@ -353,13 +341,13 @@

Modal header

obsoleteChecksumsCount : j[i].obsoleteChecksumsCount, checksumErrorCount : j[i].checksumErrorCount, maxAgeForChecksums : j[i].maxAgeForChecksums, - ageOfOldestChecksum : j[i].ageOfOldestChecksum}; + ageOfOldestChecksum : j[i].ageOfOldestChecksum + }; updateCells(j[i].pillarID); } }); } ->>>>>>> ac4d1581c (BITMAG-1142 Checksum age in GUI: underway) function startWorkflow() { let ID = $("#workflowSelector option:selected").val(); let url = integrityServiceUrl + '/integrity/IntegrityService/startWorkflow'; From fbc8f6458e3196c0013528c8faf63066bd5cb86c Mon Sep 17 00:00:00 2001 From: "Ole V. Villumsen" Date: Thu, 18 Aug 2022 11:04:42 +0200 Subject: [PATCH 38/47] BITMAG-1142 Show human-readable age of oldest checksum --- .../main/java/org/bitrepository/common/utils/SettingsUtils.java | 1 + .../src/main/java/org/bitrepository/common/utils/TimeUtils.java | 1 - .../test/java/org/bitrepository/common/utils/TimeUtilsTest.java | 1 + 3 files changed, 2 insertions(+), 1 deletion(-) diff --git a/bitrepository-core/src/main/java/org/bitrepository/common/utils/SettingsUtils.java b/bitrepository-core/src/main/java/org/bitrepository/common/utils/SettingsUtils.java index c97eb183e..c8c367344 100644 --- a/bitrepository-core/src/main/java/org/bitrepository/common/utils/SettingsUtils.java +++ b/bitrepository-core/src/main/java/org/bitrepository/common/utils/SettingsUtils.java @@ -24,6 +24,7 @@ import org.bitrepository.common.settings.Settings; import org.bitrepository.settings.referencesettings.IntegrityServiceSettings; import org.bitrepository.settings.referencesettings.PillarIntegrityDetails; +import org.bitrepository.settings.referencesettings.PillarSettings; import org.bitrepository.settings.referencesettings.PillarType; import org.bitrepository.settings.repositorysettings.Collection; diff --git a/bitrepository-core/src/main/java/org/bitrepository/common/utils/TimeUtils.java b/bitrepository-core/src/main/java/org/bitrepository/common/utils/TimeUtils.java index a786a5e65..7e176e8bf 100644 --- a/bitrepository-core/src/main/java/org/bitrepository/common/utils/TimeUtils.java +++ b/bitrepository-core/src/main/java/org/bitrepository/common/utils/TimeUtils.java @@ -155,7 +155,6 @@ public static String humanDifference(ZonedDateTime start, ZonedDateTime end) { // Since in practice the text is updated a few seconds later and any minutes 1 minute later, // it is not expected to be a problem for the user. List elements = new ArrayList<>(6); - if (periodBetween.getYears() != 0) { elements.add(periodBetween.getYears() + "y"); } diff --git a/bitrepository-core/src/test/java/org/bitrepository/common/utils/TimeUtilsTest.java b/bitrepository-core/src/test/java/org/bitrepository/common/utils/TimeUtilsTest.java index d0297d9de..1fc9d6d56 100644 --- a/bitrepository-core/src/test/java/org/bitrepository/common/utils/TimeUtilsTest.java +++ b/bitrepository-core/src/test/java/org/bitrepository/common/utils/TimeUtilsTest.java @@ -92,6 +92,7 @@ public void differencesPrintHumanly() { addDescription("TimeUtils.humanDifference() should return" + " similar human readable strings to those from millisecondsToHuman()"); +<<<<<<< HEAD addStep("Call humanDifference() with same time twice", "The output should be '0s'"); String zeroTimeString = TimeUtils.humanDifference(BASE, BASE); assertEquals(zeroTimeString, "0s"); From 1a30eb88e28976eb4a9b16e486939632ecc95aab Mon Sep 17 00:00:00 2001 From: "Ole V. Villumsen" Date: Tue, 30 Aug 2022 14:46:23 +0200 Subject: [PATCH 39/47] BITMAG-1142 Complete rebase and conflict resolution --- .../common/utils/SettingsUtils.java | 28 ++++++++++++++++++- .../cache/database/IntegrityDAO.java | 19 +++++++++++++ 2 files changed, 46 insertions(+), 1 deletion(-) diff --git a/bitrepository-core/src/main/java/org/bitrepository/common/utils/SettingsUtils.java b/bitrepository-core/src/main/java/org/bitrepository/common/utils/SettingsUtils.java index c8c367344..bd97c1028 100644 --- a/bitrepository-core/src/main/java/org/bitrepository/common/utils/SettingsUtils.java +++ b/bitrepository-core/src/main/java/org/bitrepository/common/utils/SettingsUtils.java @@ -24,7 +24,6 @@ import org.bitrepository.common.settings.Settings; import org.bitrepository.settings.referencesettings.IntegrityServiceSettings; import org.bitrepository.settings.referencesettings.PillarIntegrityDetails; -import org.bitrepository.settings.referencesettings.PillarSettings; import org.bitrepository.settings.referencesettings.PillarType; import org.bitrepository.settings.repositorysettings.Collection; @@ -120,6 +119,33 @@ public static String getPillarName(String pillarID) { return null; } +<<<<<<< HEAD +======= + // TODO Ole V. delete method +// /** +// * Get the configured max age for checksums for the given pillarID from the ReferenceSettings. +// * +// * @param pillarID The pillarID for which the max checksum age is wanted. +// * @return human-readable maximum age for checksums for the given pillar ID. +// */ +// public static String getMaxAgeForChecksums(String pillarID) { +// PillarSettings pillarSettings = settings.getReferenceSettings().getPillarSettings(); +// if (pillarSettings == null) { +// return "Not set (no pillar settings)"; +// } +// if (! pillarSettings.getPillarID().equals(pillarID)) { +// return "Unknown"; +// } +// BigInteger maxAge = pillarSettings.getMaxAgeForChecksums(); +// try { +// return TimeUtils.millisecondsToHuman(maxAge.longValueExact()); +// } +// catch (ArithmeticException ae) { +// return String.format(Locale.getDefault(Locale.Category.FORMAT), "Extremely long; %d ms", maxAge); +// } +// } + +>>>>>>> 6c9197535 (BITMAG-1142 Display configured max age from MaxChecksumAgeProvider) public static IntegrityServiceSettings getIntegrityServiceSettings() { return settings.getReferenceSettings().getIntegrityServiceSettings(); } diff --git a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/database/IntegrityDAO.java b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/database/IntegrityDAO.java index 436d78170..40aca7fa9 100644 --- a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/database/IntegrityDAO.java +++ b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/database/IntegrityDAO.java @@ -540,7 +540,11 @@ public List getLatestPillarStats(String collectionID) { String pillarType = (SettingsUtils.getPillarType(pillarID) != null) ? Objects.requireNonNull(SettingsUtils.getPillarType(pillarID)).value() : "Unknown"; String maxAgeForChecksums = getMaxAgeForChecksums(pillarID); +<<<<<<< HEAD Instant oldestChecksumTimestamp = getOldestChecksumTimestamp(dbResult); +======= + String ageOfOldestChecksum = getAgeOfOldestChecksum(dbResult); +>>>>>>> 6c9197535 (BITMAG-1142 Display configured max age from MaxChecksumAgeProvider) PillarCollectionStat p = new PillarCollectionStat(pillarID, collectionID, pillarName, pillarType, fileCount, dataSize, missingFiles, checksumErrors, missingChecksums, obsoleteChecksums, @@ -568,9 +572,24 @@ private String getMaxAgeForChecksums(String pillarID) { return maxAge == 0 ? "unlimited" : TimeUtils.millisecondsToHuman(maxAge); } +<<<<<<< HEAD private Instant getOldestChecksumTimestamp(ResultSet dbResult) throws SQLException { long oldestChecksumTimestamp = dbResult.getLong("oldest_checksum_timestamp"); return dbResult.wasNull() ? null : Instant.ofEpochMilli(oldestChecksumTimestamp); +======= + @NotNull + private String getAgeOfOldestChecksum(ResultSet dbResult) throws SQLException { + long oldestChecksumTimestamp = dbResult.getLong("oldest_checksum_timestamp"); + String ageOfOldestChecksum; + if (dbResult.wasNull()) { + ageOfOldestChecksum = "N/A"; + } else { + ZoneId zone = ZoneId.systemDefault(); + ZonedDateTime oldestChecksumZdt = Instant.ofEpochMilli(oldestChecksumTimestamp).atZone(zone); + ageOfOldestChecksum = TimeUtils.humanDifference(oldestChecksumZdt, ZonedDateTime.now(zone)); + } + return ageOfOldestChecksum; +>>>>>>> 6c9197535 (BITMAG-1142 Display configured max age from MaxChecksumAgeProvider) } /** From 7be63be2be48e5b92e84343c24a579bc01cb325a Mon Sep 17 00:00:00 2001 From: "Ole V. Villumsen" Date: Tue, 30 Aug 2022 15:02:53 +0200 Subject: [PATCH 40/47] BITMAG-1142 Solve more merge conflicts --- .../common/utils/SettingsUtils.java | 27 ------------------- .../common/utils/TimeUtilsTest.java | 1 - .../cache/database/IntegrityDAO.java | 19 ------------- 3 files changed, 47 deletions(-) diff --git a/bitrepository-core/src/main/java/org/bitrepository/common/utils/SettingsUtils.java b/bitrepository-core/src/main/java/org/bitrepository/common/utils/SettingsUtils.java index bd97c1028..c97eb183e 100644 --- a/bitrepository-core/src/main/java/org/bitrepository/common/utils/SettingsUtils.java +++ b/bitrepository-core/src/main/java/org/bitrepository/common/utils/SettingsUtils.java @@ -119,33 +119,6 @@ public static String getPillarName(String pillarID) { return null; } -<<<<<<< HEAD -======= - // TODO Ole V. delete method -// /** -// * Get the configured max age for checksums for the given pillarID from the ReferenceSettings. -// * -// * @param pillarID The pillarID for which the max checksum age is wanted. -// * @return human-readable maximum age for checksums for the given pillar ID. -// */ -// public static String getMaxAgeForChecksums(String pillarID) { -// PillarSettings pillarSettings = settings.getReferenceSettings().getPillarSettings(); -// if (pillarSettings == null) { -// return "Not set (no pillar settings)"; -// } -// if (! pillarSettings.getPillarID().equals(pillarID)) { -// return "Unknown"; -// } -// BigInteger maxAge = pillarSettings.getMaxAgeForChecksums(); -// try { -// return TimeUtils.millisecondsToHuman(maxAge.longValueExact()); -// } -// catch (ArithmeticException ae) { -// return String.format(Locale.getDefault(Locale.Category.FORMAT), "Extremely long; %d ms", maxAge); -// } -// } - ->>>>>>> 6c9197535 (BITMAG-1142 Display configured max age from MaxChecksumAgeProvider) public static IntegrityServiceSettings getIntegrityServiceSettings() { return settings.getReferenceSettings().getIntegrityServiceSettings(); } diff --git a/bitrepository-core/src/test/java/org/bitrepository/common/utils/TimeUtilsTest.java b/bitrepository-core/src/test/java/org/bitrepository/common/utils/TimeUtilsTest.java index 1fc9d6d56..d0297d9de 100644 --- a/bitrepository-core/src/test/java/org/bitrepository/common/utils/TimeUtilsTest.java +++ b/bitrepository-core/src/test/java/org/bitrepository/common/utils/TimeUtilsTest.java @@ -92,7 +92,6 @@ public void differencesPrintHumanly() { addDescription("TimeUtils.humanDifference() should return" + " similar human readable strings to those from millisecondsToHuman()"); -<<<<<<< HEAD addStep("Call humanDifference() with same time twice", "The output should be '0s'"); String zeroTimeString = TimeUtils.humanDifference(BASE, BASE); assertEquals(zeroTimeString, "0s"); diff --git a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/database/IntegrityDAO.java b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/database/IntegrityDAO.java index 40aca7fa9..436d78170 100644 --- a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/database/IntegrityDAO.java +++ b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/database/IntegrityDAO.java @@ -540,11 +540,7 @@ public List getLatestPillarStats(String collectionID) { String pillarType = (SettingsUtils.getPillarType(pillarID) != null) ? Objects.requireNonNull(SettingsUtils.getPillarType(pillarID)).value() : "Unknown"; String maxAgeForChecksums = getMaxAgeForChecksums(pillarID); -<<<<<<< HEAD Instant oldestChecksumTimestamp = getOldestChecksumTimestamp(dbResult); -======= - String ageOfOldestChecksum = getAgeOfOldestChecksum(dbResult); ->>>>>>> 6c9197535 (BITMAG-1142 Display configured max age from MaxChecksumAgeProvider) PillarCollectionStat p = new PillarCollectionStat(pillarID, collectionID, pillarName, pillarType, fileCount, dataSize, missingFiles, checksumErrors, missingChecksums, obsoleteChecksums, @@ -572,24 +568,9 @@ private String getMaxAgeForChecksums(String pillarID) { return maxAge == 0 ? "unlimited" : TimeUtils.millisecondsToHuman(maxAge); } -<<<<<<< HEAD private Instant getOldestChecksumTimestamp(ResultSet dbResult) throws SQLException { long oldestChecksumTimestamp = dbResult.getLong("oldest_checksum_timestamp"); return dbResult.wasNull() ? null : Instant.ofEpochMilli(oldestChecksumTimestamp); -======= - @NotNull - private String getAgeOfOldestChecksum(ResultSet dbResult) throws SQLException { - long oldestChecksumTimestamp = dbResult.getLong("oldest_checksum_timestamp"); - String ageOfOldestChecksum; - if (dbResult.wasNull()) { - ageOfOldestChecksum = "N/A"; - } else { - ZoneId zone = ZoneId.systemDefault(); - ZonedDateTime oldestChecksumZdt = Instant.ofEpochMilli(oldestChecksumTimestamp).atZone(zone); - ageOfOldestChecksum = TimeUtils.humanDifference(oldestChecksumZdt, ZonedDateTime.now(zone)); - } - return ageOfOldestChecksum; ->>>>>>> 6c9197535 (BITMAG-1142 Display configured max age from MaxChecksumAgeProvider) } /** From 1f82fd0e66fb62d2b43b4087ae0dd86cce339b2b Mon Sep 17 00:00:00 2001 From: "Ole V. Villumsen" Date: Tue, 30 Aug 2022 16:00:16 +0200 Subject: [PATCH 41/47] =?UTF-8?q?BITMAG-1142=20Don=E2=80=99t=20print=20ove?= =?UTF-8?q?rly=20precise=20checksum=20ages?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../bitrepository/common/utils/TimeUtils.java | 34 +++++++----- .../common/utils/TimeUtilsTest.java | 52 ++++++++++++++----- 2 files changed, 60 insertions(+), 26 deletions(-) diff --git a/bitrepository-core/src/main/java/org/bitrepository/common/utils/TimeUtils.java b/bitrepository-core/src/main/java/org/bitrepository/common/utils/TimeUtils.java index 7e176e8bf..ff5b28ab1 100644 --- a/bitrepository-core/src/main/java/org/bitrepository/common/utils/TimeUtils.java +++ b/bitrepository-core/src/main/java/org/bitrepository/common/utils/TimeUtils.java @@ -131,6 +131,13 @@ public static String millisecondsToHuman(long ms) { return sb.toString(); } + /** + * Generate a human-readable difference between start and end like "5y 2m 23d" or "7d 23m". + * + * Include years, months and days if they are non-zero. Include hours if months are 6 or less. + * Include minutes if days are 8 or less. Never include seconds. + * This generally gives the user a precision of 0.5 % of the difference or finer. + */ public static String humanDifference(ZonedDateTime start, ZonedDateTime end) { ArgumentValidator.checkTrue(! end.isBefore(start), start + " > " + end); @@ -141,19 +148,23 @@ public static String humanDifference(ZonedDateTime start, ZonedDateTime end) { periodBetween = Period.between(start.toLocalDate(), end.toLocalDate().minusDays(1)); afterPeriod = start.plus(periodBetween); } - // Round duration to whole seconds + // Round duration to whole minutes Duration durationBetween = Duration.between(afterPeriod, end) - .plusMillis(500) - .truncatedTo(ChronoUnit.SECONDS); + .plusSeconds(30) + .truncatedTo(ChronoUnit.MINUTES); if (periodBetween.isZero() && durationBetween.isZero()) { - return "0s"; + return "0m"; } - // The following gives an ambiguous string like "3m" or "3y 11m 5s" - // in the very rare cases where months *or* minutes are non-zero and days and hours are zero. - // Since in practice the text is updated a few seconds later and any minutes 1 minute later, - // it is not expected to be a problem for the user. + boolean includeHours = periodBetween.getYears() == 0 && periodBetween.getMonths() <= 6; + boolean includeMinutes = periodBetween.getYears() == 0 + && periodBetween.getMonths() == 0 + && periodBetween.getDays() <= 8; + + // The following gives an ambiguous string like "3m" + // in the very rare cases where months or minutes are non-zero and days and hours are zero. + // It is not expected to be a problem for the user in practice. List elements = new ArrayList<>(6); if (periodBetween.getYears() != 0) { elements.add(periodBetween.getYears() + "y"); @@ -164,15 +175,12 @@ public static String humanDifference(ZonedDateTime start, ZonedDateTime end) { if (periodBetween.getDays() != 0) { elements.add(periodBetween.getDays() + "d"); } - if (durationBetween.toHours() != 0) { + if (includeHours && durationBetween.toHours() != 0) { elements.add(durationBetween.toHours() + "h"); } - if (durationBetween.toMinutesPart() != 0) { + if (includeMinutes && durationBetween.toMinutesPart() != 0) { elements.add(durationBetween.toMinutesPart() + "m"); } - if (durationBetween.toSecondsPart() != 0) { - elements.add(durationBetween.toSecondsPart() + "s"); - } return String.join(" ", elements); } diff --git a/bitrepository-core/src/test/java/org/bitrepository/common/utils/TimeUtilsTest.java b/bitrepository-core/src/test/java/org/bitrepository/common/utils/TimeUtilsTest.java index d0297d9de..4fe2a4617 100644 --- a/bitrepository-core/src/test/java/org/bitrepository/common/utils/TimeUtilsTest.java +++ b/bitrepository-core/src/test/java/org/bitrepository/common/utils/TimeUtilsTest.java @@ -92,21 +92,17 @@ public void differencesPrintHumanly() { addDescription("TimeUtils.humanDifference() should return" + " similar human readable strings to those from millisecondsToHuman()"); - addStep("Call humanDifference() with same time twice", "The output should be '0s'"); + addStep("Call humanDifference() with same time twice", "The output should be '0m'"); String zeroTimeString = TimeUtils.humanDifference(BASE, BASE); - assertEquals(zeroTimeString, "0s"); + assertEquals(zeroTimeString, "0m"); addStep("Call humanDifference() with a difference obtained from a Duration", "Expect corresponding readable output"); - // Don’t print fraction of second - testHumanDifference("0s", Duration.ofNanos(1)); - testHumanDifference("0s", Duration.ofMillis(1)); - testHumanDifference("0s", Duration.ofNanos(499_999_999)); - testHumanDifference("1s", Duration.ofNanos(500_000_000)); - testHumanDifference("1s", Duration.ofSeconds(1)); + // Don’t print seconds + testHumanDifference("0m", Duration.ofSeconds(1)); testHumanDifference("1m", Duration.ofMinutes(1)); testHumanDifference("1h", Duration.ofHours(1)); - testHumanDifference("2h 3m 5s", Duration.parse("PT2H3M5.000000007S")); + testHumanDifference("2h 3m", Duration.parse("PT2H3M5.000000007S")); addStep("Call humanDifference() with a difference obtained from a Period", "Expect corresponding readable output"); @@ -117,8 +113,11 @@ public void differencesPrintHumanly() { addStep("Call humanDifference() with a difference obtained from a combo of a Period and a Duration", "Expect corresponding readable output"); - testHumanDifference("3y 5m 7d 11h 13m 17s", + testHumanDifference("3y 5m 7d", Period.of(3, 5, 7), Duration.parse("PT11H13M17.023S")); + testHumanDifference("2m 7d 11h", + Period.of(0, 2, 7), Duration.parse("PT11H13M17.023S")); + testHumanDifference("1d 11h 13m", Period.ofDays(1), Duration.parse("PT11H13M17.023S")); addStep("Call humanDifference()" + " with dates that are 2 days apart but times that cause the diff to be less than 2 full days", @@ -128,8 +127,35 @@ public void differencesPrintHumanly() { ZonedDateTime.of(2021, 1, 31, 12, 0, 0, 0, testZoneId), ZonedDateTime.of(2021, 2, 2, - 11, 59, 59, 0, testZoneId)); - assertEquals("1d 23h 59m 59s", oneDaySomethingString); + 11, 59, 29, 0, testZoneId)); + assertEquals(oneDaySomethingString, "1d 23h 59m"); + } + + @Test(groups = {"regressiontest"}) + public void differencesPrintsWithAppropriatePrecision() { + // Include hours if months are 6 or less. + testHumanDifference("11m", Period.ofMonths(11), Duration.ofHours(23)); + testHumanDifference("1y 1d", Period.of(1, 0, 1), Duration.ofHours(23)); + testHumanDifference("2m 1h", Period.ofMonths(2), Duration.ofHours(1)); + // Include minutes if days are 8 or less. + testHumanDifference("1y", Period.ofYears(1), Duration.ofMinutes(23)); + testHumanDifference("1m", Period.ofMonths(1), Duration.ofMinutes(23)); + testHumanDifference("27d", Period.ofDays(27), Duration.ofMinutes(23)); + testHumanDifference("2d 3m", Period.ofDays(2), Duration.ofMinutes(3)); + // Round to whole minutes + testHumanDifference("2d 3m", Period.ofDays(2), Duration.ofMinutes(2).plusSeconds(30)); + testHumanDifference("2d 3m", Period.ofDays(2), Duration.ofMinutes(3).plusSeconds(29)); + // Never include seconds. + testHumanDifference("1y", Period.ofYears(1), Duration.ofSeconds(55)); + testHumanDifference("1m", Period.ofMonths(1), Duration.ofSeconds(55)); + testHumanDifference("1d", Period.ofDays(1), Duration.ofSeconds(29)); + testHumanDifference("22h", Duration.ofHours(22).plusSeconds(29)); + testHumanDifference("4m", Duration.ofMinutes(4).plusSeconds(29)); + testHumanDifference("0m", Duration.ofSeconds(2).plusMillis(1)); + testHumanDifference("0m", Duration.ofNanos(500_000_000)); + testHumanDifference("0m", Duration.ofNanos(499_999_999)); + testHumanDifference("0m", Duration.ofMillis(1)); + testHumanDifference("0m", Duration.ofNanos(1)); } /** @@ -146,7 +172,7 @@ private void testHumanDifference(String expected, TemporalAmount... amounts) { } /* - * The test only ensures that the output format is fixed. Which timezone the the date is + * The test only ensures that the output format is fixed. Which timezone the date is * formatted to depends on the default/system timezone. At some time the use of the old java Date * api should be discontinued and the new Java Time api used instead. */ From 14f5425637f4b0809de8a29097c79ed44b50c050 Mon Sep 17 00:00:00 2001 From: "Ole V. Villumsen" Date: Wed, 31 Aug 2022 14:41:47 +0200 Subject: [PATCH 42/47] BITMAG-1142 Add forgotten postgres/integrityDB7to8migration.sql file --- .../sql/postgres/integrityDB7to8migration.sql | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 bitrepository-integrity-service/src/main/resources/sql/postgres/integrityDB7to8migration.sql diff --git a/bitrepository-integrity-service/src/main/resources/sql/postgres/integrityDB7to8migration.sql b/bitrepository-integrity-service/src/main/resources/sql/postgres/integrityDB7to8migration.sql new file mode 100644 index 000000000..ad5140807 --- /dev/null +++ b/bitrepository-integrity-service/src/main/resources/sql/postgres/integrityDB7to8migration.sql @@ -0,0 +1,29 @@ +--- +-- #%L +-- Bitrepository Integrity Client +-- %% +-- Copyright (C) 2010 - 2022 Royal Danish Library and The State Archives, Denmark +-- %% +-- This program is free software: you can redistribute it and/or modify +-- it under the terms of the GNU Lesser General Public License as +-- published by the Free Software Foundation, either version 2.1 of the +-- License, or (at your option) any later version. +-- +-- This program is distributed in the hope that it will be useful, +-- but WITHOUT ANY WARRANTY; without even the implied warranty of +-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +-- GNU General Lesser Public License for more details. +-- +-- You should have received a copy of the GNU General Lesser Public +-- License along with this program. If not, see +-- . +-- #L% +--- + +-- database version +UPDATE tableversions SET version = 8 WHERE tablename = 'integritydb'; + +-- table version +UPDATE tableversions SET version = 3 WHERE tablename = 'pillarstats'; + +ALTER TABLE pillarstats ADD COLUMN oldest_checksum_timestamp BIGINT DEFAULT NULL; From d2607672a2f25d0efc4a4da67243d86050798513 Mon Sep 17 00:00:00 2001 From: "Ole V. Villumsen" Date: Wed, 31 Aug 2022 14:46:19 +0200 Subject: [PATCH 43/47] BITMAG-1142 Fix imports, do not use .* --- .../integrityservice/cache/database/IntegrityDAO.java | 1 - .../integrityservice/cache/database/StatisticsCreator.java | 4 +++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/database/IntegrityDAO.java b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/database/IntegrityDAO.java index 436d78170..c66acb116 100644 --- a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/database/IntegrityDAO.java +++ b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/database/IntegrityDAO.java @@ -54,7 +54,6 @@ import java.util.Map; import java.util.Objects; import java.util.Optional; -import java.util.*; /** * Common parts of the implementation of the access to the integrity db. diff --git a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/database/StatisticsCreator.java b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/database/StatisticsCreator.java index c0409ee6d..d5854d648 100644 --- a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/database/StatisticsCreator.java +++ b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/database/StatisticsCreator.java @@ -28,7 +28,9 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.sql.*; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.SQLException; import java.util.Date; import java.util.List; From d4a62db9600f988826104600ab5f06e89eadd48e Mon Sep 17 00:00:00 2001 From: "Ole V. Villumsen" Date: Wed, 31 Aug 2022 15:35:21 +0200 Subject: [PATCH 44/47] BITMAG-1142 Solve further merge problems (duplicated functions, import) and correct JS indentation --- .../cache/database/StatisticsCreator.java | 1 + .../src/main/webapp/integrity-service.html | 170 ++---------------- 2 files changed, 14 insertions(+), 157 deletions(-) diff --git a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/database/StatisticsCreator.java b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/database/StatisticsCreator.java index d5854d648..0ab1e389a 100644 --- a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/database/StatisticsCreator.java +++ b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/database/StatisticsCreator.java @@ -31,6 +31,7 @@ import java.sql.Connection; import java.sql.PreparedStatement; import java.sql.SQLException; +import java.sql.Types; import java.util.Date; import java.util.List; diff --git a/bitrepository-webclient/src/main/webapp/integrity-service.html b/bitrepository-webclient/src/main/webapp/integrity-service.html index e3ad40db2..198e1b575 100644 --- a/bitrepository-webclient/src/main/webapp/integrity-service.html +++ b/bitrepository-webclient/src/main/webapp/integrity-service.html @@ -204,150 +204,6 @@

Modal header

}); } - function getPagingLimit(id, member) { - var myID = id; - var myMember = member; - return function() { - return pillars[myID][myMember]; - } - } - - function showModalPager(id, member, title, url) { - var myTitle = title; - var myUrl = url; - var myID = id; - var myMember = member; - return function() { - pager = new Pager(getPagingLimit(myID, myMember), 20, myUrl, "#modalPager", "#modalPagerBody"); - $("#modalPagerLabel").html(myTitle); - $("#modalPagerBody").html("

Loading

"); - pager.getPage(1)(); - $("#modalPagerDialog").modal('show'); - } - } - - function getCellContext(id, type) { - var context = {}; - context.url = integrityServiceUrl + "/integrity/IntegrityService/"; - if(type == "Pillar Hostname") { - context.element = id + "-pillarHostname"; - context.title = type + " on " + id; - context.member = "pillarHostname"; - } else if(type == "Pillar Type") { - context.element = id + "-pillarType"; - context.title = type + " on " + id; - context.member = "pillarType"; - } else if(type == "Total files") { - context.element = id + "-totalFileCount"; - context.title = type + " on " + id; - context.member = "totalFileCount"; - context.url += "getAllFileIDs/"; - } else if(type == "Missing files") { - context.element = id + "-missingFiles"; - context.title = type + " on " + id; - context.member = "missingFilesCount"; - context.url += "getMissingFileIDs/"; - } else if(type == "Missing checksums") { - context.element = id + "-missingChecksums"; - context.title = type + " on " + id; - context.member = "missingChecksumsCount"; - context.url += "getMissingChecksumsFileIDs/"; - } else if(type == "Obsolete checksums") { - context.element = id + "-obsoleteChecksums"; - context.title = type + " on " + id; - context.member = "obsoleteChecksumsCount"; - context.url += "getObsoleteChecksumsFileIDs/"; - } else if (type == "Inconsistent checksums") { - context.element = id + "-checksumErrors"; - context.title = type + " on " + id; - context.member = "checksumErrorCount"; - context.url += "getChecksumErrorFileIDs/"; - } else if(type == "Checksum age limit") { - context.element = id + "-maxAgeForChecksums"; - context.title = type + " on " + id; - context.member = "maxAgeForChecksums"; - } else if(type == "Oldest checksum age") { - context.element = id + "-ageOfOldestChecksum"; - context.title = type + " on " + id; - context.member = "ageOfOldestChecksum"; - } - context.url += "?pillarID=" + id + "&collectionID=" + getCollectionID(); - return context; - } - - function makePillarRow(id) { - var html = ""; - html += ""; - html += "
" + id + "
"; - html += ""; - html += ""; - html += ""; - html += ""; - html += ""; - html += ""; - html += ""; - html += ""; - html += ""; - html += ""; - return html; - } - - function updateCells(pillarID) { - updateStringCell(pillarID, "Pillar Hostname", pillars[pillarID].pillarHostname); - updateStringCell(pillarID, "Pillar Type", pillars[pillarID].pillarType); - updateIntCell(pillarID, "Total files", pillars[pillarID].totalFileCount); - updateIntCell(pillarID, "Missing files", pillars[pillarID].missingFilesCount); - updateIntCell(pillarID, "Missing checksums", pillars[pillarID].missingChecksumsCount); - updateIntCell(pillarID, "Obsolete checksums", pillars[pillarID].obsoleteChecksumsCount); - updateIntCell(pillarID, "Inconsistent checksums", pillars[pillarID].checksumErrorCount); - updateStringCell(pillarID, "Checksum age limit", pillars[pillarID].maxAgeForChecksums); - updateStringCell(pillarID, "Oldest checksum age", pillars[pillarID].ageOfOldestChecksum); - } - - function updateStringCell(id, type, cellValue) { - var context = getCellContext(id, type); - var html = "

" + cellValue + "

"; - $("#" + context.element).html(html); - } - - function updateIntCell(id, type, cellValue) { - var context = getCellContext(id, type); - if(cellValue == 0) { - var html = ""; - $("#" + context.element).html(html); - } else { - var innerElement = context.element + "-a"; - var html = "" + formatInt(cellValue) + ""; - $("#" + context.element).html(html); - $("#" + innerElement).click(showModalPager(id, context.member, context.title, context.url)); - } - } - - function getIntegrityStatus() { - var url = integrityServiceUrl + "/integrity/IntegrityService/getIntegrityStatus/?collectionID=" - + getCollectionID(); - $.getJSON(url, {}, function(j){ - var htmlTable; - for (var i = 0; i < j.length; i++) { - if(pillars[j[i].pillarID] == null) { - $("#integrity-status-table-body").append(makePillarRow(j[i].pillarID)); - } - pillars[j[i].pillarID] = { - pillarHostname : j[i].pillarHostname, - pillarType : j[i].pillarType, - totalFileCount : j[i].totalFileCount, - missingFilesCount : j[i].missingFilesCount, - missingChecksumsCount : j[i].missingChecksumsCount, - obsoleteChecksumsCount : j[i].obsoleteChecksumsCount, - checksumErrorCount : j[i].checksumErrorCount, - maxAgeForChecksums : j[i].maxAgeForChecksums, - ageOfOldestChecksum : j[i].ageOfOldestChecksum - }; - updateCells(j[i].pillarID); - } - }); - } - function startWorkflow() { let ID = $("#workflowSelector option:selected").val(); let url = integrityServiceUrl + '/integrity/IntegrityService/startWorkflow'; @@ -358,7 +214,7 @@

Modal header

} /** - * Callback method to get the value of a specific property from a pillars integrity status. + * Callback method to get the value of a specific property from a pillar’s integrity status. * The callback is intended for use by {@see TableModal}. * * NOTE: The reason for returning a callback instead of the property directly is because of the modal's nature of @@ -419,15 +275,15 @@

Modal header

context.title = type + " on " + id.toUpperCase(); context.propertyName = "checksumErrorCount"; context.url += "getChecksumErrorFileIDs"; - } else if(type == "Checksum age limit") { - context.element = id + "-maxAgeForChecksums"; - context.title = type + " on " + id.toUpperCase(); - context.propertyName = "maxAgeForChecksums"; + } else if(type === "Checksum age limit") { + context.element = id + "-maxAgeForChecksums"; + context.title = type + " on " + id.toUpperCase(); + context.propertyName = "maxAgeForChecksums"; context.url += "getMaxAgeForChecksums"; - } else if(type == "Oldest checksum age") { - context.element = id + "-ageOfOldestChecksum"; - context.title = type + " on " + id.toUpperCase(); - context.propertyName = "ageOfOldestChecksum"; + } else if(type === "Oldest checksum age") { + context.element = id + "-ageOfOldestChecksum"; + context.title = type + " on " + id.toUpperCase(); + context.propertyName = "ageOfOldestChecksum"; context.url += "getAgeOfOldestChecksum"; } context.url += "?pillarID=" + id + "&collectionID=" + getCollectionID(); @@ -462,10 +318,10 @@

Modal header

html += ""; html += ""; html += ""; - html += ""; - html += ""; - html += ""; - html += ""; + html += ""; + html += ""; + html += ""; + html += ""; return html; } From 5c715956301cc46c477bfea74e1ab9c0f5c1fc7b Mon Sep 17 00:00:00 2001 From: bohlski Date: Thu, 1 Sep 2022 12:57:50 +0200 Subject: [PATCH 45/47] Fixed last merge problem and wrote ObsoleteChecksumSettings for quickstart integrity-service so it actually shows correct maxChecksumAge for pillars --- .../integrityservice/ReferenceSettings.xml | 14 ++++++++++++++ .../src/main/webapp/integrity-service.html | 19 +++++++------------ 2 files changed, 21 insertions(+), 12 deletions(-) diff --git a/bitrepository-integration/src/main/resources/quickstart/conf/integrityservice/ReferenceSettings.xml b/bitrepository-integration/src/main/resources/quickstart/conf/integrityservice/ReferenceSettings.xml index 961b2ae32..54be9246b 100644 --- a/bitrepository-integration/src/main/resources/quickstart/conf/integrityservice/ReferenceSettings.xml +++ b/bitrepository-integration/src/main/resources/quickstart/conf/integrityservice/ReferenceSettings.xml @@ -93,6 +93,20 @@ org.apache.derby.jdbc.EmbeddedDriver jdbc:derby:conf/integrityservice/auditcontributerdb + + + checksum-pillar + 3600000000000 + + + file1-pillar + 3600000 + + + file2-pillar + 3600000 + + conf/integrityservice/reportdir diff --git a/bitrepository-webclient/src/main/webapp/integrity-service.html b/bitrepository-webclient/src/main/webapp/integrity-service.html index 198e1b575..45e45ef00 100644 --- a/bitrepository-webclient/src/main/webapp/integrity-service.html +++ b/bitrepository-webclient/src/main/webapp/integrity-service.html @@ -230,7 +230,8 @@

Modal header

function showModal(type, propertyName, title, url, pillarID) { return function () { - modal = new TableModal(type, pillarID, url, "#modalBody", getIntegrityStatusPropertyCountCallback(pillarID, propertyName), 100); + modal = new TableModal(type, pillarID, url, "#modalBody", + getIntegrityStatusPropertyCountCallback(pillarID, propertyName), 100); $("#modalLabel").html(title); $("#modalBody").html("

Loading

"); modal.getModal(1); @@ -244,12 +245,8 @@

Modal header

if (type === "Pillar Name") { context.element = id + "-pillarName"; - context.title = type + " on " + id.toUpperCase(); - context.propertyName = "pillarName"; } else if (type === "Pillar Type") { context.element = id + "-pillarType"; - context.title = type + " on " + id.toUpperCase(); - context.propertyName = "pillarType"; } else if (type === "Total files") { context.element = id + "-totalFileCount"; context.title = type + " on " + id.toUpperCase(); @@ -277,14 +274,8 @@

Modal header

context.url += "getChecksumErrorFileIDs"; } else if(type === "Checksum age limit") { context.element = id + "-maxAgeForChecksums"; - context.title = type + " on " + id.toUpperCase(); - context.propertyName = "maxAgeForChecksums"; - context.url += "getMaxAgeForChecksums"; } else if(type === "Oldest checksum age") { context.element = id + "-ageOfOldestChecksum"; - context.title = type + " on " + id.toUpperCase(); - context.propertyName = "ageOfOldestChecksum"; - context.url += "getAgeOfOldestChecksum"; } context.url += "?pillarID=" + id + "&collectionID=" + getCollectionID(); return context; @@ -348,6 +339,8 @@

Modal header

updateIntCell(pillarID, "Missing checksums", pillarIntegrityStatuses[pillarID].missingChecksumsCount); updateIntCell(pillarID, "Obsolete checksums", pillarIntegrityStatuses[pillarID].obsoleteChecksumsCount); updateIntCell(pillarID, "Inconsistent checksums", pillarIntegrityStatuses[pillarID].checksumErrorCount); + updateStringCell(pillarID, "Checksum age limit", pillarIntegrityStatuses[pillarID].maxAgeForChecksums); + updateStringCell(pillarID, "Oldest checksum age", pillarIntegrityStatuses[pillarID].ageOfOldestChecksum); } function updateStringCell(id, type, cellValue) { @@ -430,7 +423,9 @@

Modal header

missingFilesCount: json[i].missingFilesCount, missingChecksumsCount: json[i].missingChecksumsCount, obsoleteChecksumsCount: json[i].obsoleteChecksumsCount, - checksumErrorCount: json[i].checksumErrorCount + checksumErrorCount: json[i].checksumErrorCount, + maxAgeForChecksums: json[i].maxAgeForChecksums, + ageOfOldestChecksum: json[i].ageOfOldestChecksum }; updateTableBody(json[i].pillarID); updateTableHeader(); From edd1668be2d33a40915594bbfc48c2b36421e9b3 Mon Sep 17 00:00:00 2001 From: "Ole V. Villumsen" Date: Thu, 1 Sep 2022 14:38:48 +0200 Subject: [PATCH 46/47] BITMAG-1142 Print checksum max age using years and months, not only days and smaller --- .../bitrepository/common/utils/TimeUtils.java | 67 ++++++++++++++----- .../common/utils/TimeUtilsTest.java | 18 +++++ .../cache/database/IntegrityDAO.java | 3 +- 3 files changed, 69 insertions(+), 19 deletions(-) diff --git a/bitrepository-core/src/main/java/org/bitrepository/common/utils/TimeUtils.java b/bitrepository-core/src/main/java/org/bitrepository/common/utils/TimeUtils.java index ff5b28ab1..c540fbef6 100644 --- a/bitrepository-core/src/main/java/org/bitrepository/common/utils/TimeUtils.java +++ b/bitrepository-core/src/main/java/org/bitrepository/common/utils/TimeUtils.java @@ -22,6 +22,7 @@ package org.bitrepository.common.utils; import org.bitrepository.common.ArgumentValidator; +import org.jetbrains.annotations.NotNull; import javax.xml.datatype.XMLGregorianCalendar; import java.text.DateFormat; @@ -131,6 +132,31 @@ public static String millisecondsToHuman(long ms) { return sb.toString(); } + /** + * Formats a non-negative Duration to an approximate human-readable string like "1y 2m" or "3h 45m". + * The conversion uses approximate average values for the lengths of days, months and years. + * + * The duration must be non-negative and not longer than 4 382 910 hours (approximately 500 years) + * + * @throws IllegalArgumentException if dur is negative or longer than 4 382 910 hours + */ + public static String durationToHuman(Duration dur) { + ArgumentValidator.checkTrue(! dur.isNegative(), "Cannot handle a negative duration; got " + dur); + ArgumentValidator.checkTrue(dur.compareTo(Duration.ofHours(4_382_910)) <= 0, + "Duration is too long: " + dur); + + int years = Math.toIntExact(dur.dividedBy(ChronoUnit.YEARS.getDuration())); + dur = dur.minus(ChronoUnit.YEARS.getDuration().multipliedBy(years)); + int months = Math.toIntExact(dur.dividedBy(ChronoUnit.MONTHS.getDuration())); + dur = dur.minus(ChronoUnit.MONTHS.getDuration().multipliedBy(months)); + int days = Math.toIntExact(dur.dividedBy(ChronoUnit.DAYS.getDuration())); + dur = dur.minus(ChronoUnit.DAYS.getDuration().multipliedBy(days)); + + Period p = Period.of(years, months, days); + + return humanPeriodAndDuration(p, dur); + } + /** * Generate a human-readable difference between start and end like "5y 2m 23d" or "7d 23m". * @@ -148,38 +174,43 @@ public static String humanDifference(ZonedDateTime start, ZonedDateTime end) { periodBetween = Period.between(start.toLocalDate(), end.toLocalDate().minusDays(1)); afterPeriod = start.plus(periodBetween); } + Duration durationBetween = Duration.between(afterPeriod, end); + + return humanPeriodAndDuration(periodBetween, durationBetween); + } + + @NotNull + private static String humanPeriodAndDuration(Period period, Duration dur) { // Round duration to whole minutes - Duration durationBetween = Duration.between(afterPeriod, end) - .plusSeconds(30) - .truncatedTo(ChronoUnit.MINUTES); + dur = dur.plusSeconds(30).truncatedTo(ChronoUnit.MINUTES); - if (periodBetween.isZero() && durationBetween.isZero()) { + if (period.isZero() && dur.isZero()) { return "0m"; } - boolean includeHours = periodBetween.getYears() == 0 && periodBetween.getMonths() <= 6; - boolean includeMinutes = periodBetween.getYears() == 0 - && periodBetween.getMonths() == 0 - && periodBetween.getDays() <= 8; + boolean includeHours = period.getYears() == 0 && period.getMonths() <= 6; + boolean includeMinutes = period.getYears() == 0 + && period.getMonths() == 0 + && period.getDays() <= 8; // The following gives an ambiguous string like "3m" // in the very rare cases where months or minutes are non-zero and days and hours are zero. // It is not expected to be a problem for the user in practice. List elements = new ArrayList<>(6); - if (periodBetween.getYears() != 0) { - elements.add(periodBetween.getYears() + "y"); + if (period.getYears() != 0) { + elements.add(period.getYears() + "y"); } - if (periodBetween.getMonths() != 0) { - elements.add(periodBetween.getMonths() + "m"); + if (period.getMonths() != 0) { + elements.add(period.getMonths() + "m"); } - if (periodBetween.getDays() != 0) { - elements.add(periodBetween.getDays() + "d"); + if (period.getDays() != 0) { + elements.add(period.getDays() + "d"); } - if (includeHours && durationBetween.toHours() != 0) { - elements.add(durationBetween.toHours() + "h"); + if (includeHours && dur.toHours() != 0) { + elements.add(dur.toHours() + "h"); } - if (includeMinutes && durationBetween.toMinutesPart() != 0) { - elements.add(durationBetween.toMinutesPart() + "m"); + if (includeMinutes && dur.toMinutesPart() != 0) { + elements.add(dur.toMinutesPart() + "m"); } return String.join(" ", elements); diff --git a/bitrepository-core/src/test/java/org/bitrepository/common/utils/TimeUtilsTest.java b/bitrepository-core/src/test/java/org/bitrepository/common/utils/TimeUtilsTest.java index 4fe2a4617..dd66ed0f4 100644 --- a/bitrepository-core/src/test/java/org/bitrepository/common/utils/TimeUtilsTest.java +++ b/bitrepository-core/src/test/java/org/bitrepository/common/utils/TimeUtilsTest.java @@ -25,6 +25,7 @@ import org.testng.Assert; import org.testng.annotations.Test; +import javax.swing.border.TitledBorder; import java.text.DateFormat; import java.text.SimpleDateFormat; import java.time.Duration; @@ -33,6 +34,7 @@ import java.time.ZoneId; import java.time.ZoneOffset; import java.time.ZonedDateTime; +import java.time.temporal.ChronoUnit; import java.time.temporal.TemporalAmount; import java.util.Date; import java.util.Locale; @@ -79,6 +81,22 @@ public void timeTester() throws Exception { assertTrue(human.contains(expectedDays), human); } + @Test(groups = {"regressiontest"}) + public void printsHumanDuration() { + assertEquals(TimeUtils.durationToHuman(ChronoUnit.YEARS.getDuration()), "1y"); + assertEquals(TimeUtils.durationToHuman(ChronoUnit.MONTHS.getDuration()), "1m"); + assertEquals(TimeUtils.durationToHuman(ChronoUnit.DAYS.getDuration()), "1d"); + assertEquals(TimeUtils.durationToHuman(ChronoUnit.HOURS.getDuration()), "1h"); + assertEquals(TimeUtils.durationToHuman(ChronoUnit.MINUTES.getDuration()), "1m"); + // Don’t print seconds + assertEquals(TimeUtils.durationToHuman(ChronoUnit.SECONDS.getDuration()), "0m"); + assertEquals(TimeUtils.durationToHuman(Duration.parse("PT2H3M5S")), "2h 3m"); + + addStep("Test the limits of what the method handles", "0m and 500y respectively"); + assertEquals(TimeUtils.durationToHuman(Duration.ZERO), "0m"); + assertEquals(TimeUtils.durationToHuman(Duration.ofHours(4_382_910)), "500y"); + } + @Test(groups = {"regressiontest"}) public void zeroIntervalTest() throws Exception { addDescription("Verifies that a 0 ms interval is represented correctly"); diff --git a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/database/IntegrityDAO.java b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/database/IntegrityDAO.java index c66acb116..5d8f2fedf 100644 --- a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/database/IntegrityDAO.java +++ b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/database/IntegrityDAO.java @@ -45,6 +45,7 @@ import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; +import java.time.Duration; import java.time.Instant; import java.util.ArrayList; import java.util.Arrays; @@ -564,7 +565,7 @@ private String getMaxAgeForChecksums(String pillarID) { new MaxChecksumAgeProvider(HandleObsoleteChecksumsStep.DEFAULT_MAX_CHECKSUM_AGE, obsoleteChecksumSettings); long maxAge = maxChecksumAgeProvider.getMaxChecksumAge(pillarID); - return maxAge == 0 ? "unlimited" : TimeUtils.millisecondsToHuman(maxAge); + return maxAge == 0 ? "unlimited" : TimeUtils.durationToHuman(Duration.ofMillis(maxAge)); } private Instant getOldestChecksumTimestamp(ResultSet dbResult) throws SQLException { From 06e5feab3c6c410cb195a91e1083802b67573114 Mon Sep 17 00:00:00 2001 From: "Ole V. Villumsen" Date: Fri, 2 Sep 2022 16:12:12 +0200 Subject: [PATCH 47/47] Change name of method since it conflicts with a method in another branch --- .../bitrepository/common/utils/TimeUtils.java | 7 ++++--- .../common/utils/TimeUtilsTest.java | 19 +++++++++---------- .../cache/database/IntegrityDAO.java | 2 +- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/bitrepository-core/src/main/java/org/bitrepository/common/utils/TimeUtils.java b/bitrepository-core/src/main/java/org/bitrepository/common/utils/TimeUtils.java index c540fbef6..07acd23f7 100644 --- a/bitrepository-core/src/main/java/org/bitrepository/common/utils/TimeUtils.java +++ b/bitrepository-core/src/main/java/org/bitrepository/common/utils/TimeUtils.java @@ -134,13 +134,14 @@ public static String millisecondsToHuman(long ms) { /** * Formats a non-negative Duration to an approximate human-readable string like "1y 2m" or "3h 45m". - * The conversion uses approximate average values for the lengths of days, months and years. + * The conversion uses estimated/approximate average values for the lengths of days, months and years. + * The method is therefore suitable for durations longer than a month. * - * The duration must be non-negative and not longer than 4 382 910 hours (approximately 500 years) + * The duration must be non-negative and not longer than 4 382 910 hours (approximately 500 years). * * @throws IllegalArgumentException if dur is negative or longer than 4 382 910 hours */ - public static String durationToHuman(Duration dur) { + public static String durationToHumanUsingEstimates(Duration dur) { ArgumentValidator.checkTrue(! dur.isNegative(), "Cannot handle a negative duration; got " + dur); ArgumentValidator.checkTrue(dur.compareTo(Duration.ofHours(4_382_910)) <= 0, "Duration is too long: " + dur); diff --git a/bitrepository-core/src/test/java/org/bitrepository/common/utils/TimeUtilsTest.java b/bitrepository-core/src/test/java/org/bitrepository/common/utils/TimeUtilsTest.java index dd66ed0f4..346dc08e5 100644 --- a/bitrepository-core/src/test/java/org/bitrepository/common/utils/TimeUtilsTest.java +++ b/bitrepository-core/src/test/java/org/bitrepository/common/utils/TimeUtilsTest.java @@ -25,7 +25,6 @@ import org.testng.Assert; import org.testng.annotations.Test; -import javax.swing.border.TitledBorder; import java.text.DateFormat; import java.text.SimpleDateFormat; import java.time.Duration; @@ -83,18 +82,18 @@ public void timeTester() throws Exception { @Test(groups = {"regressiontest"}) public void printsHumanDuration() { - assertEquals(TimeUtils.durationToHuman(ChronoUnit.YEARS.getDuration()), "1y"); - assertEquals(TimeUtils.durationToHuman(ChronoUnit.MONTHS.getDuration()), "1m"); - assertEquals(TimeUtils.durationToHuman(ChronoUnit.DAYS.getDuration()), "1d"); - assertEquals(TimeUtils.durationToHuman(ChronoUnit.HOURS.getDuration()), "1h"); - assertEquals(TimeUtils.durationToHuman(ChronoUnit.MINUTES.getDuration()), "1m"); + assertEquals(TimeUtils.durationToHumanUsingEstimates(ChronoUnit.YEARS.getDuration()), "1y"); + assertEquals(TimeUtils.durationToHumanUsingEstimates(ChronoUnit.MONTHS.getDuration()), "1m"); + assertEquals(TimeUtils.durationToHumanUsingEstimates(ChronoUnit.DAYS.getDuration()), "1d"); + assertEquals(TimeUtils.durationToHumanUsingEstimates(ChronoUnit.HOURS.getDuration()), "1h"); + assertEquals(TimeUtils.durationToHumanUsingEstimates(ChronoUnit.MINUTES.getDuration()), "1m"); // Don’t print seconds - assertEquals(TimeUtils.durationToHuman(ChronoUnit.SECONDS.getDuration()), "0m"); - assertEquals(TimeUtils.durationToHuman(Duration.parse("PT2H3M5S")), "2h 3m"); + assertEquals(TimeUtils.durationToHumanUsingEstimates(ChronoUnit.SECONDS.getDuration()), "0m"); + assertEquals(TimeUtils.durationToHumanUsingEstimates(Duration.parse("PT2H3M5S")), "2h 3m"); addStep("Test the limits of what the method handles", "0m and 500y respectively"); - assertEquals(TimeUtils.durationToHuman(Duration.ZERO), "0m"); - assertEquals(TimeUtils.durationToHuman(Duration.ofHours(4_382_910)), "500y"); + assertEquals(TimeUtils.durationToHumanUsingEstimates(Duration.ZERO), "0m"); + assertEquals(TimeUtils.durationToHumanUsingEstimates(Duration.ofHours(4_382_910)), "500y"); } @Test(groups = {"regressiontest"}) diff --git a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/database/IntegrityDAO.java b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/database/IntegrityDAO.java index 5d8f2fedf..7ca5a1135 100644 --- a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/database/IntegrityDAO.java +++ b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/cache/database/IntegrityDAO.java @@ -565,7 +565,7 @@ private String getMaxAgeForChecksums(String pillarID) { new MaxChecksumAgeProvider(HandleObsoleteChecksumsStep.DEFAULT_MAX_CHECKSUM_AGE, obsoleteChecksumSettings); long maxAge = maxChecksumAgeProvider.getMaxChecksumAge(pillarID); - return maxAge == 0 ? "unlimited" : TimeUtils.durationToHuman(Duration.ofMillis(maxAge)); + return maxAge == 0 ? "unlimited" : TimeUtils.durationToHumanUsingEstimates(Duration.ofMillis(maxAge)); } private Instant getOldestChecksumTimestamp(ResultSet dbResult) throws SQLException {