From e907185d229ffdd5645f3c5fe5dbbcd3be0e978a Mon Sep 17 00:00:00 2001 From: bohlski Date: Tue, 21 Jun 2022 17:17:14 +0200 Subject: [PATCH 1/3] Fixed parent folder also being part of zip and refactored part of the file handling to use NIO --- .../audittrails/preserver/AuditPacker.java | 48 ++++++++----------- .../preserver/LocalAuditTrailPreserver.java | 2 +- .../bitrepository/common/utils/FileUtils.java | 18 ++++++- 3 files changed, 38 insertions(+), 30 deletions(-) diff --git a/bitrepository-audit-trail-service/src/main/java/org/bitrepository/audittrails/preserver/AuditPacker.java b/bitrepository-audit-trail-service/src/main/java/org/bitrepository/audittrails/preserver/AuditPacker.java index 01edfa8a2..f5e138c0e 100644 --- a/bitrepository-audit-trail-service/src/main/java/org/bitrepository/audittrails/preserver/AuditPacker.java +++ b/bitrepository-audit-trail-service/src/main/java/org/bitrepository/audittrails/preserver/AuditPacker.java @@ -30,12 +30,14 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.File; -import java.io.FileOutputStream; import java.io.IOException; +import java.io.OutputStream; import java.io.OutputStreamWriter; import java.io.PrintWriter; import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -54,15 +56,11 @@ public class AuditPacker { /** * The directory where the temporary files are stored. */ - private final File directory; + private final Path directory; /** * Map between the contributor id and the reached preservation sequence number. */ private final Map seqReached = new HashMap<>(); - /** - * Whether the output stream should be appended to the file. - */ - private static final boolean APPEND = true; /** * Constructor. @@ -74,7 +72,7 @@ public class AuditPacker { public AuditPacker(AuditTrailStore store, AuditTrailPreservation settings, String collectionID) { this.store = store; this.collectionID = collectionID; - this.directory = FileUtils.retrieveDirectory(settings.getAuditTrailPreservationTemporaryDirectory()); + this.directory = FileUtils.retrieveDirectory(settings.getAuditTrailPreservationTemporaryDirectory()).toPath(); this.contributors.addAll(SettingsUtils.getAuditContributorsForCollection(collectionID)); initialiseReachedSequenceNumbers(); @@ -106,22 +104,20 @@ private void initialiseReachedSequenceNumbers() { * * @return A compressed file with all the audit trails. */ - public synchronized File createNewPackage() { - File container = new File(directory, collectionID + "-audit-trails-" + System.currentTimeMillis()); + public synchronized Path createNewPackage() { + Path container = directory.resolve(collectionID + "-audit-trails-" + System.currentTimeMillis()); try { - if (container.createNewFile()) { - packContributors(container); - return createCompressedFile(container); - } + Files.createFile(container); + packContributors(container); + return createCompressedFile(container); } catch (IOException e) { throw new IllegalStateException("Cannot package the newest audit trails.", e); } finally { // cleaning up. - if (container.exists()) { - FileUtils.delete(container); + if (Files.exists(container)) { + FileUtils.delete(container.toFile()); // TODO fix with nio Path } } - return null; } /** @@ -130,18 +126,13 @@ public synchronized File createNewPackage() { * @param container The file where the audit trails should be written. * @throws IOException If writing to the file somehow fails. */ - private void packContributors(File container) throws IOException { - PrintWriter writer = null; - try { - writer = new PrintWriter(new OutputStreamWriter(new FileOutputStream(container, APPEND), StandardCharsets.UTF_8)); + private void packContributors(Path container) throws IOException { + try (OutputStream os = Files.newOutputStream(container, StandardOpenOption.APPEND); + OutputStreamWriter osw = new OutputStreamWriter(os, StandardCharsets.UTF_8); + PrintWriter writer = new PrintWriter(osw)) { for (String contributor : contributors) { packContributor(contributor, writer); } - } finally { - if (writer != null) { - writer.flush(); - writer.close(); - } } } @@ -189,8 +180,9 @@ private void packContributor(String contributorId, PrintWriter writer) { * @return The compressed file. * @throws IOException If anything goes wrong. */ - private File createCompressedFile(File fileToCompress) throws IOException { - File zippedFile = new File(directory, fileToCompress.getName() + ".zip"); + private Path createCompressedFile(Path fileToCompress) throws IOException { + Path zippedFile = directory.resolve(fileToCompress.getFileName() + ".zip"); + //FileUtils.zipFile(fileToCompress, zippedFile); FileUtils.zipFile(fileToCompress, zippedFile); return zippedFile; } diff --git a/bitrepository-audit-trail-service/src/main/java/org/bitrepository/audittrails/preserver/LocalAuditTrailPreserver.java b/bitrepository-audit-trail-service/src/main/java/org/bitrepository/audittrails/preserver/LocalAuditTrailPreserver.java index b83ab1204..03c70fb32 100644 --- a/bitrepository-audit-trail-service/src/main/java/org/bitrepository/audittrails/preserver/LocalAuditTrailPreserver.java +++ b/bitrepository-audit-trail-service/src/main/java/org/bitrepository/audittrails/preserver/LocalAuditTrailPreserver.java @@ -131,7 +131,7 @@ public void preserveRepositoryAuditTrails() { */ private synchronized void performAuditTrailPreservation(String collectionID) { try { - File auditPackage = auditPackers.get(collectionID).createNewPackage(); + File auditPackage = auditPackers.get(collectionID).createNewPackage().toFile(); URL url = uploadFile(auditPackage); log.info("Uploaded the file '" + auditPackage + "' to '" + url.toExternalForm() + "'"); diff --git a/bitrepository-core/src/main/java/org/bitrepository/common/utils/FileUtils.java b/bitrepository-core/src/main/java/org/bitrepository/common/utils/FileUtils.java index b6b762d7f..002f9e11d 100644 --- a/bitrepository-core/src/main/java/org/bitrepository/common/utils/FileUtils.java +++ b/bitrepository-core/src/main/java/org/bitrepository/common/utils/FileUtils.java @@ -33,6 +33,9 @@ import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; import java.util.Enumeration; import java.util.Objects; import java.util.zip.ZipEntry; @@ -307,7 +310,7 @@ public static void unzip(File zipFile, File toDir) throws IOException { * @param outputFile The file where the compressed content will be placed. * @throws IOException If any error occurs while writing the stream to a file */ - public static void zipFile(File inputFile, File outputFile) throws IOException { + /*public static void zipFile(File inputFile, File outputFile) throws IOException { ArgumentValidator.checkNotNull(inputFile, "File zipFile"); ArgumentValidator.checkNotNull(outputFile, "File toDir"); ArgumentValidator.checkTrue(inputFile.canRead(), "can't read '" + inputFile + "'"); @@ -326,6 +329,19 @@ public static void zipFile(File inputFile, File outputFile) throws IOException { } outStream.flush(); } + }*/ + + public static void zipFile(Path inputFile, Path outputZipFile) throws IOException { + ArgumentValidator.checkNotNull(inputFile, "Path inputFile"); + ArgumentValidator.checkNotNull(outputZipFile, "Path outputZipFile"); + ArgumentValidator.checkTrue(Files.isWritable(inputFile), "Can't read '" + inputFile + "'"); + + try (ZipOutputStream zs = new ZipOutputStream(Files.newOutputStream(outputZipFile))) { // TODO series of stream wrappers? + ZipEntry zipEntry = new ZipEntry(inputFile.toString()); + zs.putNextEntry(zipEntry); + Files.copy(inputFile, zs); + zs.closeEntry(); + } } /** From fc8d2c838cb48d8c79c5d62c2a43c8a0e2fa0624 Mon Sep 17 00:00:00 2001 From: bohlski Date: Thu, 23 Jun 2022 13:16:46 +0200 Subject: [PATCH 2/3] Small fixes on last commit --- .../audittrails/preserver/AuditPacker.java | 17 ++++----- .../preserver/LocalAuditTrailPreserver.java | 16 +++++---- .../bitrepository/common/utils/FileUtils.java | 36 +++++-------------- 3 files changed, 25 insertions(+), 44 deletions(-) diff --git a/bitrepository-audit-trail-service/src/main/java/org/bitrepository/audittrails/preserver/AuditPacker.java b/bitrepository-audit-trail-service/src/main/java/org/bitrepository/audittrails/preserver/AuditPacker.java index f5e138c0e..5a70dce2e 100644 --- a/bitrepository-audit-trail-service/src/main/java/org/bitrepository/audittrails/preserver/AuditPacker.java +++ b/bitrepository-audit-trail-service/src/main/java/org/bitrepository/audittrails/preserver/AuditPacker.java @@ -104,19 +104,16 @@ private void initialiseReachedSequenceNumbers() { * * @return A compressed file with all the audit trails. */ - public synchronized Path createNewPackage() { - Path container = directory.resolve(collectionID + "-audit-trails-" + System.currentTimeMillis()); + public synchronized Path createNewPackage() throws IOException { + Path auditTrailsFile = directory.resolve(collectionID + "-audit-trails-" + System.currentTimeMillis()); try { - Files.createFile(container); - packContributors(container); - return createCompressedFile(container); + Files.createFile(auditTrailsFile); + packContributors(auditTrailsFile); + return createCompressedFile(auditTrailsFile); } catch (IOException e) { throw new IllegalStateException("Cannot package the newest audit trails.", e); } finally { - // cleaning up. - if (Files.exists(container)) { - FileUtils.delete(container.toFile()); // TODO fix with nio Path - } + Files.deleteIfExists(auditTrailsFile); } } @@ -130,6 +127,7 @@ private void packContributors(Path container) throws IOException { try (OutputStream os = Files.newOutputStream(container, StandardOpenOption.APPEND); OutputStreamWriter osw = new OutputStreamWriter(os, StandardCharsets.UTF_8); PrintWriter writer = new PrintWriter(osw)) { + for (String contributor : contributors) { packContributor(contributor, writer); } @@ -182,7 +180,6 @@ private void packContributor(String contributorId, PrintWriter writer) { */ private Path createCompressedFile(Path fileToCompress) throws IOException { Path zippedFile = directory.resolve(fileToCompress.getFileName() + ".zip"); - //FileUtils.zipFile(fileToCompress, zippedFile); FileUtils.zipFile(fileToCompress, zippedFile); return zippedFile; } diff --git a/bitrepository-audit-trail-service/src/main/java/org/bitrepository/audittrails/preserver/LocalAuditTrailPreserver.java b/bitrepository-audit-trail-service/src/main/java/org/bitrepository/audittrails/preserver/LocalAuditTrailPreserver.java index 03c70fb32..0a9b40013 100644 --- a/bitrepository-audit-trail-service/src/main/java/org/bitrepository/audittrails/preserver/LocalAuditTrailPreserver.java +++ b/bitrepository-audit-trail-service/src/main/java/org/bitrepository/audittrails/preserver/LocalAuditTrailPreserver.java @@ -60,12 +60,12 @@ public class LocalAuditTrailPreserver implements AuditTrailPreserver { private final Logger log = LoggerFactory.getLogger(getClass()); private final AuditTrailStore store; private final BlockingPutFileClient client; - private Timer timer; - private AuditPreservationTimerTask auditTask = null; private final Map auditPackers = new HashMap<>(); private final AuditTrailPreservation preservationSettings; private final Settings settings; private final FileExchange exchange; + private Timer timer; + private AuditPreservationTimerTask auditTask = null; /** * @param settings The preservationSettings for the audit trail service. @@ -131,16 +131,18 @@ public void preserveRepositoryAuditTrails() { */ private synchronized void performAuditTrailPreservation(String collectionID) { try { - File auditPackage = auditPackers.get(collectionID).createNewPackage().toFile(); + AuditPacker collectionAuditPacker = auditPackers.get(collectionID); + File auditPackage = collectionAuditPacker.createNewPackage().toFile(); URL url = uploadFile(auditPackage); log.info("Uploaded the file '" + auditPackage + "' to '" + url.toExternalForm() + "'"); ChecksumDataForFileTYPE checksumData = getValidationChecksumDataForFile(auditPackage); - EventHandler eventHandler = new AuditPreservationEventHandler(auditPackers.get(collectionID).getSequenceNumbersReached(), store, - collectionID); - client.putFile(preservationSettings.getAuditTrailPreservationCollection(), url, auditPackage.getName(), auditPackage.length(), - checksumData, null, eventHandler, "Preservation of audit trails from the AuditTrail service."); + EventHandler eventHandler = new AuditPreservationEventHandler( + collectionAuditPacker.getSequenceNumbersReached(), store, collectionID); + client.putFile(preservationSettings.getAuditTrailPreservationCollection(), url, auditPackage.getName(), + auditPackage.length(), checksumData, null, eventHandler, + "Preservation of audit trails from the AuditTrail service."); log.debug("Cleanup of the uploaded audit trail package."); FileUtils.delete(auditPackage); diff --git a/bitrepository-core/src/main/java/org/bitrepository/common/utils/FileUtils.java b/bitrepository-core/src/main/java/org/bitrepository/common/utils/FileUtils.java index 002f9e11d..fd92a33d7 100644 --- a/bitrepository-core/src/main/java/org/bitrepository/common/utils/FileUtils.java +++ b/bitrepository-core/src/main/java/org/bitrepository/common/utils/FileUtils.java @@ -33,9 +33,9 @@ import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; +import java.io.OutputStream; import java.nio.file.Files; import java.nio.file.Path; -import java.nio.file.Paths; import java.util.Enumeration; import java.util.Objects; import java.util.zip.ZipEntry; @@ -258,6 +258,7 @@ public static void writeStreamToFile(InputStream in, File f) throws IOException } /** + * TODO remove? - only used in migration tests * Unzip a zipFile into a directory. This will create subdirectories * as needed. * @@ -306,38 +307,19 @@ public static void unzip(File zipFile, File toDir) throws IOException { /** * Zips a file. * - * @param inputFile The file to zip. - * @param outputFile The file where the compressed content will be placed. + * @param inputFile The file to zip. + * @param outputZipFile The file where the compressed content will be placed. * @throws IOException If any error occurs while writing the stream to a file */ - /*public static void zipFile(File inputFile, File outputFile) throws IOException { - ArgumentValidator.checkNotNull(inputFile, "File zipFile"); - ArgumentValidator.checkNotNull(outputFile, "File toDir"); - ArgumentValidator.checkTrue(inputFile.canRead(), "can't read '" + inputFile + "'"); - - byte[] buffer = new byte[BYTE_ARRAY_SIZE]; - int bytesRead; - - try (InputStream inputStream = new FileInputStream(inputFile); - ZipOutputStream outStream = new ZipOutputStream(new FileOutputStream(outputFile))) { - - ZipEntry entry = new ZipEntry(inputFile.getPath()); - outStream.putNextEntry(entry); - - while ((bytesRead = inputStream.read(buffer)) > 0) { - outStream.write(buffer, 0, bytesRead); - } - outStream.flush(); - } - }*/ - public static void zipFile(Path inputFile, Path outputZipFile) throws IOException { ArgumentValidator.checkNotNull(inputFile, "Path inputFile"); ArgumentValidator.checkNotNull(outputZipFile, "Path outputZipFile"); - ArgumentValidator.checkTrue(Files.isWritable(inputFile), "Can't read '" + inputFile + "'"); + ArgumentValidator.checkTrue(Files.isReadable(inputFile), "Can't read '" + inputFile + "'"); + + try (OutputStream os = Files.newOutputStream(outputZipFile); + ZipOutputStream zs = new ZipOutputStream(os)) { - try (ZipOutputStream zs = new ZipOutputStream(Files.newOutputStream(outputZipFile))) { // TODO series of stream wrappers? - ZipEntry zipEntry = new ZipEntry(inputFile.toString()); + ZipEntry zipEntry = new ZipEntry(inputFile.getFileName().toString()); zs.putNextEntry(zipEntry); Files.copy(inputFile, zs); zs.closeEntry(); From 3c890a867f9958cf5e4cb01a05b25149a023bb25 Mon Sep 17 00:00:00 2001 From: bohlski Date: Tue, 23 Aug 2022 13:22:54 +0200 Subject: [PATCH 3/3] Fixed AuditPackerTest and small logging improvement --- .../audittrails/preserver/LocalAuditTrailPreserver.java | 3 ++- .../bitrepository/audittrails/preserver/AuditPackerTest.java | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/bitrepository-audit-trail-service/src/main/java/org/bitrepository/audittrails/preserver/LocalAuditTrailPreserver.java b/bitrepository-audit-trail-service/src/main/java/org/bitrepository/audittrails/preserver/LocalAuditTrailPreserver.java index 49513969a..b4c54eaba 100644 --- a/bitrepository-audit-trail-service/src/main/java/org/bitrepository/audittrails/preserver/LocalAuditTrailPreserver.java +++ b/bitrepository-audit-trail-service/src/main/java/org/bitrepository/audittrails/preserver/LocalAuditTrailPreserver.java @@ -177,7 +177,8 @@ private synchronized void performAuditTrailPreservation(String collectionID) { preservedAuditCount += auditPacker.getPackedAuditCount(); } else { - log.info("No new audit trails to preserve. No preservation file uploaded."); + log.info("No new audit trails to preserve for collection '{}'. No preservation file uploaded.", + collectionID); } log.debug("Cleanup of the audit trail package."); diff --git a/bitrepository-audit-trail-service/src/test/java/org/bitrepository/audittrails/preserver/AuditPackerTest.java b/bitrepository-audit-trail-service/src/test/java/org/bitrepository/audittrails/preserver/AuditPackerTest.java index 6dfd5efe4..f2ec094e3 100644 --- a/bitrepository-audit-trail-service/src/test/java/org/bitrepository/audittrails/preserver/AuditPackerTest.java +++ b/bitrepository-audit-trail-service/src/test/java/org/bitrepository/audittrails/preserver/AuditPackerTest.java @@ -9,6 +9,7 @@ import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; +import java.io.IOException; import java.util.List; import java.util.Map; @@ -34,7 +35,7 @@ public void setup() { } @Test - public void testCreateNewPackage() { + public void testCreateNewPackage() throws IOException { AuditPacker packer = new AuditPacker(store, preservationSettings, collectionID); Map seqNumsReached = packer.getSequenceNumbersReached(); assertEquals(seqNumsReached.entrySet().size(), 3);