From 1b03171bc99398f9c903b316e8f5d7d6d8b96377 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mathias=20S=C3=B8by?= Date: Thu, 4 Aug 2022 10:39:35 +0200 Subject: [PATCH 1/9] Now, when the PutFileCmd performs the PutFile operation; it will simply check if a salt arg (-S) has been supplied. If this is the case, it will compute the salted hash of the provided file by pulling it from FileExchange. There should never be an issues with this, since the instance where the information is returned happens after the pillar has verified that the checksum of the provided file matches the one already present on the pillar. --- .../bitrepository/commandline/PutFileCmd.java | 87 +++++++++++++------ .../eventhandler/PutFileEventHandler.java | 16 ++-- .../modify/putfile/BlockingPutFileClient.java | 19 ++-- .../utils/ChecksumExtractionUtilsTest.java | 43 +++++---- .../common/utils/ChecksumUtils.java | 2 +- .../workflow/SaltedChecksumWorkflow.java | 9 +- 6 files changed, 98 insertions(+), 78 deletions(-) diff --git a/bitrepository-client/src/main/java/org/bitrepository/commandline/PutFileCmd.java b/bitrepository-client/src/main/java/org/bitrepository/commandline/PutFileCmd.java index 4e67b5568..5928930ac 100644 --- a/bitrepository-client/src/main/java/org/bitrepository/commandline/PutFileCmd.java +++ b/bitrepository-client/src/main/java/org/bitrepository/commandline/PutFileCmd.java @@ -24,16 +24,37 @@ import org.apache.commons.cli.Option; import org.bitrepository.bitrepositoryelements.ChecksumDataForFileTYPE; import org.bitrepository.bitrepositoryelements.ChecksumSpecTYPE; +import org.bitrepository.bitrepositoryelements.ChecksumType; import org.bitrepository.client.eventhandler.OperationEvent; import org.bitrepository.client.eventhandler.OperationEvent.OperationEventType; import org.bitrepository.commandline.eventhandler.CompleteEventAwaiter; import org.bitrepository.commandline.eventhandler.PutFileEventHandler; +import org.bitrepository.common.utils.Base16Utils; +import org.bitrepository.common.utils.ChecksumUtils; import org.bitrepository.modify.ModifyComponentFactory; import org.bitrepository.modify.putfile.PutFileClient; +import org.bitrepository.protocol.FileExchange; +import org.bitrepository.protocol.ProtocolComponentFactory; +import java.io.IOException; import java.net.URL; import static org.bitrepository.commandline.Constants.ARGUMENT_IS_NOT_REQUIRED; +import static org.bitrepository.commandline.Constants.CHECKSUM_ARG; +import static org.bitrepository.commandline.Constants.DELETE_FILE_ARG; +import static org.bitrepository.commandline.Constants.DELETE_FILE_DESC; +import static org.bitrepository.commandline.Constants.EXIT_ARGUMENT_FAILURE; +import static org.bitrepository.commandline.Constants.EXIT_OPERATION_FAILURE; +import static org.bitrepository.commandline.Constants.EXIT_SUCCESS; +import static org.bitrepository.commandline.Constants.FILE_ARG; +import static org.bitrepository.commandline.Constants.FILE_ID_ARG; +import static org.bitrepository.commandline.Constants.HAS_ARGUMENT; +import static org.bitrepository.commandline.Constants.NO_ARGUMENT; +import static org.bitrepository.commandline.Constants.REQUEST_CHECKSUM_SALT_ARG; +import static org.bitrepository.commandline.Constants.REQUEST_CHECKSUM_SALT_DESC; +import static org.bitrepository.commandline.Constants.REQUEST_CHECKSUM_TYPE_ARG; +import static org.bitrepository.commandline.Constants.REQUEST_CHECKSUM_TYPE_DESC; +import static org.bitrepository.commandline.Constants.URL_ARG; public class PutFileCmd extends CommandLineClient { private final PutFileClient client; @@ -46,10 +67,10 @@ public static void main(String[] args) { PutFileCmd client = new PutFileCmd(args); client.runCommand(); } catch (IllegalArgumentException iae) { - System.exit(Constants.EXIT_ARGUMENT_FAILURE); + System.exit(EXIT_ARGUMENT_FAILURE); } catch (Exception e) { e.printStackTrace(); - System.exit(Constants.EXIT_OPERATION_FAILURE); + System.exit(EXIT_OPERATION_FAILURE); } } @@ -82,9 +103,9 @@ public void performOperation() { OperationEvent finalEvent = putTheFile(); output.completeEvent("Results of the PutFile operation for the file '" + getFileIDForMessage() + "'", finalEvent); if (finalEvent.getEventType() == OperationEventType.COMPLETE) { - System.exit(Constants.EXIT_SUCCESS); + System.exit(EXIT_SUCCESS); } else { - System.exit(Constants.EXIT_OPERATION_FAILURE); + System.exit(EXIT_OPERATION_FAILURE); } } @@ -92,32 +113,29 @@ public void performOperation() { protected void createOptionsForCmdArgumentHandler() { super.createOptionsForCmdArgumentHandler(); - Option fileOption = new Option(Constants.FILE_ARG, Constants.HAS_ARGUMENT, - "The path to the file to be uploaded. Is required, unless a URL is given."); + Option fileOption = new Option(FILE_ARG, HAS_ARGUMENT, "The path to the file to be uploaded. Is required, unless a URL is given."); fileOption.setRequired(ARGUMENT_IS_NOT_REQUIRED); cmdHandler.addOption(fileOption); - Option urlOption = new Option(Constants.URL_ARG, Constants.HAS_ARGUMENT, + Option urlOption = new Option(URL_ARG, HAS_ARGUMENT, "The URL for the file to be uploaded. Is required, unless a local file is given."); urlOption.setRequired(ARGUMENT_IS_NOT_REQUIRED); cmdHandler.addOption(urlOption); - Option checksumOption = new Option(Constants.CHECKSUM_ARG, Constants.HAS_ARGUMENT, + Option checksumOption = new Option(CHECKSUM_ARG, HAS_ARGUMENT, "The checksum for the file to be retrieved. Is required if using a URL."); checksumOption.setRequired(ARGUMENT_IS_NOT_REQUIRED); cmdHandler.addOption(checksumOption); - Option checksumTypeOption = new Option(Constants.REQUEST_CHECKSUM_TYPE_ARG, Constants.HAS_ARGUMENT, - Constants.REQUEST_CHECKSUM_TYPE_DESC); + Option checksumTypeOption = new Option(REQUEST_CHECKSUM_TYPE_ARG, HAS_ARGUMENT, REQUEST_CHECKSUM_TYPE_DESC); checksumTypeOption.setRequired(ARGUMENT_IS_NOT_REQUIRED); cmdHandler.addOption(checksumTypeOption); - Option checksumSaltOption = new Option(Constants.REQUEST_CHECKSUM_SALT_ARG, Constants.HAS_ARGUMENT, - Constants.REQUEST_CHECKSUM_SALT_DESC); + Option checksumSaltOption = new Option(REQUEST_CHECKSUM_SALT_ARG, HAS_ARGUMENT, REQUEST_CHECKSUM_SALT_DESC); checksumSaltOption.setRequired(ARGUMENT_IS_NOT_REQUIRED); cmdHandler.addOption(checksumSaltOption); - Option deleteOption = new Option(Constants.DELETE_FILE_ARG, Constants.NO_ARGUMENT, Constants.DELETE_FILE_DESC); + Option deleteOption = new Option(DELETE_FILE_ARG, NO_ARGUMENT, DELETE_FILE_DESC); deleteOption.setRequired(ARGUMENT_IS_NOT_REQUIRED); cmdHandler.addOption(deleteOption); } @@ -130,16 +148,16 @@ protected void createOptionsForCmdArgumentHandler() { protected void validateArguments() { super.validateArguments(); - if (cmdHandler.hasOption(Constants.FILE_ARG) && cmdHandler.hasOption(Constants.URL_ARG)) { + if (cmdHandler.hasOption(FILE_ARG) && cmdHandler.hasOption(URL_ARG)) { throw new IllegalArgumentException("Cannot take both a file (-f) and a URL (-u) as argument."); } - if (!(cmdHandler.hasOption(Constants.FILE_ARG) || cmdHandler.hasOption(Constants.URL_ARG))) { + if (!(cmdHandler.hasOption(FILE_ARG) || cmdHandler.hasOption(URL_ARG))) { throw new IllegalArgumentException("Providing either file argument (-f) or URL argument (-u) is required."); } - if (cmdHandler.hasOption(Constants.URL_ARG) && !cmdHandler.hasOption(Constants.CHECKSUM_ARG)) { + if (cmdHandler.hasOption(URL_ARG) && !cmdHandler.hasOption(CHECKSUM_ARG)) { throw new IllegalArgumentException("Using URL argument (-u) requires the checksum argument (-C)."); } - if (cmdHandler.hasOption(Constants.URL_ARG) && !cmdHandler.hasOption(Constants.FILE_ID_ARG)) { + if (cmdHandler.hasOption(URL_ARG) && !cmdHandler.hasOption(FILE_ID_ARG)) { throw new IllegalArgumentException("Using URL argument (-u) requires the file ID argument (-i)."); } } @@ -158,14 +176,28 @@ private OperationEvent putTheFile() { ChecksumDataForFileTYPE validationChecksum = getValidationChecksum(); ChecksumSpecTYPE requestChecksum = getRequestChecksumSpecOrNull(); - boolean printChecksums = cmdHandler.hasOption(Constants.REQUEST_CHECKSUM_TYPE_ARG); + boolean printChecksums = cmdHandler.hasOption(REQUEST_CHECKSUM_TYPE_ARG); + String saltedChecksum = null; + if (cmdHandler.hasOption(REQUEST_CHECKSUM_SALT_ARG)) { + String saltedAlg = cmdHandler.getOptionValue(REQUEST_CHECKSUM_TYPE_ARG); + String salt = cmdHandler.getOptionValue(REQUEST_CHECKSUM_SALT_ARG); + try { + FileExchange fileexchange = ProtocolComponentFactory.getInstance().getFileExchange(settings); + ChecksumSpecTYPE checksumSpecType = new ChecksumSpecTYPE(); + checksumSpecType.setChecksumType(ChecksumType.valueOf(saltedAlg)); + checksumSpecType.setChecksumSalt(Base16Utils.encodeBase16(salt)); + saltedChecksum = ChecksumUtils.generateChecksum(fileexchange.getFile(url), checksumSpecType); + } catch (IOException e) { + throw new IllegalArgumentException("Could not convert the given URL to URI."); + } + } - CompleteEventAwaiter eventHandler = new PutFileEventHandler(settings, output, printChecksums); + CompleteEventAwaiter eventHandler = new PutFileEventHandler(settings, output, printChecksums, saltedChecksum); client.putFile(getCollectionID(), url, fileID, getSizeOfFileOrZero(), validationChecksum, requestChecksum, eventHandler, null); OperationEvent finalEvent = eventHandler.getFinish(); - if (cmdHandler.hasOption(Constants.DELETE_FILE_ARG)) { + if (cmdHandler.hasOption(DELETE_FILE_ARG)) { deleteFileAfterwards(url); } @@ -181,10 +213,10 @@ private OperationEvent putTheFile() { * @return The spec-type of the checksum as {@link ChecksumDataForFileTYPE}. */ protected ChecksumDataForFileTYPE getValidationChecksum() { - if (cmdHandler.hasOption(Constants.FILE_ARG)) { + if (cmdHandler.hasOption(FILE_ARG)) { return getValidationChecksumDataForFile(findTheFile()); } else { - return getValidationChecksumDataFromArgument(Constants.CHECKSUM_ARG); + return getValidationChecksumDataFromArgument(CHECKSUM_ARG); } } @@ -192,13 +224,12 @@ protected ChecksumDataForFileTYPE getValidationChecksum() { * @return The filename (FileID) for the file to upload. */ private String getFileIDForMessage() { - if (cmdHandler.hasOption(Constants.URL_ARG)) { - return cmdHandler.getOptionValue(Constants.URL_ARG) + " (with the id '" + cmdHandler.getOptionValue(Constants.FILE_ID_ARG) + - "')"; + if (cmdHandler.hasOption(URL_ARG)) { + return cmdHandler.getOptionValue(URL_ARG) + " (with the id '" + cmdHandler.getOptionValue(FILE_ID_ARG) + "')"; } - if (cmdHandler.hasOption(Constants.FILE_ARG)) { - return cmdHandler.getOptionValue(Constants.FILE_ARG) + (cmdHandler.hasOption(Constants.FILE_ID_ARG) ? - " (with the id '" + cmdHandler.getOptionValue(Constants.FILE_ID_ARG) + "')" : ""); + if (cmdHandler.hasOption(FILE_ARG)) { + return cmdHandler.getOptionValue(FILE_ARG) + + (cmdHandler.hasOption(FILE_ID_ARG) ? " (with the id '" + cmdHandler.getOptionValue(FILE_ID_ARG) + "')" : ""); } return "Failed"; } diff --git a/bitrepository-client/src/main/java/org/bitrepository/commandline/eventhandler/PutFileEventHandler.java b/bitrepository-client/src/main/java/org/bitrepository/commandline/eventhandler/PutFileEventHandler.java index 3afe45335..c69e95970 100644 --- a/bitrepository-client/src/main/java/org/bitrepository/commandline/eventhandler/PutFileEventHandler.java +++ b/bitrepository-client/src/main/java/org/bitrepository/commandline/eventhandler/PutFileEventHandler.java @@ -32,8 +32,8 @@ * Prints out checksum results, if any. */ public class PutFileEventHandler extends CompleteEventAwaiter { - private final Boolean printOutput; + private final String saltedChecksum; /** * Constructor. @@ -42,9 +42,10 @@ public class PutFileEventHandler extends CompleteEventAwaiter { * @param outputHandler The {@link OutputHandler} for handling output * @param printOutput Setting for determining if output should be printed. */ - public PutFileEventHandler(Settings settings, OutputHandler outputHandler, boolean printOutput) { + public PutFileEventHandler(Settings settings, OutputHandler outputHandler, boolean printOutput, String saltedChecksum) { super(settings, outputHandler); this.printOutput = printOutput; + this.saltedChecksum = saltedChecksum; if (printOutput) { output.resultHeader("PillarId \t Checksum"); @@ -54,15 +55,18 @@ public PutFileEventHandler(Settings settings, OutputHandler outputHandler, boole @Override public void handleComponentComplete(OperationEvent event) { if (!(event instanceof PutFileCompletePillarEvent)) { - output.warn("PutFileEventHandler received a component complete, which is not a " - + PutFileCompletePillarEvent.class.getName()); + output.warn("PutFileEventHandler received a component complete, which is not a " + PutFileCompletePillarEvent.class.getName()); } assert event instanceof PutFileCompletePillarEvent; PutFileCompletePillarEvent pillarEvent = (PutFileCompletePillarEvent) event; if (printOutput && pillarEvent.getChecksums() != null) { - output.resultLine( - pillarEvent.getContributorID() + " \t " + Base16Utils.decodeBase16(pillarEvent.getChecksums().getChecksumValue())); + if (saltedChecksum != null) { + output.resultLine(pillarEvent.getContributorID() + " \t " + saltedChecksum); + } else { + String checksum = Base16Utils.decodeBase16(pillarEvent.getChecksums().getChecksumValue()); + output.resultLine(pillarEvent.getContributorID() + " \t " + checksum); + } } } diff --git a/bitrepository-client/src/main/java/org/bitrepository/modify/putfile/BlockingPutFileClient.java b/bitrepository-client/src/main/java/org/bitrepository/modify/putfile/BlockingPutFileClient.java index 46549d9f4..ed43c85ad 100644 --- a/bitrepository-client/src/main/java/org/bitrepository/modify/putfile/BlockingPutFileClient.java +++ b/bitrepository-client/src/main/java/org/bitrepository/modify/putfile/BlockingPutFileClient.java @@ -63,20 +63,13 @@ public BlockingPutFileClient(PutFileClient client) { * @return The list of {@link ContributorEvent}s received during the operation * @throws OperationFailedException The operation didn't complete successfully. */ - public List putFile( - String collectionID, - URL url, - String fileID, - long sizeOfFile, - ChecksumDataForFileTYPE checksumForValidationAtPillar, - ChecksumSpecTYPE checksumRequestsForValidation, - EventHandler eventHandler, - String auditTrailInformation) - throws OperationFailedException { + public List putFile(String collectionID, URL url, String fileID, long sizeOfFile, + ChecksumDataForFileTYPE checksumForValidationAtPillar, + ChecksumSpecTYPE checksumRequestsForValidation, EventHandler eventHandler, + String auditTrailInformation) throws OperationFailedException { BlockingEventHandler blocker = new BlockingEventHandler(eventHandler); - client.putFile(collectionID, url, fileID, sizeOfFile, checksumForValidationAtPillar, - checksumRequestsForValidation, - blocker, auditTrailInformation); + client.putFile(collectionID, url, fileID, sizeOfFile, checksumForValidationAtPillar, checksumRequestsForValidation, blocker, + auditTrailInformation); OperationEvent finishEvent = blocker.awaitFinished(); if (finishEvent.getEventType() == OperationEvent.OperationEventType.COMPLETE) { return blocker.getResults(); diff --git a/bitrepository-client/src/test/java/org/bitrepository/commandline/utils/ChecksumExtractionUtilsTest.java b/bitrepository-client/src/test/java/org/bitrepository/commandline/utils/ChecksumExtractionUtilsTest.java index 019fd6b5e..e669ea875 100644 --- a/bitrepository-client/src/test/java/org/bitrepository/commandline/utils/ChecksumExtractionUtilsTest.java +++ b/bitrepository-client/src/test/java/org/bitrepository/commandline/utils/ChecksumExtractionUtilsTest.java @@ -5,16 +5,16 @@ * Copyright (C) 2010 - 2012 The State and University Library, The Royal Library and The State Archives, Denmark * %% * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as - * published by the Free Software Foundation, either version 2.1 of the + * it under the terms of the GNU Lesser General Public License as + * published by the Free Software Foundation, either version 2.1 of the * License, or (at your option) any later version. - * + * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Lesser Public License for more details. - * - * You should have received a copy of the GNU General Lesser Public + * + * You should have received a copy of the GNU General Lesser Public * License along with this program. If not, see * . * #L% @@ -45,66 +45,63 @@ public void setup() { cmdHandler.addOption(new Option(Constants.REQUEST_CHECKSUM_TYPE_ARG, Constants.HAS_ARGUMENT, "")); output = mock(OutputHandler.class); } - - @Test(groups = { "regressiontest" }) + + @Test(groups = {"regressiontest"}) public void testDefaultChecksumSpec() throws Exception { addDescription("Test that the default checksum is retrieved when no arguments are given."); - cmdHandler.parseArguments(new String[] {}); + cmdHandler.parseArguments(new String[]{}); ChecksumType type = ChecksumExtractionUtils.extractChecksumType(cmdHandler, settingsForCUT, output); assertEquals(type.name(), settingsForCUT.getRepositorySettings().getProtocolSettings().getDefaultChecksumType()); } - @Test(groups = { "regressiontest" }) + @Test(groups = {"regressiontest"}) public void testDefaultChecksumSpecWithSaltArgument() throws Exception { addDescription("Test that the HMAC version of default checksum is retrieved when the salt arguments are given."); - cmdHandler.parseArguments(new String[] {"-" + Constants.REQUEST_CHECKSUM_SALT_ARG + "0110"}); + cmdHandler.parseArguments(new String[]{"-" + Constants.REQUEST_CHECKSUM_SALT_ARG + "0110"}); ChecksumType type = ChecksumExtractionUtils.extractChecksumType(cmdHandler, settingsForCUT, output); assertEquals(type.name(), "HMAC_" + settingsForCUT.getRepositorySettings().getProtocolSettings().getDefaultChecksumType()); } - @Test(groups = { "regressiontest" }) + @Test(groups = {"regressiontest"}) public void testNonSaltChecksumSpecWithoutSaltArgument() throws Exception { addDescription("Test that a non-salt checksum type is retrieved when it is given as argument, and no salt arguments are given."); ChecksumType enteredType = ChecksumType.SHA384; - cmdHandler.parseArguments(new String[] {"-" + Constants.REQUEST_CHECKSUM_TYPE_ARG + enteredType}); + cmdHandler.parseArguments(new String[]{"-" + Constants.REQUEST_CHECKSUM_TYPE_ARG + enteredType}); ChecksumType type = ChecksumExtractionUtils.extractChecksumType(cmdHandler, settingsForCUT, output); assertEquals(type, enteredType); } - @Test(groups = { "regressiontest" }) + @Test(groups = {"regressiontest"}) public void testNonSaltChecksumSpecWithSaltArgument() throws Exception { addDescription("Test that a salt checksum type is retrieved even though a non-salt checksum algorithm it is given as argument, " + "but a salt argument also is given."); ChecksumType enteredType = ChecksumType.SHA512; - cmdHandler.parseArguments(new String[] { - "-" + Constants.REQUEST_CHECKSUM_TYPE_ARG + enteredType, - "-" + Constants.REQUEST_CHECKSUM_SALT_ARG + "0110"}); + cmdHandler.parseArguments("-" + Constants.REQUEST_CHECKSUM_TYPE_ARG + enteredType, + "-" + Constants.REQUEST_CHECKSUM_SALT_ARG + "0110"); ChecksumType type = ChecksumExtractionUtils.extractChecksumType(cmdHandler, settingsForCUT, output); assertNotEquals(type, enteredType); assertEquals(type.name(), "HMAC_" + enteredType.name()); } - @Test(groups = { "regressiontest" }) + @Test(groups = {"regressiontest"}) public void testSaltChecksumSpecWithoutSaltArgument() throws Exception { addDescription("Test that a non-salt checksum type is retrieved even though a salt checksum algorithm it is given as argument, " + "but no salt argument also is given."); ChecksumType enteredType = ChecksumType.HMAC_SHA256; - cmdHandler.parseArguments(new String[] { - "-" + Constants.REQUEST_CHECKSUM_TYPE_ARG + enteredType}); + cmdHandler.parseArguments("-" + Constants.REQUEST_CHECKSUM_TYPE_ARG + enteredType); ChecksumType type = ChecksumExtractionUtils.extractChecksumType(cmdHandler, settingsForCUT, output); assertNotEquals(type, enteredType); assertTrue(enteredType.name().contains("HMAC")); assertEquals(type.name(), enteredType.name().replace("HMAC_", "")); } - @Test(groups = { "regressiontest" }) + @Test(groups = {"regressiontest"}) public void testSaltChecksumSpecWithSaltArgument() throws Exception { addDescription("Test that a salt checksum type is retrieved when the salt checksum algorithm it is given as argument, " + "and a salt argument also is given."); ChecksumType enteredType = ChecksumType.HMAC_SHA256; - cmdHandler.parseArguments(new String[] { - "-" + Constants.REQUEST_CHECKSUM_TYPE_ARG + enteredType, - "-" + Constants.REQUEST_CHECKSUM_SALT_ARG + "0110"}); + cmdHandler.parseArguments("-" + Constants.REQUEST_CHECKSUM_TYPE_ARG + enteredType, + "-" + Constants.REQUEST_CHECKSUM_SALT_ARG + "0110"); ChecksumType type = ChecksumExtractionUtils.extractChecksumType(cmdHandler, settingsForCUT, output); assertEquals(type, enteredType); } diff --git a/bitrepository-core/src/main/java/org/bitrepository/common/utils/ChecksumUtils.java b/bitrepository-core/src/main/java/org/bitrepository/common/utils/ChecksumUtils.java index bd8c1a939..f8f17ac28 100644 --- a/bitrepository-core/src/main/java/org/bitrepository/common/utils/ChecksumUtils.java +++ b/bitrepository-core/src/main/java/org/bitrepository/common/utils/ChecksumUtils.java @@ -119,7 +119,7 @@ public static String generateChecksum(File file, ChecksumType checksumType, byte * @return The HMAC calculated checksum in hexadecimal. */ public static String generateChecksum(InputStream content, ChecksumSpecTYPE csSpec) { - byte[] digest = null; + byte[] digest; ChecksumType algorithm = csSpec.getChecksumType(); try { diff --git a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/workflow/SaltedChecksumWorkflow.java b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/workflow/SaltedChecksumWorkflow.java index acab1033f..6354a7a20 100644 --- a/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/workflow/SaltedChecksumWorkflow.java +++ b/bitrepository-integrity-service/src/main/java/org/bitrepository/integrityservice/workflow/SaltedChecksumWorkflow.java @@ -35,7 +35,6 @@ import org.slf4j.LoggerFactory; import java.util.ArrayList; -import java.util.Date; import java.util.List; import java.util.Map; import java.util.UUID; @@ -50,13 +49,9 @@ */ public class SaltedChecksumWorkflow extends Workflow { private final static Logger log = LoggerFactory.getLogger(SaltedChecksumWorkflow.class); - /** - * The context for the workflow. - */ protected IntegrityWorkflowContext context; protected String collectionID; protected IntegrityContributors integrityContributors; - protected Date workflowStart; protected WorkflowStep step = null; /** @@ -69,7 +64,7 @@ public class SaltedChecksumWorkflow extends Workflow { protected ChecksumSpecTYPE currentChecksumSpec = null; /** - * Remember to call the initialize method needs to be called before the start method. + * REMEMBER: the initialize method needs to be called before the start method. */ public SaltedChecksumWorkflow() {} @@ -85,7 +80,7 @@ public void initialise(WorkflowContext context, String collectionID) { @Override public void start() { if (context == null) { - throw new IllegalStateException("The workflow can not be started before the initialise method has been " + "called."); + throw new IllegalStateException("The workflow can not be started before the initialise method has been called."); } super.start(); From 1444db1c201383a3d00a2fed0ff49cb38cd9b174 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mathias=20S=C3=B8by?= Date: Mon, 8 Aug 2022 15:04:15 +0200 Subject: [PATCH 2/9] Created a small improvement. --- .../org/bitrepository/commandline/PutFileCmd.java | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/bitrepository-client/src/main/java/org/bitrepository/commandline/PutFileCmd.java b/bitrepository-client/src/main/java/org/bitrepository/commandline/PutFileCmd.java index 5928930ac..785594592 100644 --- a/bitrepository-client/src/main/java/org/bitrepository/commandline/PutFileCmd.java +++ b/bitrepository-client/src/main/java/org/bitrepository/commandline/PutFileCmd.java @@ -24,12 +24,10 @@ import org.apache.commons.cli.Option; import org.bitrepository.bitrepositoryelements.ChecksumDataForFileTYPE; import org.bitrepository.bitrepositoryelements.ChecksumSpecTYPE; -import org.bitrepository.bitrepositoryelements.ChecksumType; import org.bitrepository.client.eventhandler.OperationEvent; import org.bitrepository.client.eventhandler.OperationEvent.OperationEventType; import org.bitrepository.commandline.eventhandler.CompleteEventAwaiter; import org.bitrepository.commandline.eventhandler.PutFileEventHandler; -import org.bitrepository.common.utils.Base16Utils; import org.bitrepository.common.utils.ChecksumUtils; import org.bitrepository.modify.ModifyComponentFactory; import org.bitrepository.modify.putfile.PutFileClient; @@ -178,15 +176,11 @@ private OperationEvent putTheFile() { boolean printChecksums = cmdHandler.hasOption(REQUEST_CHECKSUM_TYPE_ARG); String saltedChecksum = null; - if (cmdHandler.hasOption(REQUEST_CHECKSUM_SALT_ARG)) { - String saltedAlg = cmdHandler.getOptionValue(REQUEST_CHECKSUM_TYPE_ARG); - String salt = cmdHandler.getOptionValue(REQUEST_CHECKSUM_SALT_ARG); + if (requestChecksum != null && requestChecksum.isSetChecksumSalt()) { try { - FileExchange fileexchange = ProtocolComponentFactory.getInstance().getFileExchange(settings); - ChecksumSpecTYPE checksumSpecType = new ChecksumSpecTYPE(); - checksumSpecType.setChecksumType(ChecksumType.valueOf(saltedAlg)); - checksumSpecType.setChecksumSalt(Base16Utils.encodeBase16(salt)); - saltedChecksum = ChecksumUtils.generateChecksum(fileexchange.getFile(url), checksumSpecType); + FileExchange fileExchange = ProtocolComponentFactory.getInstance().getFileExchange(settings); + saltedChecksum = ChecksumUtils.generateChecksum(fileExchange.getFile(url), requestChecksum); + // TODO: Compare checksum of URL with the cmd-input checksum? } catch (IOException e) { throw new IllegalArgumentException("Could not convert the given URL to URI."); } From 4258b150abeca7ff87e46056833b29956fb4a19c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mathias=20S=C3=B8by?= Date: Tue, 9 Aug 2022 09:59:44 +0200 Subject: [PATCH 3/9] Small change to error message. --- .../src/main/java/org/bitrepository/commandline/PutFileCmd.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bitrepository-client/src/main/java/org/bitrepository/commandline/PutFileCmd.java b/bitrepository-client/src/main/java/org/bitrepository/commandline/PutFileCmd.java index 785594592..5123be10f 100644 --- a/bitrepository-client/src/main/java/org/bitrepository/commandline/PutFileCmd.java +++ b/bitrepository-client/src/main/java/org/bitrepository/commandline/PutFileCmd.java @@ -182,7 +182,7 @@ private OperationEvent putTheFile() { saltedChecksum = ChecksumUtils.generateChecksum(fileExchange.getFile(url), requestChecksum); // TODO: Compare checksum of URL with the cmd-input checksum? } catch (IOException e) { - throw new IllegalArgumentException("Could not convert the given URL to URI."); + throw new IllegalArgumentException("Could not retrieve file from " + url); } } From b68252255c125c461ed55d6da65257674b870caa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mathias=20S=C3=B8by?= Date: Mon, 15 Aug 2022 15:09:32 +0200 Subject: [PATCH 4/9] Small change to CMD that might be used. --- .../bitrepository/commandline/PutFileCmd.java | 31 ++++++++++++++----- 1 file changed, 24 insertions(+), 7 deletions(-) diff --git a/bitrepository-client/src/main/java/org/bitrepository/commandline/PutFileCmd.java b/bitrepository-client/src/main/java/org/bitrepository/commandline/PutFileCmd.java index 5123be10f..206288a55 100644 --- a/bitrepository-client/src/main/java/org/bitrepository/commandline/PutFileCmd.java +++ b/bitrepository-client/src/main/java/org/bitrepository/commandline/PutFileCmd.java @@ -169,35 +169,52 @@ private OperationEvent putTheFile() { output.debug("Uploading the file to the FileExchange."); URL url = getURLOrUploadFile(); String fileID = retrieveFileID(); + FileExchange fileExchange = ProtocolComponentFactory.getInstance().getFileExchange(settings); output.debug("Initiating the PutFile conversation."); ChecksumDataForFileTYPE validationChecksum = getValidationChecksum(); ChecksumSpecTYPE requestChecksum = getRequestChecksumSpecOrNull(); +// if ((cmdHandler.hasOption(URL_ARG) && cmdHandler.hasOption(CHECKSUM_ARG)) && !cmdHandler.hasOption(PILLAR_ARG)) { +// String checksumFromURL = getChecksumFromURL(url, fileExchange, requestChecksum); +// String checksumFromArg = new String(validationChecksum.getChecksumValue(), StandardCharsets.UTF_8); +// if (!checksumFromURL.equals(checksumFromArg)) { +// throw new IllegalArgumentException( +// "Checksum from URL: " + checksumFromURL + " does not match checksum from argument: " + checksumFromArg); +// } +// } + boolean printChecksums = cmdHandler.hasOption(REQUEST_CHECKSUM_TYPE_ARG); String saltedChecksum = null; if (requestChecksum != null && requestChecksum.isSetChecksumSalt()) { - try { - FileExchange fileExchange = ProtocolComponentFactory.getInstance().getFileExchange(settings); - saltedChecksum = ChecksumUtils.generateChecksum(fileExchange.getFile(url), requestChecksum); - // TODO: Compare checksum of URL with the cmd-input checksum? - } catch (IOException e) { - throw new IllegalArgumentException("Could not retrieve file from " + url); - } + saltedChecksum = getChecksumFromURL(url, fileExchange, requestChecksum); } + output.debug("Performing the PutFile conversation."); CompleteEventAwaiter eventHandler = new PutFileEventHandler(settings, output, printChecksums, saltedChecksum); client.putFile(getCollectionID(), url, fileID, getSizeOfFileOrZero(), validationChecksum, requestChecksum, eventHandler, null); + output.debug("Awaiting PutFile conversation final event."); OperationEvent finalEvent = eventHandler.getFinish(); if (cmdHandler.hasOption(DELETE_FILE_ARG)) { + output.debug("Deleting file."); deleteFileAfterwards(url); } return finalEvent; } + private String getChecksumFromURL(URL url, FileExchange fileExchange, ChecksumSpecTYPE requestChecksum) { + String saltedChecksum; + try { + saltedChecksum = ChecksumUtils.generateChecksum(fileExchange.getFile(url), requestChecksum); + } catch (IOException e) { + throw new IllegalArgumentException("Could not retrieve file from " + url); + } + return saltedChecksum; + } + /** * Retrieves the Checksum of the file, used by the pillars to validate. * This checksum is either taken from the actual file, or from the checksum argument. From 780f3ba0ae41ced860e5c3f9605e7e52a554012d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mathias=20S=C3=B8by?= Date: Tue, 16 Aug 2022 11:08:57 +0200 Subject: [PATCH 5/9] Put in a fixme. Checksum pillar still stores the checksum if the URL is invalid. --- .../bitrepository/commandline/PutFileCmd.java | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/bitrepository-client/src/main/java/org/bitrepository/commandline/PutFileCmd.java b/bitrepository-client/src/main/java/org/bitrepository/commandline/PutFileCmd.java index 206288a55..fd9cb00ef 100644 --- a/bitrepository-client/src/main/java/org/bitrepository/commandline/PutFileCmd.java +++ b/bitrepository-client/src/main/java/org/bitrepository/commandline/PutFileCmd.java @@ -36,6 +36,7 @@ import java.io.IOException; import java.net.URL; +import java.nio.charset.StandardCharsets; import static org.bitrepository.commandline.Constants.ARGUMENT_IS_NOT_REQUIRED; import static org.bitrepository.commandline.Constants.CHECKSUM_ARG; @@ -48,6 +49,7 @@ import static org.bitrepository.commandline.Constants.FILE_ID_ARG; import static org.bitrepository.commandline.Constants.HAS_ARGUMENT; import static org.bitrepository.commandline.Constants.NO_ARGUMENT; +import static org.bitrepository.commandline.Constants.PILLAR_ARG; import static org.bitrepository.commandline.Constants.REQUEST_CHECKSUM_SALT_ARG; import static org.bitrepository.commandline.Constants.REQUEST_CHECKSUM_SALT_DESC; import static org.bitrepository.commandline.Constants.REQUEST_CHECKSUM_TYPE_ARG; @@ -175,14 +177,15 @@ private OperationEvent putTheFile() { ChecksumDataForFileTYPE validationChecksum = getValidationChecksum(); ChecksumSpecTYPE requestChecksum = getRequestChecksumSpecOrNull(); -// if ((cmdHandler.hasOption(URL_ARG) && cmdHandler.hasOption(CHECKSUM_ARG)) && !cmdHandler.hasOption(PILLAR_ARG)) { -// String checksumFromURL = getChecksumFromURL(url, fileExchange, requestChecksum); -// String checksumFromArg = new String(validationChecksum.getChecksumValue(), StandardCharsets.UTF_8); -// if (!checksumFromURL.equals(checksumFromArg)) { -// throw new IllegalArgumentException( -// "Checksum from URL: " + checksumFromURL + " does not match checksum from argument: " + checksumFromArg); -// } -// } + // FIXME: The below code-snippet still does not prevent the checksum pillar from storing the checksum if the URL is invalid. + if ((cmdHandler.hasOption(URL_ARG) && cmdHandler.hasOption(CHECKSUM_ARG)) && !cmdHandler.hasOption(PILLAR_ARG)) { + String checksumFromURL = getChecksumFromURL(url, fileExchange, requestChecksum); + String checksumFromArg = new String(validationChecksum.getChecksumValue(), StandardCharsets.UTF_8); + if (!checksumFromURL.equals(checksumFromArg)) { + throw new IllegalArgumentException( + "Checksum from URL: " + checksumFromURL + " does not match checksum from argument: " + checksumFromArg); + } + } boolean printChecksums = cmdHandler.hasOption(REQUEST_CHECKSUM_TYPE_ARG); String saltedChecksum = null; From 4822439065ea957d54c3600ca7c193e7a4b9706b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mathias=20S=C3=B8by?= Date: Mon, 29 Aug 2022 15:28:13 +0200 Subject: [PATCH 6/9] Will no longer store a checksum when put-file operation is performed with an invalid URL. --- .../exceptions/InvalidChecksumException.java | 46 ++++++++++++++ .../commandline/CommandLineClient.java | 29 +++++++-- .../bitrepository/commandline/PutFileCmd.java | 44 +++++++------ .../eventhandler/PutFileEventHandler.java | 22 ++++--- .../IdentifyPillarsForPutFile.java | 2 +- .../PutFileCompletePillarEvent.java | 15 +++-- .../putfile/conversation/PuttingFile.java | 4 +- .../common/utils/ChecksumUtils.java | 62 +++++++++---------- 8 files changed, 150 insertions(+), 74 deletions(-) create mode 100644 bitrepository-client/src/main/java/org/bitrepository/client/exceptions/InvalidChecksumException.java diff --git a/bitrepository-client/src/main/java/org/bitrepository/client/exceptions/InvalidChecksumException.java b/bitrepository-client/src/main/java/org/bitrepository/client/exceptions/InvalidChecksumException.java new file mode 100644 index 000000000..b2d9c9dd1 --- /dev/null +++ b/bitrepository-client/src/main/java/org/bitrepository/client/exceptions/InvalidChecksumException.java @@ -0,0 +1,46 @@ +/* + * #%L + * Bitrepository Protocol + * + * $Id$ + * $HeadURL$ + * %% + * Copyright (C) 2010 - 2011 The State and University Library, The Royal Library and The State Archives, Denmark + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as + * published by the Free Software Foundation, either version 2.1 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Lesser Public License for more details. + * + * You should have received a copy of the GNU General Lesser Public + * License along with this program. If not, see + * . + * #L% + */ +package org.bitrepository.client.exceptions; + +/** + * Used to indicate that an unexpected response has been received. + */ +@SuppressWarnings("serial") +public class InvalidChecksumException extends RuntimeException { + public InvalidChecksumException() { + } + + public InvalidChecksumException(String message) { + super(message); + } + + public InvalidChecksumException(String message, Throwable cause) { + super(message, cause); + } + + public InvalidChecksumException(Throwable cause) { + super(cause); + } +} diff --git a/bitrepository-client/src/main/java/org/bitrepository/commandline/CommandLineClient.java b/bitrepository-client/src/main/java/org/bitrepository/commandline/CommandLineClient.java index eff5dba1d..6684312e0 100644 --- a/bitrepository-client/src/main/java/org/bitrepository/commandline/CommandLineClient.java +++ b/bitrepository-client/src/main/java/org/bitrepository/commandline/CommandLineClient.java @@ -26,6 +26,7 @@ import org.apache.commons.cli.ParseException; import org.bitrepository.bitrepositoryelements.ChecksumDataForFileTYPE; import org.bitrepository.bitrepositoryelements.ChecksumSpecTYPE; +import org.bitrepository.client.exceptions.InvalidChecksumException; import org.bitrepository.commandline.output.DefaultOutputHandler; import org.bitrepository.commandline.output.OutputHandler; import org.bitrepository.commandline.utils.ChecksumExtractionUtils; @@ -44,7 +45,7 @@ import javax.jms.JMSException; import java.io.File; -import java.net.MalformedURLException; +import java.net.HttpURLConnection; import java.net.URL; import java.security.NoSuchAlgorithmException; import java.util.List; @@ -70,6 +71,9 @@ public void runCommand() throws Exception { try { try { performOperation(); + } catch (InvalidChecksumException | IllegalArgumentException ie) { + System.out.println(ie.getMessage()); + System.exit(Constants.EXIT_OPERATION_FAILURE); } catch (Exception e) { e.printStackTrace(); System.exit(Constants.EXIT_OPERATION_FAILURE); @@ -220,6 +224,13 @@ protected ChecksumSpecTYPE getRequestChecksumSpecOrDefault() { return getRequestChecksumSpec(); } + /** + * @return Returns the default ChecksumSpecType by calling {@link ChecksumUtils#getDefault(Settings)}. + */ + protected ChecksumSpecTYPE getDefaultChecksumSpec() { + return ChecksumUtils.getDefault(settings); + } + /** * @return The requested checksum spec, or null. */ @@ -322,11 +333,19 @@ protected URL getURLOrUploadFile() { FileExchange fileexchange = ProtocolComponentFactory.getInstance().getFileExchange(settings); return fileexchange.putFile(f); } else { + String urlArg = cmdHandler.getOptionValue(Constants.URL_ARG); try { - return new URL(cmdHandler.getOptionValue(Constants.URL_ARG)); - } catch (MalformedURLException e) { - throw new IllegalArgumentException( - "The URL argument is either empty or not a valid URL: " + cmdHandler.getOptionValue(Constants.URL_ARG), e); + final URL url = new URL(urlArg); + HttpURLConnection connection = (HttpURLConnection) url.openConnection(); + int responseCode = connection.getResponseCode(); + + if (responseCode > 399) { + throw new Exception("Http URL Connection ResponseCode: " + responseCode); + } + + return url; + } catch (Exception e) { + throw new IllegalArgumentException("The URL argument is either empty or not a valid URL: " + urlArg, e); } } } diff --git a/bitrepository-client/src/main/java/org/bitrepository/commandline/PutFileCmd.java b/bitrepository-client/src/main/java/org/bitrepository/commandline/PutFileCmd.java index fd9cb00ef..12f24992e 100644 --- a/bitrepository-client/src/main/java/org/bitrepository/commandline/PutFileCmd.java +++ b/bitrepository-client/src/main/java/org/bitrepository/commandline/PutFileCmd.java @@ -26,8 +26,10 @@ import org.bitrepository.bitrepositoryelements.ChecksumSpecTYPE; import org.bitrepository.client.eventhandler.OperationEvent; import org.bitrepository.client.eventhandler.OperationEvent.OperationEventType; +import org.bitrepository.client.exceptions.InvalidChecksumException; import org.bitrepository.commandline.eventhandler.CompleteEventAwaiter; import org.bitrepository.commandline.eventhandler.PutFileEventHandler; +import org.bitrepository.common.utils.Base16Utils; import org.bitrepository.common.utils.ChecksumUtils; import org.bitrepository.modify.ModifyComponentFactory; import org.bitrepository.modify.putfile.PutFileClient; @@ -36,7 +38,6 @@ import java.io.IOException; import java.net.URL; -import java.nio.charset.StandardCharsets; import static org.bitrepository.commandline.Constants.ARGUMENT_IS_NOT_REQUIRED; import static org.bitrepository.commandline.Constants.CHECKSUM_ARG; @@ -169,33 +170,30 @@ protected void validateArguments() { */ private OperationEvent putTheFile() { output.debug("Uploading the file to the FileExchange."); - URL url = getURLOrUploadFile(); + final URL url = getURLOrUploadFile(); + String fileID = retrieveFileID(); FileExchange fileExchange = ProtocolComponentFactory.getInstance().getFileExchange(settings); output.debug("Initiating the PutFile conversation."); ChecksumDataForFileTYPE validationChecksum = getValidationChecksum(); - ChecksumSpecTYPE requestChecksum = getRequestChecksumSpecOrNull(); + ChecksumSpecTYPE requestedChecksumSpec = getRequestChecksumSpecOrDefault(); - // FIXME: The below code-snippet still does not prevent the checksum pillar from storing the checksum if the URL is invalid. - if ((cmdHandler.hasOption(URL_ARG) && cmdHandler.hasOption(CHECKSUM_ARG)) && !cmdHandler.hasOption(PILLAR_ARG)) { - String checksumFromURL = getChecksumFromURL(url, fileExchange, requestChecksum); - String checksumFromArg = new String(validationChecksum.getChecksumValue(), StandardCharsets.UTF_8); + if (cmdHandler.hasOption(URL_ARG) && !cmdHandler.hasOption(PILLAR_ARG)) { + output.debug("Generating checksum from URL."); + String checksumFromURL = getChecksumFromURL(url, fileExchange, getDefaultChecksumSpec()); + String checksumFromArg = Base16Utils.decodeBase16(validationChecksum.getChecksumValue()); + output.debug("Comparing given checksum with the checksum of the file on the URL."); if (!checksumFromURL.equals(checksumFromArg)) { - throw new IllegalArgumentException( + throw new InvalidChecksumException( "Checksum from URL: " + checksumFromURL + " does not match checksum from argument: " + checksumFromArg); } } - boolean printChecksums = cmdHandler.hasOption(REQUEST_CHECKSUM_TYPE_ARG); - String saltedChecksum = null; - if (requestChecksum != null && requestChecksum.isSetChecksumSalt()) { - saltedChecksum = getChecksumFromURL(url, fileExchange, requestChecksum); - } - output.debug("Performing the PutFile conversation."); - CompleteEventAwaiter eventHandler = new PutFileEventHandler(settings, output, printChecksums, saltedChecksum); - client.putFile(getCollectionID(), url, fileID, getSizeOfFileOrZero(), validationChecksum, requestChecksum, eventHandler, null); + CompleteEventAwaiter eventHandler = new PutFileEventHandler(settings, output, requestedChecksumSpec); + client.putFile(getCollectionID(), url, fileID, getSizeOfFileOrZero(), validationChecksum, requestedChecksumSpec, eventHandler, + null); output.debug("Awaiting PutFile conversation final event."); OperationEvent finalEvent = eventHandler.getFinish(); @@ -208,14 +206,20 @@ private OperationEvent putTheFile() { return finalEvent; } - private String getChecksumFromURL(URL url, FileExchange fileExchange, ChecksumSpecTYPE requestChecksum) { - String saltedChecksum; + /** + * Generates the checksum of the file on the given URL. + * + * @param url The URL where the file can be found. + * @param fileExchange FileExchange, used convert the file to an {@link java.io.InputStream}. + * @param checksumSpecType The requested {@link ChecksumSpecTYPE}. + * @return Returns the checksum as a {@link String}. + */ + private String getChecksumFromURL(URL url, FileExchange fileExchange, ChecksumSpecTYPE checksumSpecType) { try { - saltedChecksum = ChecksumUtils.generateChecksum(fileExchange.getFile(url), requestChecksum); + return ChecksumUtils.generateChecksum(fileExchange.getFile(url), checksumSpecType); } catch (IOException e) { throw new IllegalArgumentException("Could not retrieve file from " + url); } - return saltedChecksum; } /** diff --git a/bitrepository-client/src/main/java/org/bitrepository/commandline/eventhandler/PutFileEventHandler.java b/bitrepository-client/src/main/java/org/bitrepository/commandline/eventhandler/PutFileEventHandler.java index c69e95970..6ebaad0e1 100644 --- a/bitrepository-client/src/main/java/org/bitrepository/commandline/eventhandler/PutFileEventHandler.java +++ b/bitrepository-client/src/main/java/org/bitrepository/commandline/eventhandler/PutFileEventHandler.java @@ -21,10 +21,13 @@ */ package org.bitrepository.commandline.eventhandler; +import org.bitrepository.bitrepositoryelements.ChecksumSpecTYPE; +import org.bitrepository.bitrepositoryelements.ResponseCode; import org.bitrepository.client.eventhandler.OperationEvent; import org.bitrepository.commandline.output.OutputHandler; import org.bitrepository.common.settings.Settings; import org.bitrepository.common.utils.Base16Utils; +import org.bitrepository.common.utils.ChecksumUtils; import org.bitrepository.modify.putfile.conversation.PutFileCompletePillarEvent; /** @@ -33,22 +36,21 @@ */ public class PutFileEventHandler extends CompleteEventAwaiter { private final Boolean printOutput; - private final String saltedChecksum; + private final ChecksumSpecTYPE checksumSpecTYPE; /** * Constructor. * * @param settings The {@link Settings} * @param outputHandler The {@link OutputHandler} for handling output - * @param printOutput Setting for determining if output should be printed. */ - public PutFileEventHandler(Settings settings, OutputHandler outputHandler, boolean printOutput, String saltedChecksum) { + public PutFileEventHandler(Settings settings, OutputHandler outputHandler, ChecksumSpecTYPE checksumSpecTYPE) { super(settings, outputHandler); - this.printOutput = printOutput; - this.saltedChecksum = saltedChecksum; + this.checksumSpecTYPE = checksumSpecTYPE; + this.printOutput = checksumSpecTYPE.getChecksumType() != ChecksumUtils.getDefaultChecksumType(settings); if (printOutput) { - output.resultHeader("PillarId \t Checksum"); + output.resultHeader("PillarId \t Alg \t Checksum"); } } @@ -60,14 +62,14 @@ public void handleComponentComplete(OperationEvent event) { assert event instanceof PutFileCompletePillarEvent; PutFileCompletePillarEvent pillarEvent = (PutFileCompletePillarEvent) event; + boolean isDuplicate = pillarEvent.getResponseInfo().getResponseCode().equals(ResponseCode.DUPLICATE_FILE_FAILURE); if (printOutput && pillarEvent.getChecksums() != null) { - if (saltedChecksum != null) { - output.resultLine(pillarEvent.getContributorID() + " \t " + saltedChecksum); + if (isDuplicate) { + output.resultLine(pillarEvent.getContributorID() + " \t \t File already exists, use get-checksums."); } else { String checksum = Base16Utils.decodeBase16(pillarEvent.getChecksums().getChecksumValue()); - output.resultLine(pillarEvent.getContributorID() + " \t " + checksum); + output.resultLine(pillarEvent.getContributorID() + " \t " + checksumSpecTYPE.getChecksumType().value() + " \t " + checksum); } } } - } diff --git a/bitrepository-client/src/main/java/org/bitrepository/modify/putfile/conversation/IdentifyPillarsForPutFile.java b/bitrepository-client/src/main/java/org/bitrepository/modify/putfile/conversation/IdentifyPillarsForPutFile.java index df9970324..10dbc04c1 100644 --- a/bitrepository-client/src/main/java/org/bitrepository/modify/putfile/conversation/IdentifyPillarsForPutFile.java +++ b/bitrepository-client/src/main/java/org/bitrepository/modify/putfile/conversation/IdentifyPillarsForPutFile.java @@ -63,7 +63,7 @@ protected void handleFailureResponse(MessageResponse msg) throws UnableToFinishE if (ChecksumUtils.areEqual( response.getChecksumDataForExistingFile(), context.getChecksumForValidationAtPillar())) { PutFileCompletePillarEvent event = new PutFileCompletePillarEvent( - response.getPillarID(), response.getCollectionID(), response.getChecksumDataForExistingFile()); + response.getPillarID(), response.getCollectionID(), response.getChecksumDataForExistingFile(), response.getResponseInfo()); event.setInfo("File already existed on " + response.getPillarID()); getContext().getMonitor().contributorComplete(event); } else { diff --git a/bitrepository-client/src/main/java/org/bitrepository/modify/putfile/conversation/PutFileCompletePillarEvent.java b/bitrepository-client/src/main/java/org/bitrepository/modify/putfile/conversation/PutFileCompletePillarEvent.java index 53bfad60c..a51cfe1e2 100644 --- a/bitrepository-client/src/main/java/org/bitrepository/modify/putfile/conversation/PutFileCompletePillarEvent.java +++ b/bitrepository-client/src/main/java/org/bitrepository/modify/putfile/conversation/PutFileCompletePillarEvent.java @@ -26,25 +26,25 @@ package org.bitrepository.modify.putfile.conversation; import org.bitrepository.bitrepositoryelements.ChecksumDataForFileTYPE; +import org.bitrepository.bitrepositoryelements.ResponseInfo; import org.bitrepository.client.eventhandler.ContributorCompleteEvent; /** * Contains the result of a put file request sent to a single pillar. */ public class PutFileCompletePillarEvent extends ContributorCompleteEvent { - /** - * @see #getChecksums(). - */ private final ChecksumDataForFileTYPE result; + private final ResponseInfo responseInfo; /** * @param pillarID The pillar which generated the result * @param collectionID The ID of the collection * @param result The result returned by the pillar. */ - public PutFileCompletePillarEvent(String pillarID, String collectionID, ChecksumDataForFileTYPE result) { + public PutFileCompletePillarEvent(String pillarID, String collectionID, ChecksumDataForFileTYPE result, ResponseInfo responseInfo) { super(pillarID, collectionID); this.result = result; + this.responseInfo = responseInfo; } /** @@ -54,6 +54,13 @@ public ChecksumDataForFileTYPE getChecksums() { return result; } + /** + * @return The response info from a single pillar. + */ + public ResponseInfo getResponseInfo() { + return responseInfo; + } + @Override public String additionalInfo() { return super.additionalInfo() + ", requestedChecksumResult=" + result; diff --git a/bitrepository-client/src/main/java/org/bitrepository/modify/putfile/conversation/PuttingFile.java b/bitrepository-client/src/main/java/org/bitrepository/modify/putfile/conversation/PuttingFile.java index 80c8742c2..2899a18d9 100644 --- a/bitrepository-client/src/main/java/org/bitrepository/modify/putfile/conversation/PuttingFile.java +++ b/bitrepository-client/src/main/java/org/bitrepository/modify/putfile/conversation/PuttingFile.java @@ -62,7 +62,7 @@ protected void generateContributorCompleteEvent(MessageResponse msg) throws Unex if (msg instanceof PutFileFinalResponse) { PutFileFinalResponse response = (PutFileFinalResponse) msg; getContext().getMonitor().contributorComplete(new PutFileCompletePillarEvent(response.getPillarID(), response.getCollectionID(), - response.getChecksumDataForNewFile())); + response.getChecksumDataForNewFile(), response.getResponseInfo())); } else { throw new UnexpectedResponseException( "Received unexpected msg " + msg.getClass().getSimpleName() + " while waiting for Put file response."); @@ -149,7 +149,7 @@ protected boolean handleFailureResponse(MessageResponse msg) throws UnableToFini case DUPLICATE_FILE_FAILURE: if (ChecksumUtils.areEqual(response.getChecksumDataForExistingFile(), context.getChecksumForValidationAtPillar())) { PutFileCompletePillarEvent event = new PutFileCompletePillarEvent(response.getPillarID(), - response.getCollectionID(), response.getChecksumDataForExistingFile()); + response.getCollectionID(), response.getChecksumDataForExistingFile(), response.getResponseInfo()); event.setInfo("File already existed on " + response.getPillarID()); getContext().getMonitor().contributorComplete(event); } else { diff --git a/bitrepository-core/src/main/java/org/bitrepository/common/utils/ChecksumUtils.java b/bitrepository-core/src/main/java/org/bitrepository/common/utils/ChecksumUtils.java index f8f17ac28..4e47b983b 100644 --- a/bitrepository-core/src/main/java/org/bitrepository/common/utils/ChecksumUtils.java +++ b/bitrepository-core/src/main/java/org/bitrepository/common/utils/ChecksumUtils.java @@ -63,8 +63,7 @@ public static String generateChecksum(File file, ChecksumSpecTYPE csSpec) { try { return generateChecksum(new FileInputStream(file), csSpec); } catch (IOException e) { - throw new CoordinationLayerException("Could not calculate the checksum for the file '" - + file.getAbsolutePath() + "'.", e); + throw new CoordinationLayerException("Could not calculate the checksum for the file '" + file.getAbsolutePath() + "'.", e); } } @@ -79,8 +78,7 @@ public static String generateChecksum(FileInfo fileInfo, ChecksumSpecTYPE csSpec try { return generateChecksum(fileInfo.getInputStream(), csSpec); } catch (IOException e) { - throw new CoordinationLayerException("Could not calculate the checksum for the file '" - + fileInfo.getFileID() + "'.", e); + throw new CoordinationLayerException("Could not calculate the checksum for the file '" + fileInfo.getFileID() + "'.", e); } } @@ -125,20 +123,18 @@ public static String generateChecksum(InputStream content, ChecksumSpecTYPE csSp try { if (requiresSalt(algorithm)) { if (csSpec.getChecksumSalt() == null || csSpec.getChecksumSalt().length == 0) { - throw new IllegalArgumentException("Cannot perform a HMAC checksum calculation without salt as requested:" - + csSpec); + throw new IllegalArgumentException("Cannot perform a HMAC checksum calculation without salt as requested:" + csSpec); } digest = calculateChecksumWithHMAC(content, algorithm, csSpec.getChecksumSalt()); } else { if (csSpec.getChecksumSalt() != null && csSpec.getChecksumSalt().length > 0) { - throw new IllegalArgumentException("Cannot perform a message-digest checksum calculation with salt " - + "as requested:" + csSpec); + throw new IllegalArgumentException( + "Cannot perform a message-digest checksum calculation with salt " + "as requested:" + csSpec); } digest = calculateChecksumWithMessageDigest(content, algorithm); } } catch (NoSuchAlgorithmException e) { - throw new IllegalStateException("The checksum algorithm '" + csSpec.getChecksumType().name() - + "' is not supported."); + throw new IllegalStateException("The checksum algorithm '" + csSpec.getChecksumType().name() + "' is not supported."); } return Base16Utils.decodeBase16(digest); @@ -206,8 +202,8 @@ private static byte[] calculateChecksumWithHMAC(InputStream content, ChecksumTyp return messageAuthenticationCode.doFinal(); } catch (Exception e) { e.printStackTrace(); - throw new CoordinationLayerException("Cannot calculate the checksum with algorithm '" + algorithmName - + "' and salt '" + Arrays.toString(salt) + "'", e); + throw new CoordinationLayerException( + "Cannot calculate the checksum with algorithm '" + algorithmName + "' and salt '" + Arrays.toString(salt) + "'", e); } } @@ -222,13 +218,13 @@ public static void verifyAlgorithm(ChecksumSpecTYPE checksumSpec) throws NoSuchA if (requiresSalt(algorithm)) { if (checksumSpec.getChecksumSalt() == null) { - throw new NoSuchAlgorithmException("Cannot perform a HMAC checksum calculation without salt as " - + "requested: " + checksumSpec); + throw new NoSuchAlgorithmException( + "Cannot perform a HMAC checksum calculation without salt as " + "requested: " + checksumSpec); } } else { if (checksumSpec.getChecksumSalt() != null && checksumSpec.getChecksumSalt().length > 0) { - throw new NoSuchAlgorithmException("Cannot perform a message-digest checksum calculation with salt " - + "as requested: " + checksumSpec); + throw new NoSuchAlgorithmException( + "Cannot perform a message-digest checksum calculation with salt " + "as requested: " + checksumSpec); } } } @@ -241,17 +237,12 @@ public static void verifyAlgorithm(ChecksumSpecTYPE checksumSpec) throws NoSuchA * @throws NoSuchAlgorithmException If the algorithm is not supported. */ public static boolean requiresSalt(ChecksumType algorithm) throws NoSuchAlgorithmException { - if ((algorithm == ChecksumType.MD5) - || (algorithm == ChecksumType.SHA1) - || (algorithm == ChecksumType.SHA256) - || (algorithm == ChecksumType.SHA384) - || (algorithm == ChecksumType.SHA512)) { + if ((algorithm == ChecksumType.MD5) || (algorithm == ChecksumType.SHA1) || (algorithm == ChecksumType.SHA256) || + (algorithm == ChecksumType.SHA384) || (algorithm == ChecksumType.SHA512)) { return false; - } else if ((algorithm == ChecksumType.HMAC_MD5) - || (algorithm == ChecksumType.HMAC_SHA1) - || (algorithm == ChecksumType.HMAC_SHA256) - || (algorithm == ChecksumType.HMAC_SHA384) - || (algorithm == ChecksumType.HMAC_SHA512)) { + } else if ((algorithm == ChecksumType.HMAC_MD5) || (algorithm == ChecksumType.HMAC_SHA1) || + (algorithm == ChecksumType.HMAC_SHA256) || (algorithm == ChecksumType.HMAC_SHA384) || + (algorithm == ChecksumType.HMAC_SHA512)) { return true; } else { throw new NoSuchAlgorithmException("The checksum algorithm '" + algorithm + "' is not supported."); @@ -267,8 +258,7 @@ public static boolean requiresSalt(ChecksumType algorithm) throws NoSuchAlgorith */ public static ChecksumSpecTYPE getDefault(Settings settings) { ChecksumSpecTYPE res = new ChecksumSpecTYPE(); - res.setChecksumType(ChecksumType.valueOf( - settings.getRepositorySettings().getProtocolSettings().getDefaultChecksumType())); + res.setChecksumType(ChecksumType.valueOf(settings.getRepositorySettings().getProtocolSettings().getDefaultChecksumType())); try { verifyAlgorithm(res); @@ -279,6 +269,16 @@ public static ChecksumSpecTYPE getDefault(Settings settings) { return res; } + /** + * Returns the default checksum type, defined in settings. + * + * @param settings The settings to retrieve the default checksum type from. + * @return The default checksum type from settigs. + */ + public static ChecksumType getDefaultChecksumType(Settings settings) { + return ChecksumType.valueOf(settings.getRepositorySettings().getProtocolSettings().getDefaultChecksumType()); + } + /** * Checks whether the checksum of two checksum data elements are identical. * @@ -287,9 +287,7 @@ public static ChecksumSpecTYPE getDefault(Settings settings) { * @return Whether the checksum data elements are identical. */ public static boolean areEqual(ChecksumDataForFileTYPE checksum1, ChecksumDataForFileTYPE checksum2) { - return (checksum1 == null && checksum2 == null) || - ((checksum1 != null && checksum2 != null) && - Base16Utils.decodeBase16(checksum1.getChecksumValue()).equals( - Base16Utils.decodeBase16(checksum2.getChecksumValue()))); + return (checksum1 == null && checksum2 == null) || ((checksum1 != null && checksum2 != null) && + Base16Utils.decodeBase16(checksum1.getChecksumValue()).equals(Base16Utils.decodeBase16(checksum2.getChecksumValue()))); } } From be938315a3b92a2102fb15e68ec94da58ba13dc9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mathias=20S=C3=B8by?= Date: Tue, 30 Aug 2022 13:23:00 +0200 Subject: [PATCH 7/9] If Putfile is performed, and file already exists, and a validation checksum is requested, then the returned checksum will be the default. For every new file added to a pillar, the print will show a '*'. --- .../eventhandler/PutFileEventHandler.java | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/bitrepository-client/src/main/java/org/bitrepository/commandline/eventhandler/PutFileEventHandler.java b/bitrepository-client/src/main/java/org/bitrepository/commandline/eventhandler/PutFileEventHandler.java index 6ebaad0e1..76e40c633 100644 --- a/bitrepository-client/src/main/java/org/bitrepository/commandline/eventhandler/PutFileEventHandler.java +++ b/bitrepository-client/src/main/java/org/bitrepository/commandline/eventhandler/PutFileEventHandler.java @@ -22,6 +22,7 @@ package org.bitrepository.commandline.eventhandler; import org.bitrepository.bitrepositoryelements.ChecksumSpecTYPE; +import org.bitrepository.bitrepositoryelements.ChecksumType; import org.bitrepository.bitrepositoryelements.ResponseCode; import org.bitrepository.client.eventhandler.OperationEvent; import org.bitrepository.commandline.output.OutputHandler; @@ -30,6 +31,8 @@ import org.bitrepository.common.utils.ChecksumUtils; import org.bitrepository.modify.putfile.conversation.PutFileCompletePillarEvent; +import java.util.Locale; + /** * Complete event awaiter for GetFile. * Prints out checksum results, if any. @@ -37,6 +40,7 @@ public class PutFileEventHandler extends CompleteEventAwaiter { private final Boolean printOutput; private final ChecksumSpecTYPE checksumSpecTYPE; + private final ChecksumType defaultChecksumType; /** * Constructor. @@ -47,7 +51,8 @@ public class PutFileEventHandler extends CompleteEventAwaiter { public PutFileEventHandler(Settings settings, OutputHandler outputHandler, ChecksumSpecTYPE checksumSpecTYPE) { super(settings, outputHandler); this.checksumSpecTYPE = checksumSpecTYPE; - this.printOutput = checksumSpecTYPE.getChecksumType() != ChecksumUtils.getDefaultChecksumType(settings); + this.defaultChecksumType = ChecksumUtils.getDefaultChecksumType(settings); + this.printOutput = checksumSpecTYPE.getChecksumType() != defaultChecksumType; if (printOutput) { output.resultHeader("PillarId \t Alg \t Checksum"); @@ -64,11 +69,13 @@ public void handleComponentComplete(OperationEvent event) { PutFileCompletePillarEvent pillarEvent = (PutFileCompletePillarEvent) event; boolean isDuplicate = pillarEvent.getResponseInfo().getResponseCode().equals(ResponseCode.DUPLICATE_FILE_FAILURE); if (printOutput && pillarEvent.getChecksums() != null) { + String checksum = Base16Utils.decodeBase16(pillarEvent.getChecksums().getChecksumValue()); if (isDuplicate) { - output.resultLine(pillarEvent.getContributorID() + " \t \t File already exists, use get-checksums."); + output.resultLine( + String.format(Locale.ROOT, "%s \t %s \t %s", pillarEvent.getContributorID(), defaultChecksumType, checksum)); } else { - String checksum = Base16Utils.decodeBase16(pillarEvent.getChecksums().getChecksumValue()); - output.resultLine(pillarEvent.getContributorID() + " \t " + checksumSpecTYPE.getChecksumType().value() + " \t " + checksum); + output.resultLine(String.format(Locale.ROOT, "%s \t %s \t %s \t*", pillarEvent.getContributorID(), + checksumSpecTYPE.getChecksumType().value(), checksum)); } } } From 67d5033c8abbd8a3652aa8dd681adc366b4d9229 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mathias=20S=C3=B8by?= Date: Wed, 31 Aug 2022 14:42:56 +0200 Subject: [PATCH 8/9] Removed a condition for when the cmd client has the -u (URL) argument. --- .../main/java/org/bitrepository/commandline/PutFileCmd.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/bitrepository-client/src/main/java/org/bitrepository/commandline/PutFileCmd.java b/bitrepository-client/src/main/java/org/bitrepository/commandline/PutFileCmd.java index 12f24992e..250fd3924 100644 --- a/bitrepository-client/src/main/java/org/bitrepository/commandline/PutFileCmd.java +++ b/bitrepository-client/src/main/java/org/bitrepository/commandline/PutFileCmd.java @@ -50,7 +50,6 @@ import static org.bitrepository.commandline.Constants.FILE_ID_ARG; import static org.bitrepository.commandline.Constants.HAS_ARGUMENT; import static org.bitrepository.commandline.Constants.NO_ARGUMENT; -import static org.bitrepository.commandline.Constants.PILLAR_ARG; import static org.bitrepository.commandline.Constants.REQUEST_CHECKSUM_SALT_ARG; import static org.bitrepository.commandline.Constants.REQUEST_CHECKSUM_SALT_DESC; import static org.bitrepository.commandline.Constants.REQUEST_CHECKSUM_TYPE_ARG; @@ -179,7 +178,7 @@ private OperationEvent putTheFile() { ChecksumDataForFileTYPE validationChecksum = getValidationChecksum(); ChecksumSpecTYPE requestedChecksumSpec = getRequestChecksumSpecOrDefault(); - if (cmdHandler.hasOption(URL_ARG) && !cmdHandler.hasOption(PILLAR_ARG)) { + if (cmdHandler.hasOption(URL_ARG)) { output.debug("Generating checksum from URL."); String checksumFromURL = getChecksumFromURL(url, fileExchange, getDefaultChecksumSpec()); String checksumFromArg = Base16Utils.decodeBase16(validationChecksum.getChecksumValue()); From 2878d30dcc847aa8ff5da0767d8ad025598b5b23 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mathias=20S=C3=B8by?= Date: Thu, 1 Sep 2022 14:37:06 +0200 Subject: [PATCH 9/9] Added a shutdown to the message bus when hitting an exception. Also removed a redundant try block. --- .../commandline/CommandLineClient.java | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/bitrepository-client/src/main/java/org/bitrepository/commandline/CommandLineClient.java b/bitrepository-client/src/main/java/org/bitrepository/commandline/CommandLineClient.java index 6684312e0..2b2bc938e 100644 --- a/bitrepository-client/src/main/java/org/bitrepository/commandline/CommandLineClient.java +++ b/bitrepository-client/src/main/java/org/bitrepository/commandline/CommandLineClient.java @@ -69,15 +69,15 @@ public abstract class CommandLineClient { */ public void runCommand() throws Exception { try { - try { - performOperation(); - } catch (InvalidChecksumException | IllegalArgumentException ie) { - System.out.println(ie.getMessage()); - System.exit(Constants.EXIT_OPERATION_FAILURE); - } catch (Exception e) { - e.printStackTrace(); - System.exit(Constants.EXIT_OPERATION_FAILURE); - } + performOperation(); + } catch (InvalidChecksumException | IllegalArgumentException ie) { + output.warn(ie.getMessage()); + shutdown(); + System.exit(Constants.EXIT_OPERATION_FAILURE); + } catch (Exception e) { + output.error("Unexpected Exception.", e); + shutdown(); + System.exit(Constants.EXIT_OPERATION_FAILURE); } finally { shutdown(); }