Skip to content

Bitmag 778 - Checksum pillar stores checksum for PutFile with nonexisting URL #42

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
m-atlantis merged 10 commits into from
Sep 2, 2022
Merged

Bitmag 778 - Checksum pillar stores checksum for PutFile with nonexisting URL #42

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
1b03171
Now, when the PutFileCmd performs the PutFile operation; it will simp…
m-atlantis Aug 4, 2022
1444db1
Created a small improvement.
m-atlantis Aug 8, 2022
4258b15
Small change to error message.
m-atlantis Aug 9, 2022
b682522
Small change to CMD that might be used.
m-atlantis Aug 15, 2022
780f3ba
Put in a fixme. Checksum pillar still stores the checksum if the URL …
m-atlantis Aug 16, 2022
4822439
Will no longer store a checksum when put-file operation is performed …
m-atlantis Aug 29, 2022
be93831
If Putfile is performed, and file already exists, and a validation ch…
m-atlantis Aug 30, 2022
67d5033
Removed a condition for when the cmd client has the -u (URL) argument.
m-atlantis Aug 31, 2022
2878d30
Added a shutdown to the message bus when hitting an exception. Also r…
m-atlantis Sep 1, 2022
9f0dd85
Merge remote-tracking branch 'origin/master' into BITMAG-778
m-atlantis Sep 1, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
46 changes: 46 additions & 0 deletions ...ry-client/src/main/java/org/bitrepository/client/exceptions/InvalidChecksumException.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
/*
* #%L
* Bitrepository Protocol
*
* $Id$
* $HeadURL$
* %%
* Copyright (C) 2010 - 2011 The State and University Library, The Royal Library and The State Archives, Denmark
* %%
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as
* published by the Free Software Foundation, either version 2.1 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Lesser Public License for more details.
*
* You should have received a copy of the GNU General Lesser Public
* License along with this program. If not, see
* <http://www.gnu.org/licenses/lgpl-2.1.html>.
* #L%
*/
package org.bitrepository.client.exceptions;

/**
* Used to indicate that an unexpected response has been received.
*/
@SuppressWarnings("serial")
public class InvalidChecksumException extends RuntimeException {
public InvalidChecksumException() {
}

public InvalidChecksumException(String message) {
super(message);
}

public InvalidChecksumException(String message, Throwable cause) {
super(message, cause);
}

public InvalidChecksumException(Throwable cause) {
super(cause);
}
}
41 changes: 30 additions & 11 deletions bitrepository-client/src/main/java/org/bitrepository/commandline/CommandLineClient.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
import org.apache.commons.codec.DecoderException;
import org.bitrepository.bitrepositoryelements.ChecksumDataForFileTYPE;
import org.bitrepository.bitrepositoryelements.ChecksumSpecTYPE;
import org.bitrepository.client.exceptions.InvalidChecksumException;
import org.bitrepository.commandline.output.DefaultOutputHandler;
import org.bitrepository.commandline.output.OutputHandler;
import org.bitrepository.commandline.utils.ChecksumExtractionUtils;
Expand All @@ -45,7 +46,7 @@

import javax.jms.JMSException;
import java.io.File;
import java.net.MalformedURLException;
import java.net.HttpURLConnection;
import java.net.URL;
import java.security.NoSuchAlgorithmException;
import java.util.List;
Expand All @@ -70,12 +71,15 @@ public abstract class CommandLineClient {
*/
public void runCommand() throws Exception {
try {
Comment thread
m-atlantis marked this conversation as resolved.
try {
performOperation();
} catch (Exception e) {
e.printStackTrace();
System.exit(Constants.EXIT_OPERATION_FAILURE);
}
performOperation();
} catch (InvalidChecksumException | IllegalArgumentException ie) {
output.warn(ie.getMessage());
shutdown();
System.exit(Constants.EXIT_OPERATION_FAILURE);
} catch (Exception e) {
output.error("Unexpected Exception.", e);
shutdown();
System.exit(Constants.EXIT_OPERATION_FAILURE);
} finally {
shutdown();
}
Expand Down Expand Up @@ -224,6 +228,13 @@ protected ChecksumSpecTYPE getRequestChecksumSpecOrDefault() {
return getRequestChecksumSpec();
}

/**
* @return Returns the default ChecksumSpecType by calling {@link ChecksumUtils#getDefault(Settings)}.
*/
protected ChecksumSpecTYPE getDefaultChecksumSpec() {
return ChecksumUtils.getDefault(settings);
}

/**
* @return The requested checksum spec, or null.
*/
Expand Down Expand Up @@ -329,11 +340,19 @@ protected URL getURLOrUploadFile() {
FileExchange fileexchange = ProtocolComponentFactory.getInstance().getFileExchange(settings);
return fileexchange.putFile(f);
} else {
String urlArg = cmdHandler.getOptionValue(Constants.URL_ARG);
try {
return new URL(cmdHandler.getOptionValue(Constants.URL_ARG));
} catch (MalformedURLException e) {
throw new IllegalArgumentException(
"The URL argument is either empty or not a valid URL: " + cmdHandler.getOptionValue(Constants.URL_ARG), e);
final URL url = new URL(urlArg);
HttpURLConnection connection = (HttpURLConnection) url.openConnection();
int responseCode = connection.getResponseCode();

if (responseCode > 399) {
throw new Exception("Http URL Connection ResponseCode: " + responseCode);
}

return url;
} catch (Exception e) {
throw new IllegalArgumentException("The URL argument is either empty or not a valid URL: " + urlArg, e);
}
}
}
Expand Down
112 changes: 80 additions & 32 deletions bitrepository-client/src/main/java/org/bitrepository/commandline/PutFileCmd.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,14 +26,35 @@
import org.bitrepository.bitrepositoryelements.ChecksumSpecTYPE;
import org.bitrepository.client.eventhandler.OperationEvent;
import org.bitrepository.client.eventhandler.OperationEvent.OperationEventType;
import org.bitrepository.client.exceptions.InvalidChecksumException;
import org.bitrepository.commandline.eventhandler.CompleteEventAwaiter;
import org.bitrepository.commandline.eventhandler.PutFileEventHandler;
import org.bitrepository.common.utils.Base16Utils;
import org.bitrepository.common.utils.ChecksumUtils;
import org.bitrepository.modify.ModifyComponentFactory;
import org.bitrepository.modify.putfile.PutFileClient;
import org.bitrepository.protocol.FileExchange;
import org.bitrepository.protocol.ProtocolComponentFactory;

import java.io.IOException;
import java.net.URL;

import static org.bitrepository.commandline.Constants.ARGUMENT_IS_NOT_REQUIRED;
import static org.bitrepository.commandline.Constants.CHECKSUM_ARG;
import static org.bitrepository.commandline.Constants.DELETE_FILE_ARG;
import static org.bitrepository.commandline.Constants.DELETE_FILE_DESC;
import static org.bitrepository.commandline.Constants.EXIT_ARGUMENT_FAILURE;
import static org.bitrepository.commandline.Constants.EXIT_OPERATION_FAILURE;
import static org.bitrepository.commandline.Constants.EXIT_SUCCESS;
import static org.bitrepository.commandline.Constants.FILE_ARG;
import static org.bitrepository.commandline.Constants.FILE_ID_ARG;
import static org.bitrepository.commandline.Constants.HAS_ARGUMENT;
import static org.bitrepository.commandline.Constants.NO_ARGUMENT;
import static org.bitrepository.commandline.Constants.REQUEST_CHECKSUM_SALT_ARG;
import static org.bitrepository.commandline.Constants.REQUEST_CHECKSUM_SALT_DESC;
import static org.bitrepository.commandline.Constants.REQUEST_CHECKSUM_TYPE_ARG;
import static org.bitrepository.commandline.Constants.REQUEST_CHECKSUM_TYPE_DESC;
import static org.bitrepository.commandline.Constants.URL_ARG;

public class PutFileCmd extends CommandLineClient {
private final PutFileClient client;
Expand All @@ -46,10 +67,10 @@ public static void main(String[] args) {
PutFileCmd client = new PutFileCmd(args);
client.runCommand();
} catch (IllegalArgumentException iae) {
System.exit(Constants.EXIT_ARGUMENT_FAILURE);
System.exit(EXIT_ARGUMENT_FAILURE);
} catch (Exception e) {
e.printStackTrace();
System.exit(Constants.EXIT_OPERATION_FAILURE);
System.exit(EXIT_OPERATION_FAILURE);
}
}

Expand Down Expand Up @@ -82,42 +103,39 @@ public void performOperation() {
OperationEvent finalEvent = putTheFile();
output.completeEvent("Results of the PutFile operation for the file '" + getFileIDForMessage() + "'", finalEvent);
if (finalEvent.getEventType() == OperationEventType.COMPLETE) {
System.exit(Constants.EXIT_SUCCESS);
System.exit(EXIT_SUCCESS);
} else {
System.exit(Constants.EXIT_OPERATION_FAILURE);
System.exit(EXIT_OPERATION_FAILURE);
}
}

@Override
protected void createOptionsForCmdArgumentHandler() {
super.createOptionsForCmdArgumentHandler();

Option fileOption = new Option(Constants.FILE_ARG, Constants.HAS_ARGUMENT,
"The path to the file to be uploaded. Is required, unless a URL is given.");
Option fileOption = new Option(FILE_ARG, HAS_ARGUMENT, "The path to the file to be uploaded. Is required, unless a URL is given.");
fileOption.setRequired(ARGUMENT_IS_NOT_REQUIRED);
cmdHandler.addOption(fileOption);

Option urlOption = new Option(Constants.URL_ARG, Constants.HAS_ARGUMENT,
Option urlOption = new Option(URL_ARG, HAS_ARGUMENT,
"The URL for the file to be uploaded. Is required, unless a local file is given.");
urlOption.setRequired(ARGUMENT_IS_NOT_REQUIRED);
cmdHandler.addOption(urlOption);

Option checksumOption = new Option(Constants.CHECKSUM_ARG, Constants.HAS_ARGUMENT,
Option checksumOption = new Option(CHECKSUM_ARG, HAS_ARGUMENT,
"The checksum for the file to be retrieved. Is required if using a URL.");
checksumOption.setRequired(ARGUMENT_IS_NOT_REQUIRED);
cmdHandler.addOption(checksumOption);

Option checksumTypeOption = new Option(Constants.REQUEST_CHECKSUM_TYPE_ARG, Constants.HAS_ARGUMENT,
Constants.REQUEST_CHECKSUM_TYPE_DESC);
Option checksumTypeOption = new Option(REQUEST_CHECKSUM_TYPE_ARG, HAS_ARGUMENT, REQUEST_CHECKSUM_TYPE_DESC);
checksumTypeOption.setRequired(ARGUMENT_IS_NOT_REQUIRED);
cmdHandler.addOption(checksumTypeOption);

Option checksumSaltOption = new Option(Constants.REQUEST_CHECKSUM_SALT_ARG, Constants.HAS_ARGUMENT,
Constants.REQUEST_CHECKSUM_SALT_DESC);
Option checksumSaltOption = new Option(REQUEST_CHECKSUM_SALT_ARG, HAS_ARGUMENT, REQUEST_CHECKSUM_SALT_DESC);
checksumSaltOption.setRequired(ARGUMENT_IS_NOT_REQUIRED);
cmdHandler.addOption(checksumSaltOption);

Option deleteOption = new Option(Constants.DELETE_FILE_ARG, Constants.NO_ARGUMENT, Constants.DELETE_FILE_DESC);
Option deleteOption = new Option(DELETE_FILE_ARG, NO_ARGUMENT, DELETE_FILE_DESC);
deleteOption.setRequired(ARGUMENT_IS_NOT_REQUIRED);
cmdHandler.addOption(deleteOption);
}
Expand All @@ -130,16 +148,16 @@ protected void createOptionsForCmdArgumentHandler() {
protected void validateArguments() {
super.validateArguments();

if (cmdHandler.hasOption(Constants.FILE_ARG) && cmdHandler.hasOption(Constants.URL_ARG)) {
if (cmdHandler.hasOption(FILE_ARG) && cmdHandler.hasOption(URL_ARG)) {
throw new IllegalArgumentException("Cannot take both a file (-f) and a URL (-u) as argument.");
}
if (!(cmdHandler.hasOption(Constants.FILE_ARG) || cmdHandler.hasOption(Constants.URL_ARG))) {
if (!(cmdHandler.hasOption(FILE_ARG) || cmdHandler.hasOption(URL_ARG))) {
throw new IllegalArgumentException("Providing either file argument (-f) or URL argument (-u) is required.");
}
if (cmdHandler.hasOption(Constants.URL_ARG) && !cmdHandler.hasOption(Constants.CHECKSUM_ARG)) {
if (cmdHandler.hasOption(URL_ARG) && !cmdHandler.hasOption(CHECKSUM_ARG)) {
throw new IllegalArgumentException("Using URL argument (-u) requires the checksum argument (-C).");
}
if (cmdHandler.hasOption(Constants.URL_ARG) && !cmdHandler.hasOption(Constants.FILE_ID_ARG)) {
if (cmdHandler.hasOption(URL_ARG) && !cmdHandler.hasOption(FILE_ID_ARG)) {
throw new IllegalArgumentException("Using URL argument (-u) requires the file ID argument (-i).");
}
}
Expand All @@ -151,27 +169,58 @@ protected void validateArguments() {
*/
private OperationEvent putTheFile() {
output.debug("Uploading the file to the FileExchange.");
URL url = getURLOrUploadFile();
final URL url = getURLOrUploadFile();

String fileID = retrieveFileID();
FileExchange fileExchange = ProtocolComponentFactory.getInstance().getFileExchange(settings);

output.debug("Initiating the PutFile conversation.");
ChecksumDataForFileTYPE validationChecksum = getValidationChecksum();
ChecksumSpecTYPE requestChecksum = getRequestChecksumSpecOrNull();

boolean printChecksums = cmdHandler.hasOption(Constants.REQUEST_CHECKSUM_TYPE_ARG);
ChecksumSpecTYPE requestedChecksumSpec = getRequestChecksumSpecOrDefault();

if (cmdHandler.hasOption(URL_ARG)) {
output.debug("Generating checksum from URL.");
String checksumFromURL = getChecksumFromURL(url, fileExchange, getDefaultChecksumSpec());
String checksumFromArg = Base16Utils.decodeBase16(validationChecksum.getChecksumValue());
output.debug("Comparing given checksum with the checksum of the file on the URL.");
if (!checksumFromURL.equals(checksumFromArg)) {
throw new InvalidChecksumException(
"Checksum from URL: " + checksumFromURL + " does not match checksum from argument: " + checksumFromArg);
}
}

CompleteEventAwaiter eventHandler = new PutFileEventHandler(settings, output, printChecksums);
client.putFile(getCollectionID(), url, fileID, getSizeOfFileOrZero(), validationChecksum, requestChecksum, eventHandler, null);
output.debug("Performing the PutFile conversation.");
CompleteEventAwaiter eventHandler = new PutFileEventHandler(settings, output, requestedChecksumSpec);
client.putFile(getCollectionID(), url, fileID, getSizeOfFileOrZero(), validationChecksum, requestedChecksumSpec, eventHandler,
null);

output.debug("Awaiting PutFile conversation final event.");
OperationEvent finalEvent = eventHandler.getFinish();

if (cmdHandler.hasOption(Constants.DELETE_FILE_ARG)) {
if (cmdHandler.hasOption(DELETE_FILE_ARG)) {
output.debug("Deleting file.");
deleteFileAfterwards(url);
}

return finalEvent;
}

/**
* Generates the checksum of the file on the given URL.
*
* @param url The URL where the file can be found.
* @param fileExchange FileExchange, used convert the file to an {@link java.io.InputStream}.
* @param checksumSpecType The requested {@link ChecksumSpecTYPE}.
* @return Returns the checksum as a {@link String}.
*/
private String getChecksumFromURL(URL url, FileExchange fileExchange, ChecksumSpecTYPE checksumSpecType) {
try {
return ChecksumUtils.generateChecksum(fileExchange.getFile(url), checksumSpecType);
} catch (IOException e) {
throw new IllegalArgumentException("Could not retrieve file from " + url);
}
}

/**
* Retrieves the Checksum of the file, used by the pillars to validate.
* This checksum is either taken from the actual file, or from the checksum argument.
Expand All @@ -181,24 +230,23 @@ private OperationEvent putTheFile() {
* @return The spec-type of the checksum as {@link ChecksumDataForFileTYPE}.
*/
protected ChecksumDataForFileTYPE getValidationChecksum() {
if (cmdHandler.hasOption(Constants.FILE_ARG)) {
if (cmdHandler.hasOption(FILE_ARG)) {
return getValidationChecksumDataForFile(findTheFile());
} else {
return getValidationChecksumDataFromArgument(Constants.CHECKSUM_ARG);
return getValidationChecksumDataFromArgument(CHECKSUM_ARG);
}
}

/**
* @return The filename (FileID) for the file to upload.
*/
private String getFileIDForMessage() {
if (cmdHandler.hasOption(Constants.URL_ARG)) {
return cmdHandler.getOptionValue(Constants.URL_ARG) + " (with the id '" + cmdHandler.getOptionValue(Constants.FILE_ID_ARG) +
"')";
if (cmdHandler.hasOption(URL_ARG)) {
return cmdHandler.getOptionValue(URL_ARG) + " (with the id '" + cmdHandler.getOptionValue(FILE_ID_ARG) + "')";
}
if (cmdHandler.hasOption(Constants.FILE_ARG)) {
return cmdHandler.getOptionValue(Constants.FILE_ARG) + (cmdHandler.hasOption(Constants.FILE_ID_ARG) ?
" (with the id '" + cmdHandler.getOptionValue(Constants.FILE_ID_ARG) + "')" : "");
if (cmdHandler.hasOption(FILE_ARG)) {
return cmdHandler.getOptionValue(FILE_ARG) +
(cmdHandler.hasOption(FILE_ID_ARG) ? " (with the id '" + cmdHandler.getOptionValue(FILE_ID_ARG) + "')" : "");
}
return "Failed";
}
Expand Down
Loading