From edbd68c7e8b1982a0439a75298668bb021a9807f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Elek=20M=C3=A1rton?= Date: Tue, 24 Nov 2020 13:34:08 +0100 Subject: [PATCH 01/11] HDDS-4506. Support query parameter based v4 auth in S3g --- .../ozone/s3/AWSSignatureProcessor.java | 456 ------------------ .../hadoop/ozone/s3/OzoneClientProducer.java | 79 +-- .../ozone/s3/endpoint/BucketEndpoint.java | 1 + .../ozone/s3/endpoint/EndpointBase.java | 14 - .../ozone/s3/exception/OS3Exception.java | 14 +- .../ozone/s3/exception/S3ErrorTable.java | 2 +- .../s3/signature/AWSSignatureProcessor.java | 199 ++++++++ .../AuthorizationHeaderV2.java | 8 +- .../AuthorizationV4HeaderParser.java} | 202 +++----- .../signature/AuthorizationV4QueryParser.java | 48 ++ .../s3/{header => signature}/Credential.java | 2 +- .../ozone/s3/signature/SignatureInfo.java | 56 +++ .../ozone/s3/signature/SignatureParser.java | 10 + .../{ => signature}/SignatureProcessor.java | 32 +- .../s3/signature/StringToSignProducer.java | 282 +++++++++++ .../{header => signature}/package-info.java | 2 +- .../ozone/s3/TestAWSSignatureProcessor.java | 141 ------ .../ozone/s3/TestOzoneClientProducer.java | 17 +- .../ozone/s3/endpoint/TestBucketPut.java | 31 +- .../ozone/s3/endpoint/TestRootList.java | 24 - .../s3/header/TestAuthorizationHeaderV4.java | 354 -------------- .../signature/TestAWSSignatureProcessor.java | 127 +++++ .../TestAuthorizationHeaderV2.java | 2 +- .../TestAuthorizationV4HeaderParser.java | 354 ++++++++++++++ .../signature/TestStringToSignProducer.java | 76 +++ 25 files changed, 1305 insertions(+), 1228 deletions(-) delete mode 100644 hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/AWSSignatureProcessor.java create mode 100644 hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/AWSSignatureProcessor.java rename hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/{header => signature}/AuthorizationHeaderV2.java (98%) rename hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/{header/AuthorizationHeaderV4.java => signature/AuthorizationV4HeaderParser.java} (54%) create mode 100644 hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/AuthorizationV4QueryParser.java rename hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/{header => signature}/Credential.java (98%) create mode 100644 hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/SignatureInfo.java create mode 100644 hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/SignatureParser.java rename hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/{ => signature}/SignatureProcessor.java (61%) create mode 100644 hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/StringToSignProducer.java rename hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/{header => signature}/package-info.java (95%) delete mode 100644 hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/TestAWSSignatureProcessor.java delete mode 100644 hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/header/TestAuthorizationHeaderV4.java create mode 100644 hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/signature/TestAWSSignatureProcessor.java rename hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/{header => signature}/TestAuthorizationHeaderV2.java (98%) create mode 100644 hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/signature/TestAuthorizationV4HeaderParser.java create mode 100644 hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/signature/TestStringToSignProducer.java diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/AWSSignatureProcessor.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/AWSSignatureProcessor.java deleted file mode 100644 index 4d4510128ccc..000000000000 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/AWSSignatureProcessor.java +++ /dev/null @@ -1,456 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.ozone.s3; - -import javax.annotation.PostConstruct; -import javax.enterprise.context.RequestScoped; -import javax.ws.rs.container.ContainerRequestContext; -import javax.ws.rs.core.Context; -import javax.ws.rs.core.MultivaluedMap; -import java.io.UnsupportedEncodingException; -import java.net.InetAddress; -import java.net.URI; -import java.net.URISyntaxException; -import java.net.URLEncoder; -import java.net.UnknownHostException; -import java.security.MessageDigest; -import java.security.NoSuchAlgorithmException; -import java.time.LocalDate; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; -import java.util.Set; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -import org.apache.hadoop.ozone.s3.exception.OS3Exception; -import org.apache.hadoop.ozone.s3.header.AuthorizationHeaderV2; -import org.apache.hadoop.ozone.s3.header.AuthorizationHeaderV4; -import org.apache.hadoop.ozone.s3.header.Credential; - -import com.google.common.annotations.VisibleForTesting; -import static java.time.temporal.ChronoUnit.SECONDS; -import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.S3_AUTHINFO_CREATION_ERROR; -import org.apache.kerby.util.Hex; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Parser to process AWS V2 & V4 auth request. Creates string to sign and auth - * header. For more details refer to AWS documentation https://docs.aws - * .amazon.com/general/latest/gr/sigv4-create-canonical-request.html. - **/ -@RequestScoped -public class AWSSignatureProcessor implements SignatureProcessor { - - private final static Logger LOG = - LoggerFactory.getLogger(AWSSignatureProcessor.class); - - @Context - private ContainerRequestContext context; - - private Map headers; - private MultivaluedMap queryMap; - private String uri; - private String method; - private AuthorizationHeaderV4 v4Header; - private AuthorizationHeaderV2 v2Header; - private String stringToSign; - private Exception exception; - - @PostConstruct - public void init() - throws Exception { - //header map is MUTABLE. It's better to save it here. (with lower case - // keys!!!) - this.headers = new LowerCaseKeyStringMap(new HashMap<>()); - for (Entry> headerEntry : context.getHeaders() - .entrySet()) { - if (0 < headerEntry.getValue().size()) { - String headerKey = headerEntry.getKey(); - if (headers.containsKey(headerKey)) { - //mutiple headers from the same type are combined - headers.put(headerKey, - headers.get(headerKey) + "," + headerEntry.getValue().get(0)); - } else { - headers.put(headerKey, headerEntry.getValue().get(0)); - } - } - } - //in case of the HeaderPreprocessor executed before us, let's restore the - // original content type. - if (headers.containsKey(HeaderPreprocessor.ORIGINAL_CONTENT_TYPE)) { - headers.put(HeaderPreprocessor.CONTENT_TYPE, - headers.get(HeaderPreprocessor.ORIGINAL_CONTENT_TYPE)); - } - - - this.queryMap = context.getUriInfo().getQueryParameters(); - this.uri = context.getUriInfo().getRequestUri().getPath(); - - this.method = context.getMethod(); - String authHeader = headers.get(AUTHORIZATION_HEADER); - try { - if (authHeader != null) { - String[] split = authHeader.split(" "); - if (split[0].equals(AuthorizationHeaderV2.IDENTIFIER)) { - if (v2Header == null) { - v2Header = new AuthorizationHeaderV2(authHeader); - } - } else { - if (v4Header == null) { - v4Header = new AuthorizationHeaderV4(authHeader); - } - parse(); - } - } else { // no auth header - v4Header = null; - v2Header = null; - } - } catch (Exception ex) { - // During validation of auth header, create instance and set Exception. - // This way it can be handled in OzoneClientProducer creation of - // SignatureProcessor instance failure. - if (LOG.isDebugEnabled()) { - LOG.debug("Error during Validation of Auth Header:{}", authHeader); - } - this.exception = ex; - } - } - - - private void parse() throws Exception { - - StringBuilder strToSign = new StringBuilder(); - // According to AWS sigv4 documentation, authorization header should be - // in following format. - // Authorization: algorithm Credential=access key ID/credential scope, - // SignedHeaders=SignedHeaders, Signature=signature - - // Construct String to sign in below format. - // StringToSign = - // Algorithm + \n + - // RequestDateTime + \n + - // CredentialScope + \n + - // HashedCanonicalRequest - String algorithm, requestDateTime, credentialScope, canonicalRequest; - algorithm = v4Header.getAlgorithm(); - requestDateTime = headers.get(X_AMAZ_DATE); - Credential credential = v4Header.getCredentialObj(); - credentialScope = String.format("%s/%s/%s/%s", credential.getDate(), - credential.getAwsRegion(), credential.getAwsService(), - credential.getAwsRequest()); - - // If the absolute path is empty, use a forward slash (/) - uri = (uri.trim().length() > 0) ? uri : "/"; - // Encode URI and preserve forward slashes - strToSign.append(algorithm + NEWLINE); - strToSign.append(requestDateTime + NEWLINE); - strToSign.append(credentialScope + NEWLINE); - - canonicalRequest = buildCanonicalRequest(); - strToSign.append(hash(canonicalRequest)); - if (LOG.isDebugEnabled()) { - LOG.debug("canonicalRequest:[{}]", canonicalRequest); - } - - if (LOG.isTraceEnabled()) { - headers.keySet().forEach(k -> LOG.trace("Header:{},value:{}", k, - headers.get(k))); - } - - LOG.debug("StringToSign:[{}]", strToSign); - stringToSign = strToSign.toString(); - } - - @VisibleForTesting - protected String buildCanonicalRequest() throws OS3Exception { - - Iterable parts = split("/", uri); - List encParts = new ArrayList<>(); - for (String p : parts) { - encParts.add(urlEncode(p)); - } - String canonicalUri = join("/", encParts); - - String canonicalQueryStr = getQueryParamString(); - - StringBuilder canonicalHeaders = new StringBuilder(); - - for (String header : v4Header.getSignedHeaders()) { - canonicalHeaders.append(header.toLowerCase()); - canonicalHeaders.append(":"); - if (headers.containsKey(header)) { - String headerValue = headers.get(header); - canonicalHeaders.append(headerValue); - canonicalHeaders.append(NEWLINE); - - // Set for testing purpose only to skip date and host validation. - validateSignedHeader(header, headerValue); - - } else { - throw new RuntimeException("Header " + header + " not present in " + - "request but requested to be signed."); - } - } - - String payloadHash; - if (UNSIGNED_PAYLOAD.equals( - headers.get(X_AMZ_CONTENT_SHA256))) { - payloadHash = UNSIGNED_PAYLOAD; - } else { - payloadHash = headers.get(X_AMZ_CONTENT_SHA256); - } - - String signedHeaderStr = v4Header.getSignedHeaderString(); - String canonicalRequest = method + NEWLINE - + canonicalUri + NEWLINE - + canonicalQueryStr + NEWLINE - + canonicalHeaders + NEWLINE - + signedHeaderStr + NEWLINE - + payloadHash; - - return canonicalRequest; - } - - @VisibleForTesting - void validateSignedHeader(String header, String headerValue) - throws OS3Exception { - switch (header) { - case HOST: - try { - String schema = context.getUriInfo().getRequestUri().getScheme(); - URI hostUri = new URI(schema + "://" + headerValue); - InetAddress.getByName(hostUri.getHost()); - // TODO: Validate if current request is coming from same host. - } catch (UnknownHostException|URISyntaxException e) { - LOG.error("Host value mentioned in signed header is not valid. " + - "Host:{}", headerValue); - throw S3_AUTHINFO_CREATION_ERROR; - } - break; - case X_AMAZ_DATE: - LocalDate date = LocalDate.parse(headerValue, TIME_FORMATTER); - LocalDate now = LocalDate.now(); - if (date.isBefore(now.minus(PRESIGN_URL_MAX_EXPIRATION_SECONDS, SECONDS)) - || date.isAfter(now.plus(PRESIGN_URL_MAX_EXPIRATION_SECONDS, - SECONDS))) { - LOG.error("AWS date not in valid range. Request timestamp:{} should " + - "not be older than {} seconds.", headerValue, - PRESIGN_URL_MAX_EXPIRATION_SECONDS); - throw S3_AUTHINFO_CREATION_ERROR; - } - break; - case X_AMZ_CONTENT_SHA256: - // TODO: Construct request payload and match HEX(SHA256(requestPayload)) - break; - default: - break; - } - } - - /** - * String join that also works with empty strings. - * - * @return joined string - */ - private static String join(String glue, List parts) { - StringBuilder result = new StringBuilder(); - boolean addSeparator = false; - for (String p : parts) { - if (addSeparator) { - result.append(glue); - } - result.append(p); - addSeparator = true; - } - return result.toString(); - } - - /** - * Returns matching strings. - * - * @param regex Regular expression to split by - * @param whole The string to split - * @return pieces - */ - private static Iterable split(String regex, String whole) { - Pattern p = Pattern.compile(regex); - Matcher m = p.matcher(whole); - List result = new ArrayList<>(); - int pos = 0; - while (m.find()) { - result.add(whole.substring(pos, m.start())); - pos = m.end(); - } - result.add(whole.substring(pos)); - return result; - } - - private String urlEncode(String str) { - try { - - return URLEncoder.encode(str, UTF_8.name()) - .replaceAll("\\+", "%20") - .replaceAll("%7E", "~"); - } catch (UnsupportedEncodingException e) { - throw new RuntimeException(e); - } - } - - private String getQueryParamString() { - List params = new ArrayList<>(queryMap.keySet()); - - // Sort by name, then by value - Collections.sort(params, (o1, o2) -> o1.equals(o2) ? - queryMap.getFirst(o1).compareTo(queryMap.getFirst(o2)) : - o1.compareTo(o2)); - - StringBuilder result = new StringBuilder(); - for (String p : params) { - if (result.length() > 0) { - result.append("&"); - } - result.append(urlEncode(p)); - result.append('='); - - result.append(urlEncode(queryMap.getFirst(p))); - } - return result.toString(); - } - - public static String hash(String payload) throws NoSuchAlgorithmException { - MessageDigest md = MessageDigest.getInstance("SHA-256"); - md.update(payload.getBytes(UTF_8)); - return Hex.encode(md.digest()).toLowerCase(); - } - - public String getAwsAccessId() { - return (v4Header != null ? v4Header.getAccessKeyID() : - v2Header != null ? v2Header.getAccessKeyID() : ""); - } - - public String getSignature() { - return (v4Header != null ? v4Header.getSignature() : - v2Header != null ? v2Header.getSignature() : ""); - } - - public String getStringToSign() throws Exception { - return stringToSign; - } - - @VisibleForTesting - public void setContext(ContainerRequestContext context) { - this.context = context; - } - - @VisibleForTesting - public void setV4Header( - AuthorizationHeaderV4 v4Header) { - this.v4Header = v4Header; - } - - @VisibleForTesting - public void setV2Header(AuthorizationHeaderV2 v2Header) { - this.v2Header = v2Header; - } - - public Exception getException() { - return this.exception; - } - - /** - * A simple map which forces lower case key usage. - */ - public static class LowerCaseKeyStringMap implements Map { - - private HashMap delegate; - - public LowerCaseKeyStringMap( - HashMap delegate) { - this.delegate = delegate; - } - - @Override - public int size() { - return delegate.size(); - } - - @Override - public boolean isEmpty() { - return delegate.isEmpty(); - } - - @Override - public boolean containsKey(Object key) { - return delegate.containsKey(key.toString().toLowerCase()); - } - - @Override - public boolean containsValue(Object value) { - return delegate.containsValue(value); - } - - @Override - public String get(Object key) { - return delegate.get(key.toString().toLowerCase()); - } - - @Override - public String put(String key, String value) { - return delegate.put(key.toLowerCase(), value); - } - - @Override - public String remove(Object key) { - return delegate.remove(key.toString()); - } - - @Override - public void putAll(Map m) { - for (Entry entry : m.entrySet()) { - put(entry.getKey().toLowerCase(), entry.getValue()); - } - } - - @Override - public void clear() { - delegate.clear(); - } - - @Override - public Set keySet() { - return delegate.keySet(); - } - - @Override - public Collection values() { - return delegate.values(); - } - - @Override - public Set> entrySet() { - return delegate.entrySet(); - } - } - -} diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/OzoneClientProducer.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/OzoneClientProducer.java index 364d26343e9a..1fb396a79c29 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/OzoneClientProducer.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/OzoneClientProducer.java @@ -21,26 +21,29 @@ import javax.enterprise.context.RequestScoped; import javax.enterprise.inject.Produces; import javax.inject.Inject; +import javax.ws.rs.container.ContainerRequestContext; +import javax.ws.rs.core.Context; import java.io.IOException; -import java.net.URISyntaxException; +import java.nio.charset.Charset; import java.security.PrivilegedExceptionAction; -import com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.io.Text; import org.apache.hadoop.ozone.OzoneSecurityUtil; import org.apache.hadoop.ozone.client.OzoneClient; import org.apache.hadoop.ozone.client.OzoneClientFactory; import org.apache.hadoop.ozone.s3.exception.OS3Exception; +import org.apache.hadoop.ozone.s3.signature.SignatureInfo; +import org.apache.hadoop.ozone.s3.signature.SignatureProcessor; +import org.apache.hadoop.ozone.s3.signature.StringToSignProducer; import org.apache.hadoop.ozone.security.OzoneTokenIdentifier; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.token.Token; +import com.google.common.annotations.VisibleForTesting; import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMTokenProto.Type.S3AUTHINFO; -import static org.apache.hadoop.ozone.s3.SignatureProcessor.UTF_8; import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.INTERNAL_ERROR; import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.MALFORMED_HEADER; -import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.S3_AUTHINFO_CREATION_ERROR; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -50,12 +53,14 @@ @RequestScoped public class OzoneClientProducer { + private static final Charset UTF_8 = Charset.forName("utf-8"); + private final static Logger LOG = LoggerFactory.getLogger(OzoneClientProducer.class); private OzoneClient client; @Inject - private SignatureProcessor signatureParser; + private SignatureProcessor signatureProcessor; @Inject private OzoneConfiguration ozoneConfiguration; @@ -66,15 +71,17 @@ public class OzoneClientProducer { @Inject private String omServiceID; + @Context + private ContainerRequestContext context; @Produces public OzoneClient createClient() throws OS3Exception, IOException { client = getClient(ozoneConfiguration); return client; } - + @PreDestroy - public void destory() throws IOException { + public void destroy() throws IOException { client.close(); } @@ -82,35 +89,38 @@ private OzoneClient getClient(OzoneConfiguration config) throws OS3Exception { OzoneClient ozoneClient = null; try { - // Check if any error occurred during creation of signatureProcessor. - if (signatureParser.getException() != null) { - throw signatureParser.getException(); + SignatureInfo signatureInfo = signatureProcessor.parseSignature(); + + String stringToSign = ""; + if (signatureInfo.getAwsAccessId() != null + && signatureInfo.getAwsAccessId().length() > 0) { + stringToSign = + StringToSignProducer.createSignatureBase(signatureInfo, context); } - String awsAccessId = signatureParser.getAwsAccessId(); + + String awsAccessId = signatureInfo.getAwsAccessId(); validateAccessId(awsAccessId); UserGroupInformation remoteUser = UserGroupInformation.createRemoteUser(awsAccessId); if (OzoneSecurityUtil.isSecurityEnabled(config)) { LOG.debug("Creating s3 auth info for client."); - try { - OzoneTokenIdentifier identifier = new OzoneTokenIdentifier(); - identifier.setTokenType(S3AUTHINFO); - identifier.setStrToSign(signatureParser.getStringToSign()); - identifier.setSignature(signatureParser.getSignature()); - identifier.setAwsAccessId(awsAccessId); - identifier.setOwner(new Text(awsAccessId)); - if (LOG.isTraceEnabled()) { - LOG.trace("Adding token for service:{}", omService); - } - Token token = new Token(identifier.getBytes(), - identifier.getSignature().getBytes(UTF_8), - identifier.getKind(), - omService); - remoteUser.addToken(token); - } catch (OS3Exception | URISyntaxException ex) { - throw S3_AUTHINFO_CREATION_ERROR; + + OzoneTokenIdentifier identifier = new OzoneTokenIdentifier(); + identifier.setTokenType(S3AUTHINFO); + identifier.setStrToSign(stringToSign); + identifier.setSignature(signatureInfo.getSignature()); + identifier.setAwsAccessId(awsAccessId); + identifier.setOwner(new Text(awsAccessId)); + if (LOG.isTraceEnabled()) { + LOG.trace("Adding token for service:{}", omService); } + Token token = new Token(identifier.getBytes(), + identifier.getSignature().getBytes(UTF_8), + identifier.getKind(), + omService); + remoteUser.addToken(token); + } ozoneClient = remoteUser.doAs((PrivilegedExceptionAction)() -> { @@ -130,14 +140,17 @@ private OzoneClient getClient(OzoneConfiguration config) } catch (Throwable t) { // For any other critical errors during object creation throw Internal // error. - if (LOG.isDebugEnabled()) { - LOG.debug("Error during Client Creation: ", t); - } + // if (LOG.isDebugEnabled()) { + LOG.error("Error during Client Creation: ", t); + // } throw INTERNAL_ERROR; } return ozoneClient; } + private void getSignatureInfo() { + } + // ONLY validate aws access id when needed. private void validateAccessId(String awsAccessId) throws Exception { if (awsAccessId == null || awsAccessId.equals("")) { @@ -151,7 +164,7 @@ public void setOzoneConfiguration(OzoneConfiguration config) { } @VisibleForTesting - public void setSignatureParser(SignatureProcessor signatureParser) { - this.signatureParser = signatureParser; + public void setSignatureParser(SignatureProcessor awsSignatureProcessor) { + this.signatureProcessor = awsSignatureProcessor; } } diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java index 789bb4511027..aa86df04c13e 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java @@ -57,6 +57,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; + /** * Bucket level rest endpoints. */ diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java index b60519df0be5..1e40e920074f 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java @@ -29,7 +29,6 @@ import org.apache.hadoop.ozone.client.OzoneVolume; import org.apache.hadoop.ozone.om.exceptions.OMException; import org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes; -import org.apache.hadoop.ozone.s3.SignatureProcessor; import org.apache.hadoop.ozone.s3.exception.OS3Exception; import org.apache.hadoop.ozone.s3.exception.S3ErrorTable; @@ -43,9 +42,6 @@ public class EndpointBase { @Inject private OzoneClient client; - @Inject - private SignatureProcessor signatureProcessor; - protected OzoneBucket getBucket(OzoneVolume volume, String bucketName) throws OS3Exception, IOException { OzoneBucket bucket; @@ -156,16 +152,6 @@ private Iterator iterateBuckets( } } - public SignatureProcessor getSignatureProcessor() { - return signatureProcessor; - } - - @VisibleForTesting - public void setSignatureProcessor( - SignatureProcessor signatureProcessor) { - this.signatureProcessor = signatureProcessor; - } - @VisibleForTesting public void setClient(OzoneClient ozoneClient) { this.client = ozoneClient; diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/exception/OS3Exception.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/exception/OS3Exception.java index 12e6cfc4bc69..810aa2085f4a 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/exception/OS3Exception.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/exception/OS3Exception.java @@ -17,6 +17,12 @@ */ package org.apache.hadoop.ozone.s3.exception; +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlElement; +import javax.xml.bind.annotation.XmlRootElement; +import javax.xml.bind.annotation.XmlTransient; + import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.SerializationFeature; import com.fasterxml.jackson.dataformat.xml.XmlMapper; @@ -24,12 +30,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import javax.xml.bind.annotation.XmlAccessType; -import javax.xml.bind.annotation.XmlAccessorType; -import javax.xml.bind.annotation.XmlElement; -import javax.xml.bind.annotation.XmlTransient; -import javax.xml.bind.annotation.XmlRootElement; - /** * This class represents exceptions raised from Ozone S3 service. @@ -38,7 +38,7 @@ */ @XmlRootElement(name = "Error") @XmlAccessorType(XmlAccessType.NONE) -public class OS3Exception extends Exception { +public class OS3Exception extends Exception { private static final Logger LOG = LoggerFactory.getLogger(OS3Exception.class); private static ObjectMapper mapper; diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/exception/S3ErrorTable.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/exception/S3ErrorTable.java index 432b582bc86f..d0a5bb98d489 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/exception/S3ErrorTable.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/exception/S3ErrorTable.java @@ -116,7 +116,7 @@ public static OS3Exception newError(OS3Exception e, String resource) { OS3Exception err = new OS3Exception(e.getCode(), e.getErrorMessage(), e.getHttpCode()); err.setResource(resource); - LOG.error(err.toXml()); + LOG.error(err.toXml(), e); return err; } } diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/AWSSignatureProcessor.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/AWSSignatureProcessor.java new file mode 100644 index 000000000000..8bb3c28ab2c4 --- /dev/null +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/AWSSignatureProcessor.java @@ -0,0 +1,199 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.ozone.s3.signature; + +import javax.enterprise.context.RequestScoped; +import javax.ws.rs.container.ContainerRequestContext; +import javax.ws.rs.core.Context; +import javax.ws.rs.core.MultivaluedMap; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.apache.hadoop.ozone.s3.HeaderPreprocessor; +import org.apache.hadoop.ozone.s3.exception.OS3Exception; + +import com.google.common.annotations.VisibleForTesting; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Parser to process AWS V2 & V4 auth request. Creates string to sign and auth + * header. For more details refer to AWS documentation https://docs.aws + * .amazon.com/general/latest/gr/sigv4-create-canonical-request.html. + **/ +@RequestScoped +public class AWSSignatureProcessor implements SignatureProcessor { + + private final static Logger LOG = + LoggerFactory.getLogger(AWSSignatureProcessor.class); + String AUTHORIZATION_HEADER = "Authorization"; + + @Context + private ContainerRequestContext context; + + private SignatureInfo signatureInfo; + + public SignatureInfo parseSignature() throws OS3Exception { + + LowerCaseKeyStringMap headers = + LowerCaseKeyStringMap.fromHeaderMap(context.getHeaders()); + + String authHeader = headers.get(AUTHORIZATION_HEADER); + + List signatureParsers = new ArrayList<>(); + signatureParsers.add(new AuthorizationV4HeaderParser(authHeader)); + signatureParsers.add(new AuthorizationV4QueryParser(context.getUriInfo().getQueryParameters())); + + signatureInfo = null; + for (SignatureParser parser : signatureParsers) { + signatureInfo = parser.parseSignature(); + if (signatureInfo != null) { + break; + } + } + if (signatureInfo == null) { + signatureInfo = new SignatureInfo( + "", "", "", "", "", "" + ); + } + return signatureInfo; + } + + @VisibleForTesting + public void setContext(ContainerRequestContext context) { + this.context = context; + } + + /** + * A simple map which forces lower case key usage. + */ + public static class LowerCaseKeyStringMap implements Map { + + private HashMap delegate; + + public LowerCaseKeyStringMap( + HashMap delegate + ) { + this.delegate = delegate; + } + + public static LowerCaseKeyStringMap fromHeaderMap( + MultivaluedMap rawHeaders + ) { + + //header map is MUTABLE. It's better to save it here. (with lower case + // keys!!!) + final LowerCaseKeyStringMap headers = + new LowerCaseKeyStringMap(new HashMap<>()); + + for (Entry> headerEntry : rawHeaders.entrySet()) { + if (0 < headerEntry.getValue().size()) { + String headerKey = headerEntry.getKey(); + if (headers.containsKey(headerKey)) { + //multiple headers from the same type are combined + headers.put(headerKey, + headers.get(headerKey) + "," + headerEntry.getValue().get(0)); + } else { + headers.put(headerKey, headerEntry.getValue().get(0)); + } + } + } + + //in case of the HeaderPreprocessor executed before us, let's restore the + // original content type. + if (headers.containsKey(HeaderPreprocessor.ORIGINAL_CONTENT_TYPE)) { + headers.put(HeaderPreprocessor.CONTENT_TYPE, + headers.get(HeaderPreprocessor.ORIGINAL_CONTENT_TYPE)); + } + + if (LOG.isTraceEnabled()) { + headers.keySet().forEach(k -> LOG.trace("Header:{},value:{}", k, + headers.get(k))); + } + return headers; + } + + @Override + public int size() { + return delegate.size(); + } + + @Override + public boolean isEmpty() { + return delegate.isEmpty(); + } + + @Override + public boolean containsKey(Object key) { + return delegate.containsKey(key.toString().toLowerCase()); + } + + @Override + public boolean containsValue(Object value) { + return delegate.containsValue(value); + } + + @Override + public String get(Object key) { + return delegate.get(key.toString().toLowerCase()); + } + + @Override + public String put(String key, String value) { + return delegate.put(key.toLowerCase(), value); + } + + @Override + public String remove(Object key) { + return delegate.remove(key.toString()); + } + + @Override + public void putAll(Map m) { + for (Entry entry : m.entrySet()) { + put(entry.getKey().toLowerCase(), entry.getValue()); + } + } + + @Override + public void clear() { + delegate.clear(); + } + + @Override + public Set keySet() { + return delegate.keySet(); + } + + @Override + public Collection values() { + return delegate.values(); + } + + @Override + public Set> entrySet() { + return delegate.entrySet(); + } + } + +} diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/header/AuthorizationHeaderV2.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/AuthorizationHeaderV2.java similarity index 98% rename from hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/header/AuthorizationHeaderV2.java rename to hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/AuthorizationHeaderV2.java index fe096cee9ff5..5f7a2e46138d 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/header/AuthorizationHeaderV2.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/AuthorizationHeaderV2.java @@ -16,12 +16,12 @@ * limitations under the License. */ -package org.apache.hadoop.ozone.s3.header; +package org.apache.hadoop.ozone.s3.signature; -import com.google.common.base.Preconditions; import org.apache.hadoop.ozone.s3.exception.OS3Exception; import org.apache.hadoop.ozone.s3.exception.S3ErrorTable; +import com.google.common.base.Preconditions; import static org.apache.commons.lang3.StringUtils.isBlank; /** @@ -30,9 +30,13 @@ public class AuthorizationHeaderV2 { public final static String IDENTIFIER = "AWS"; + private String authHeader; + private String identifier; + private String accessKeyID; + private String signature; public AuthorizationHeaderV2(String auth) throws OS3Exception { diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/header/AuthorizationHeaderV4.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/AuthorizationV4HeaderParser.java similarity index 54% rename from hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/header/AuthorizationHeaderV4.java rename to hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/AuthorizationV4HeaderParser.java index 1e48689a86c7..ec10ad6c95c4 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/header/AuthorizationHeaderV4.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/AuthorizationV4HeaderParser.java @@ -1,86 +1,61 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with this - * work for additional information regarding copyright ownership. The ASF - * licenses this file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package org.apache.hadoop.ozone.s3.header; +package org.apache.hadoop.ozone.s3.signature; +import java.time.LocalDate; +import java.util.Collection; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Preconditions; -import org.apache.commons.codec.DecoderException; -import org.apache.commons.codec.binary.Hex; import org.apache.hadoop.ozone.s3.exception.OS3Exception; import org.apache.hadoop.ozone.s3.exception.S3ErrorTable; import org.apache.hadoop.util.StringUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.time.LocalDate; -import java.util.Collection; +import com.google.common.annotations.VisibleForTesting; import static java.time.temporal.ChronoUnit.DAYS; +import org.apache.commons.codec.DecoderException; +import org.apache.commons.codec.binary.Hex; import static org.apache.commons.lang3.StringUtils.isEmpty; import static org.apache.commons.lang3.StringUtils.isNotEmpty; import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.MALFORMED_HEADER; -import static org.apache.hadoop.ozone.s3.AWSSignatureProcessor.AWS4_SIGNING_ALGORITHM; -import static org.apache.hadoop.ozone.s3.AWSSignatureProcessor.DATE_FORMATTER; +import static org.apache.hadoop.ozone.s3.signature.SignatureProcessor.AWS4_SIGNING_ALGORITHM; +import static org.apache.hadoop.ozone.s3.signature.SignatureProcessor.DATE_FORMATTER; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -/** - * S3 Authorization header. - * Ref: https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-auth-using - * -authorization-header.html - */ -public class AuthorizationHeaderV4 { +public class AuthorizationV4HeaderParser implements SignatureParser { - private final static Logger LOG = LoggerFactory.getLogger( - AuthorizationHeaderV4.class); + private static final Logger LOG = + LoggerFactory.getLogger(AuthorizationV4HeaderParser.class); private final static String CREDENTIAL = "Credential="; private final static String SIGNEDHEADERS = "SignedHeaders="; private final static String SIGNATURE = "Signature="; private String authHeader; - private String algorithm; - private String credential; - private String signedHeadersStr; - private String signature; - private Credential credentialObj; - private Collection signedHeaders; - /** - * Construct AuthorizationHeader object. - * @param header - */ - public AuthorizationHeaderV4(String header) throws OS3Exception { - Preconditions.checkNotNull(header); - this.authHeader = header; - parseAuthHeader(); + public AuthorizationV4HeaderParser(String authHeader) { + if (authHeader != null) { + String[] split = authHeader.split(" "); + if (!split[0].equals(AuthorizationHeaderV2.IDENTIFIER)) { + this.authHeader = authHeader; + } + } } /** * This method parses authorization header. + *

+ * Authorization Header sample: + * AWS4-HMAC-SHA256 Credential=AKIAJWFJK62WUTKNFJJA/20181009/us-east-1/s3 + * /aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, + * Signature + * =db81b057718d7c1b3b8dffa29933099551c51d787b3b13b9e0f9ebed45982bf2 * - * Authorization Header sample: - * AWS4-HMAC-SHA256 Credential=AKIAJWFJK62WUTKNFJJA/20181009/us-east-1/s3 - * /aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, - * Signature=db81b057718d7c1b3b8dffa29933099551c51d787b3b13b9e0f9ebed45982bf2 * @throws OS3Exception */ @SuppressWarnings("StringSplitter") - public void parseAuthHeader() throws OS3Exception { + @Override + public SignatureInfo parseSignature() throws OS3Exception { + if (authHeader == null) { + return null; + } int firstSep = authHeader.indexOf(' '); if (firstSep < 0) { throw S3ErrorTable.newError(MALFORMED_HEADER, authHeader); @@ -93,29 +68,38 @@ public void parseAuthHeader() throws OS3Exception { throw S3ErrorTable.newError(MALFORMED_HEADER, authHeader); } - algorithm = authHeader.substring(0, firstSep); - validateAlgorithm(); - credential = split[0]; - signedHeadersStr = split[1]; - signature = split[2]; - validateCredentials(); - validateSignedHeaders(); - validateSignature(); - + String algorithm = parseAlgorithm(authHeader.substring(0, firstSep)); + Credential credentialObj = parseCredentials(split[0]); + String signedHeaders = parseSignedHeaders(split[1]); + String signature = parseSignature(split[2]); + return new SignatureInfo( + credentialObj.getDate(), + credentialObj.getAccessKeyID(), + signature, + signedHeaders, + String.format("%s/%s/%s/%s", credentialObj.getDate(), + credentialObj.getAwsRegion(), credentialObj.getAwsService(), + credentialObj.getAwsRequest()), + algorithm + ); } /** * Validate Signed headers. - * */ - private void validateSignedHeaders() throws OS3Exception { + */ + private String parseSignedHeaders(String signedHeadersStr) + throws OS3Exception { if (isNotEmpty(signedHeadersStr) && signedHeadersStr.startsWith(SIGNEDHEADERS)) { - signedHeadersStr = signedHeadersStr.substring(SIGNEDHEADERS.length()); - signedHeaders = StringUtils.getStringCollection(signedHeadersStr, ";"); + String parsedSignedHeaders = + signedHeadersStr.substring(SIGNEDHEADERS.length()); + Collection signedHeaders = + StringUtils.getStringCollection(signedHeadersStr, ";"); if (signedHeaders.size() == 0) { LOG.error("No signed headers found. Authheader:{}", authHeader); throw S3ErrorTable.newError(MALFORMED_HEADER, authHeader); } + return parsedSignedHeaders; } else { LOG.error("No signed headers found. Authheader:{}", authHeader); throw S3ErrorTable.newError(MALFORMED_HEADER, authHeader); @@ -124,21 +108,22 @@ private void validateSignedHeaders() throws OS3Exception { /** * Validate signature. - * */ - private void validateSignature() throws OS3Exception { + */ + private String parseSignature(String signature) throws OS3Exception { if (signature.startsWith(SIGNATURE)) { - signature = signature.substring(SIGNATURE.length()); - if (isEmpty(signature)) { + String parsedSignature = signature.substring(SIGNATURE.length()); + if (isEmpty(parsedSignature)) { LOG.error("Signature can't be empty: {}", signature); throw S3ErrorTable.newError(MALFORMED_HEADER, authHeader); } try { - Hex.decodeHex(signature); + Hex.decodeHex(parsedSignature); } catch (DecoderException e) { LOG.error("Signature:{} should be in hexa-decimal encoding.", signature); throw S3ErrorTable.newError(MALFORMED_HEADER, authHeader); } + return parsedSignature; } else { LOG.error("No signature found: {}", signature); throw S3ErrorTable.newError(MALFORMED_HEADER, authHeader); @@ -147,8 +132,10 @@ private void validateSignature() throws OS3Exception { /** * Validate credentials. - * */ - private void validateCredentials() throws OS3Exception { + */ + private Credential parseCredentials(String credential) + throws OS3Exception { + Credential credentialObj = null; if (isNotEmpty(credential) && credential.startsWith(CREDENTIAL)) { credential = credential.substring(CREDENTIAL.length()); // Parse credential. Other parts of header are not validated yet. When @@ -159,7 +146,8 @@ private void validateCredentials() throws OS3Exception { } if (credentialObj.getAccessKeyID().isEmpty()) { - LOG.error("AWS access id shouldn't be empty. credential:{}", credential); + LOG.error("AWS access id shouldn't be empty. credential:{}", + credential); throw S3ErrorTable.newError(MALFORMED_HEADER, authHeader); } if (credentialObj.getAwsRegion().isEmpty()) { @@ -178,82 +166,36 @@ private void validateCredentials() throws OS3Exception { // Date should not be empty and within valid range. if (!credentialObj.getDate().isEmpty()) { - validateDateRange(); + validateDateRange(credentialObj); } else { LOG.error("AWS date shouldn't be empty. credential:{}", credential); throw S3ErrorTable.newError(MALFORMED_HEADER, authHeader); } + return credentialObj; } @VisibleForTesting - public void validateDateRange() throws OS3Exception { + public void validateDateRange(Credential credentialObj) throws OS3Exception { LocalDate date = LocalDate.parse(credentialObj.getDate(), DATE_FORMATTER); LocalDate now = LocalDate.now(); if (date.isBefore(now.minus(1, DAYS)) || date.isAfter(now.plus(1, DAYS))) { LOG.error("AWS date not in valid range. Date:{} should not be older " + - "than 1 day(i.e yesterday) and greater than 1 day(i.e " + - "tomorrow).", - getDate()); + "than 1 day(i.e yesterday) and greater than 1 day(i.e " + + "tomorrow).", date); throw S3ErrorTable.newError(MALFORMED_HEADER, authHeader); } } /** * Validate if algorithm is in expected format. - * */ - private void validateAlgorithm() throws OS3Exception { + */ + private String parseAlgorithm(String algorithm) throws OS3Exception { if (isEmpty(algorithm) || !algorithm.equals(AWS4_SIGNING_ALGORITHM)) { LOG.error("Unexpected hash algorithm. Algo:{}", algorithm); throw S3ErrorTable.newError(MALFORMED_HEADER, authHeader); } - } - - public String getAuthHeader() { - return authHeader; - } - - public String getAlgorithm() { return algorithm; } - public String getCredential() { - return credential; - } - - public String getSignedHeaderString() { - return signedHeadersStr; - } - - public String getSignature() { - return signature; - } - - public String getAccessKeyID() { - return credentialObj.getAccessKeyID(); - } - - public String getDate() { - return credentialObj.getDate(); - } - - public String getAwsRegion() { - return credentialObj.getAwsRegion(); - } - - public String getAwsService() { - return credentialObj.getAwsService(); - } - - public String getAwsRequest() { - return credentialObj.getAwsRequest(); - } - - public Collection getSignedHeaders() { - return signedHeaders; - } - - public Credential getCredentialObj() { - return credentialObj; - } } diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/AuthorizationV4QueryParser.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/AuthorizationV4QueryParser.java new file mode 100644 index 000000000000..d0fe4e82548a --- /dev/null +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/AuthorizationV4QueryParser.java @@ -0,0 +1,48 @@ +package org.apache.hadoop.ozone.s3.signature; + +import javax.ws.rs.core.MultivaluedMap; + +import org.apache.hadoop.ozone.s3.exception.OS3Exception; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Parser for getting auth info from query parameters. + *

+ * See: https://docs.aws.amazon + * .com/AmazonS3/latest/API/sigv4-query-string-auth.html + */ +public class AuthorizationV4QueryParser implements SignatureParser { + + private static final Logger LOG = + LoggerFactory.getLogger(AuthorizationV4QueryParser.class); + + private final MultivaluedMap queryParameters; + + public AuthorizationV4QueryParser(MultivaluedMap queryParameters) { + this.queryParameters = queryParameters; + } + + @Override + public SignatureInfo parseSignature() throws OS3Exception { + + if (!queryParameters.containsKey("X-Amz-Signature")) { + return null; + } + + Credential credential = + new Credential(queryParameters.getFirst("X-Amz-Credential")); + + return new SignatureInfo( + queryParameters.getFirst("X-Amz-Date"), + credential.getAccessKeyID(), + queryParameters.getFirst("X-Amz-Signature"), + queryParameters.getFirst("X-Amz-SignedHeaders"), + String.format("%s/%s/%s/%s", credential.getDate(), + credential.getAwsRegion(), credential.getAwsService(), + credential.getAwsRequest()), + queryParameters.getFirst("X-Amz-Algorithm") + ); + } +} diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/header/Credential.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/Credential.java similarity index 98% rename from hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/header/Credential.java rename to hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/Credential.java index 883980af7ec6..504e42b8357c 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/header/Credential.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/Credential.java @@ -15,7 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.hadoop.ozone.s3.header; +package org.apache.hadoop.ozone.s3.signature; import org.apache.hadoop.ozone.s3.exception.OS3Exception; import org.apache.hadoop.ozone.s3.exception.S3ErrorTable; diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/SignatureInfo.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/SignatureInfo.java new file mode 100644 index 000000000000..fb29e7699a68 --- /dev/null +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/SignatureInfo.java @@ -0,0 +1,56 @@ +package org.apache.hadoop.ozone.s3.signature; + +public class SignatureInfo { + + private String date; + + private String awsAccessId; + + private String signature; + + private String signedHeaders; + + private String credentialScope; + + private String algorithm; + + public SignatureInfo( + String date, + String awsAccessId, + String signature, + String signedHeaders, + String credentialScope, + String algorithm + ) { + this.date = date; + this.awsAccessId = awsAccessId; + this.signature = signature; + this.signedHeaders = signedHeaders; + this.credentialScope = credentialScope; + this.algorithm = algorithm; + } + + public String getAwsAccessId() { + return awsAccessId; + } + + public String getSignature() { + return signature; + } + + public String getDate() { + return date; + } + + public String getSignedHeaders() { + return signedHeaders; + } + + public String getCredentialScope() { + return credentialScope; + } + + public String getAlgorithm() { + return algorithm; + } +} diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/SignatureParser.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/SignatureParser.java new file mode 100644 index 000000000000..f6805257318b --- /dev/null +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/SignatureParser.java @@ -0,0 +1,10 @@ +package org.apache.hadoop.ozone.s3.signature; + +import org.apache.hadoop.ozone.s3.exception.OS3Exception; + +public interface SignatureParser { + + String AUTHORIZATION_HEADER = "Authorization"; + + SignatureInfo parseSignature() throws OS3Exception; +} diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/SignatureProcessor.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/SignatureProcessor.java similarity index 61% rename from hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/SignatureProcessor.java rename to hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/SignatureProcessor.java index e3cb6afefbbb..c5f1393704de 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/SignatureProcessor.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/SignatureProcessor.java @@ -15,52 +15,30 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.hadoop.ozone.s3; +package org.apache.hadoop.ozone.s3.signature; -import java.nio.charset.Charset; -import java.time.ZoneOffset; import java.time.format.DateTimeFormatter; +import org.apache.hadoop.ozone.s3.exception.OS3Exception; + /** * Parser to request auth parser for http request. */ public interface SignatureProcessor { - String UNSIGNED_PAYLOAD = "UNSIGNED-PAYLOAD"; - String NEWLINE = "\n"; String CONTENT_TYPE = "content-type"; - String X_AMAZ_DATE = "X-Amz-Date"; + String CONTENT_MD5 = "content-md5"; - String AUTHORIZATION_HEADER = "Authorization"; - Charset UTF_8 = Charset.forName("utf-8"); - String X_AMZ_CONTENT_SHA256 = "X-Amz-Content-SHA256"; - String HOST = "host"; String AWS4_SIGNING_ALGORITHM = "AWS4-HMAC-SHA256"; - /** - * Seconds in a week, which is the max expiration time Sig-v4 accepts. - */ - long PRESIGN_URL_MAX_EXPIRATION_SECONDS = - 60 * 60 * 24 * 7; - String HOST_HEADER = "Host"; DateTimeFormatter DATE_FORMATTER = DateTimeFormatter.ofPattern("yyyyMMdd"); - DateTimeFormatter TIME_FORMATTER = - DateTimeFormatter.ofPattern("yyyyMMdd'T'HHmmss'Z'") - .withZone(ZoneOffset.UTC); - /** * API to return string to sign. */ - String getStringToSign() throws Exception; - - String getSignature(); - - String getAwsAccessId(); - - Exception getException(); + SignatureInfo parseSignature() throws OS3Exception; } diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/StringToSignProducer.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/StringToSignProducer.java new file mode 100644 index 000000000000..896146263c26 --- /dev/null +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/StringToSignProducer.java @@ -0,0 +1,282 @@ +package org.apache.hadoop.ozone.s3.signature; + +import javax.ws.rs.container.ContainerRequestContext; +import javax.ws.rs.core.MultivaluedMap; +import java.io.UnsupportedEncodingException; +import java.net.InetAddress; +import java.net.URI; +import java.net.URISyntaxException; +import java.net.URLEncoder; +import java.net.UnknownHostException; +import java.nio.charset.Charset; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.time.LocalDate; +import java.time.ZoneOffset; +import java.time.format.DateTimeFormatter; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.apache.hadoop.ozone.s3.exception.OS3Exception; +import org.apache.hadoop.ozone.s3.signature.AWSSignatureProcessor.LowerCaseKeyStringMap; +import org.apache.hadoop.util.StringUtils; + +import com.google.common.annotations.VisibleForTesting; +import static java.time.temporal.ChronoUnit.SECONDS; +import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.S3_AUTHINFO_CREATION_ERROR; +import org.apache.kerby.util.Hex; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class StringToSignProducer { + + public static final String X_AMZ_CONTENT_SHA256 = "X-Amz-Content-SHA256"; + public static final String X_AMAZ_DATE = "X-Amz-Date"; + private static final Logger LOG = + LoggerFactory.getLogger(StringToSignProducer.class); + private static final Charset UTF_8 = Charset.forName("utf-8"); + private static final String NEWLINE = "\n"; + private static final String HOST = "host"; + private static final String UNSIGNED_PAYLOAD = "UNSIGNED-PAYLOAD"; + /** + * Seconds in a week, which is the max expiration time Sig-v4 accepts. + */ + private static final long PRESIGN_URL_MAX_EXPIRATION_SECONDS = + 60 * 60 * 24 * 7; + private static final DateTimeFormatter TIME_FORMATTER = + DateTimeFormatter.ofPattern("yyyyMMdd'T'HHmmss'Z'") + .withZone(ZoneOffset.UTC); + + public static String createSignatureBase( + SignatureInfo signatureInfo, + ContainerRequestContext context + ) throws Exception { + return createSignatureBase(signatureInfo, + context.getUriInfo().getRequestUri().getScheme(), + context.getMethod(), + context.getUriInfo().getRequestUri().getPath(), + LowerCaseKeyStringMap.fromHeaderMap(context.getHeaders()), + context.getUriInfo().getQueryParameters()); + } + + public static String createSignatureBase( + SignatureInfo signatureInfo, + String scheme, + String method, + String uri, + Map headers, + MultivaluedMap queryParams + ) throws Exception { + StringBuilder strToSign = new StringBuilder(); + // According to AWS sigv4 documentation, authorization header should be + // in following format. + // Authorization: algorithm Credential=access key ID/credential scope, + // SignedHeaders=SignedHeaders, Signature=signature + + // Construct String to sign in below format. + // StringToSign = + // Algorithm + \n + + // RequestDateTime + \n + + // CredentialScope + \n + + // HashedCanonicalRequest + String credentialScope, canonicalRequest; + credentialScope = signatureInfo.getCredentialScope(); + + // If the absolute path is empty, use a forward slash (/) + uri = (uri.trim().length() > 0) ? uri : "/"; + // Encode URI and preserve forward slashes + strToSign.append(signatureInfo.getAlgorithm() + NEWLINE); + strToSign.append(headers.get(X_AMAZ_DATE) + NEWLINE); + strToSign.append(credentialScope + NEWLINE); + + canonicalRequest = buildCanonicalRequest( + scheme, + method, + uri, + signatureInfo.getSignedHeaders(), + headers, + queryParams); + strToSign.append(hash(canonicalRequest)); + if (LOG.isDebugEnabled()) { + LOG.debug("canonicalRequest:[{}]", canonicalRequest); + } + + LOG.debug("StringToSign:[{}]", strToSign); + return strToSign.toString(); + } + + public static String hash(String payload) throws NoSuchAlgorithmException { + MessageDigest md = MessageDigest.getInstance("SHA-256"); + md.update(payload.getBytes(UTF_8)); + return Hex.encode(md.digest()).toLowerCase(); + } + + @VisibleForTesting + public static String buildCanonicalRequest( + String schema, + String method, + String uri, + String signedHeaders, + Map headers, + MultivaluedMap queryParams + ) throws OS3Exception { + + Iterable parts = split("/", uri); + List encParts = new ArrayList<>(); + for (String p : parts) { + encParts.add(urlEncode(p)); + } + String canonicalUri = join("/", encParts); + + String canonicalQueryStr = getQueryParamString(queryParams); + + StringBuilder canonicalHeaders = new StringBuilder(); + + for (String header : StringUtils.getStringCollection(signedHeaders, ";")) { + canonicalHeaders.append(header.toLowerCase()); + canonicalHeaders.append(":"); + if (headers.containsKey(header)) { + String headerValue = headers.get(header); + canonicalHeaders.append(headerValue); + canonicalHeaders.append(NEWLINE); + + // Set for testing purpose only to skip date and host validation. + validateSignedHeader(schema, header, headerValue); + + } else { + throw new RuntimeException("Header " + header + " not present in " + + "request but requested to be signed."); + } + } + + String payloadHash; + if (UNSIGNED_PAYLOAD.equals( + headers.get(X_AMZ_CONTENT_SHA256))) { + payloadHash = UNSIGNED_PAYLOAD; + } else { + payloadHash = headers.get(X_AMZ_CONTENT_SHA256); + } + String canonicalRequest = method + NEWLINE + + canonicalUri + NEWLINE + + canonicalQueryStr + NEWLINE + + canonicalHeaders + NEWLINE + + signedHeaders + NEWLINE + + payloadHash; + + return canonicalRequest; + } + + /** + * String join that also works with empty strings. + * + * @return joined string + */ + private static String join(String glue, List parts) { + StringBuilder result = new StringBuilder(); + boolean addSeparator = false; + for (String p : parts) { + if (addSeparator) { + result.append(glue); + } + result.append(p); + addSeparator = true; + } + return result.toString(); + } + + /** + * Returns matching strings. + * + * @param regex Regular expression to split by + * @param whole The string to split + * @return pieces + */ + private static Iterable split(String regex, String whole) { + Pattern p = Pattern.compile(regex); + Matcher m = p.matcher(whole); + List result = new ArrayList<>(); + int pos = 0; + while (m.find()) { + result.add(whole.substring(pos, m.start())); + pos = m.end(); + } + result.add(whole.substring(pos)); + return result; + } + + private static String urlEncode(String str) { + try { + + return URLEncoder.encode(str, UTF_8.name()) + .replaceAll("\\+", "%20") + .replaceAll("%7E", "~"); + } catch (UnsupportedEncodingException e) { + throw new RuntimeException(e); + } + } + + private static String getQueryParamString(MultivaluedMap queryMap) { + List params = new ArrayList<>(queryMap.keySet()); + + // Sort by name, then by value + Collections.sort(params, (o1, o2) -> o1.equals(o2) ? + queryMap.getFirst(o1).compareTo(queryMap.getFirst(o2)) : + o1.compareTo(o2)); + + StringBuilder result = new StringBuilder(); + for (String p : params) { + if (result.length() > 0) { + result.append("&"); + } + result.append(urlEncode(p)); + result.append('='); + + result.append(urlEncode(queryMap.getFirst(p))); + } + return result.toString(); + } + + @VisibleForTesting + static void validateSignedHeader( + String schema, + String header, + String headerValue + ) + throws OS3Exception { + switch (header) { + case HOST: + try { + URI hostUri = new URI(schema + "://" + headerValue); + InetAddress.getByName(hostUri.getHost()); + // TODO: Validate if current request is coming from same host. + } catch (UnknownHostException | URISyntaxException e) { + LOG.error("Host value mentioned in signed header is not valid. " + + "Host:{}", headerValue); + throw S3_AUTHINFO_CREATION_ERROR; + } + break; + case X_AMAZ_DATE: + LocalDate date = LocalDate.parse(headerValue, TIME_FORMATTER); + LocalDate now = LocalDate.now(); + if (date.isBefore(now.minus(PRESIGN_URL_MAX_EXPIRATION_SECONDS, SECONDS)) + || date.isAfter(now.plus(PRESIGN_URL_MAX_EXPIRATION_SECONDS, + SECONDS))) { + LOG.error("AWS date not in valid range. Request timestamp:{} should " + + "not be older than {} seconds.", headerValue, + PRESIGN_URL_MAX_EXPIRATION_SECONDS); + throw S3_AUTHINFO_CREATION_ERROR; + } + break; + case X_AMZ_CONTENT_SHA256: + // TODO: Construct request payload and match HEX(SHA256(requestPayload)) + break; + default: + break; + } + } + +} diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/header/package-info.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/package-info.java similarity index 95% rename from hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/header/package-info.java rename to hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/package-info.java index 40bc78bd337b..63f18a65a391 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/header/package-info.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/package-info.java @@ -19,4 +19,4 @@ /** * This package contains Ozone S3 Authorization header. */ -package org.apache.hadoop.ozone.s3.header; \ No newline at end of file +package org.apache.hadoop.ozone.s3.signature; \ No newline at end of file diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/TestAWSSignatureProcessor.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/TestAWSSignatureProcessor.java deleted file mode 100644 index 239e2857957b..000000000000 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/TestAWSSignatureProcessor.java +++ /dev/null @@ -1,141 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.ozone.s3; - -import javax.ws.rs.container.ContainerRequestContext; -import javax.ws.rs.core.MultivaluedHashMap; -import javax.ws.rs.core.MultivaluedMap; -import javax.ws.rs.core.UriInfo; -import java.net.URI; - -import org.apache.hadoop.ozone.s3.exception.OS3Exception; -import org.apache.hadoop.ozone.s3.header.AuthorizationHeaderV2; -import org.apache.hadoop.ozone.s3.header.AuthorizationHeaderV4; - -import org.junit.Assert; -import org.junit.Test; -import org.mockito.Mockito; - -/** - * Test the Auth parser. - */ -public class TestAWSSignatureProcessor { - - @Test - public void testV4Initialization() throws Exception { - - MultivaluedMap headers = new MultivaluedHashMap<>(); - headers.putSingle("Content-Length", "123"); - headers.putSingle("Host", "0.0.0.0:9878"); - headers.putSingle("X-AMZ-Content-Sha256", "Content-SHA"); - headers.putSingle("X-AMZ-Date", "123"); - headers.putSingle("Content-Type", "ozone/mpu"); - headers.putSingle(HeaderPreprocessor.ORIGINAL_CONTENT_TYPE, "streaming"); - - String authHeader = - "AWS4-HMAC-SHA256 Credential=AKIAJWFJK62WUTKNFJJA/20181009/us-east-1" - + "/s3/aws4_request, " - + "SignedHeaders=host;x-amz-content-sha256;x-amz-date;" - + "content-type, " - + "Signature" - + - "=db81b057718d7c1b3b8dffa29933099551c51d787b3b13b9e0f9ebed45982bf2"; - headers.putSingle("Authorization", - authHeader); - - AuthorizationHeaderV4 parserAuthHeader = - new AuthorizationHeaderV4(authHeader) { - @Override - public void validateDateRange() throws OS3Exception { - } - }; - - MultivaluedMap queryParameters = new MultivaluedHashMap<>(); - - UriInfo uriInfo = Mockito.mock(UriInfo.class); - Mockito.when(uriInfo.getQueryParameters()).thenReturn(queryParameters); - Mockito.when(uriInfo.getRequestUri()) - .thenReturn(new URI("http://localhost/buckets")); - - ContainerRequestContext mock = Mockito.mock(ContainerRequestContext.class); - Mockito.when(mock.getHeaders()).thenReturn(headers); - Mockito.when(mock.getMethod()).thenReturn("GET"); - Mockito.when(mock.getUriInfo()).thenReturn(uriInfo); - - AWSSignatureProcessor parser = new AWSSignatureProcessor() { - @Override - void validateSignedHeader(String header, String headerValue) - throws OS3Exception { - super.validateSignedHeader(header, headerValue); - } - }; - parser.setV4Header(parserAuthHeader); - parser.setContext(mock); - parser.init(); - - Assert.assertTrue( - "the ozone/mpu header is not changed back before signature processing", - parser.buildCanonicalRequest().contains("content-type:streaming")); - - Assert.assertEquals( - "String to sign is invalid", - "AWS4-HMAC-SHA256\n" - + "123\n" - + "20181009/us-east-1/s3/aws4_request\n" - + - "f20d4de80af2271545385e8d4c7df608cae70a791c69b97aab1527ed93a0d665", - parser.getStringToSign()); - } - - @Test - public void testV2Initialization() throws Exception { - - MultivaluedMap headers = new MultivaluedHashMap<>(); - String authHeader = "AWS root:ixWQAgWvJDuqLUqgDG9o4b2HF7c="; - headers.putSingle("Authorization", authHeader); - - AuthorizationHeaderV2 parserAuthHeader = - new AuthorizationHeaderV2(authHeader); - - MultivaluedMap queryParameters = new MultivaluedHashMap<>(); - - UriInfo uriInfo = Mockito.mock(UriInfo.class); - Mockito.when(uriInfo.getQueryParameters()).thenReturn(queryParameters); - Mockito.when(uriInfo.getRequestUri()) - .thenReturn(new URI("http://localhost/buckets")); - - ContainerRequestContext mock = Mockito.mock(ContainerRequestContext.class); - Mockito.when(mock.getHeaders()).thenReturn(headers); - Mockito.when(mock.getMethod()).thenReturn("GET"); - Mockito.when(mock.getUriInfo()).thenReturn(uriInfo); - - AWSSignatureProcessor parser = new AWSSignatureProcessor() { - @Override - void validateSignedHeader(String header, String headerValue) - throws OS3Exception { - super.validateSignedHeader(header, headerValue); - } - }; - parser.setV2Header(parserAuthHeader); - parser.setContext(mock); - parser.init(); - - Assert.assertEquals("root", parser.getAwsAccessId()); - Assert.assertEquals("ixWQAgWvJDuqLUqgDG9o4b2HF7c=", parser.getSignature()); - } -} \ No newline at end of file diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/TestOzoneClientProducer.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/TestOzoneClientProducer.java index d1b5e0875b42..f0f0581e2957 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/TestOzoneClientProducer.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/TestOzoneClientProducer.java @@ -29,13 +29,15 @@ import org.apache.hadoop.ozone.OzoneConfigKeys; import org.apache.hadoop.ozone.om.OMConfigKeys; import org.apache.hadoop.ozone.s3.exception.OS3Exception; - -import static org.apache.hadoop.ozone.s3.SignatureProcessor.AUTHORIZATION_HEADER; -import static org.apache.hadoop.ozone.s3.SignatureProcessor.CONTENT_MD5; -import static org.apache.hadoop.ozone.s3.SignatureProcessor.CONTENT_TYPE; -import static org.apache.hadoop.ozone.s3.SignatureProcessor.HOST_HEADER; -import static org.apache.hadoop.ozone.s3.SignatureProcessor.X_AMAZ_DATE; -import static org.apache.hadoop.ozone.s3.SignatureProcessor.X_AMZ_CONTENT_SHA256; +import org.apache.hadoop.ozone.s3.signature.AWSSignatureProcessor; + +//import static org.apache.hadoop.ozone.s3.signature.SignatureProcessor.AUTHORIZATION_HEADER; +import static org.apache.hadoop.ozone.s3.signature.SignatureParser.AUTHORIZATION_HEADER; +import static org.apache.hadoop.ozone.s3.signature.SignatureProcessor.CONTENT_MD5; +import static org.apache.hadoop.ozone.s3.signature.SignatureProcessor.CONTENT_TYPE; +import static org.apache.hadoop.ozone.s3.signature.SignatureProcessor.HOST_HEADER; +import static org.apache.hadoop.ozone.s3.signature.StringToSignProducer.X_AMAZ_DATE; +import static org.apache.hadoop.ozone.s3.signature.StringToSignProducer.X_AMZ_CONTENT_SHA256; import static org.junit.Assert.fail; import org.junit.Assert; @@ -115,7 +117,6 @@ private void setupContext() throws Exception { AWSSignatureProcessor awsSignatureProcessor = new AWSSignatureProcessor(); awsSignatureProcessor.setContext(context); - awsSignatureProcessor.init(); producer.setSignatureParser(awsSignatureProcessor); } diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestBucketPut.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestBucketPut.java index 6c41509aa201..42affd182cd4 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestBucketPut.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestBucketPut.java @@ -21,26 +21,22 @@ package org.apache.hadoop.ozone.s3.endpoint; import javax.ws.rs.core.Response; +import java.time.LocalDate; import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.client.OzoneClient; import org.apache.hadoop.ozone.client.OzoneClientStub; - -import org.apache.hadoop.ozone.s3.SignatureProcessor; import org.apache.hadoop.ozone.s3.exception.OS3Exception; import static java.net.HttpURLConnection.HTTP_NOT_FOUND; -import static org.apache.hadoop.ozone.s3.AWSSignatureProcessor.DATE_FORMATTER; import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.MALFORMED_HEADER; +import static org.apache.hadoop.ozone.s3.signature.SignatureProcessor.DATE_FORMATTER; +import org.junit.Assert; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; - -import org.junit.Assert; import org.junit.Before; import org.junit.Test; -import java.time.LocalDate; - /** * This class test Create Bucket functionality. */ @@ -59,27 +55,6 @@ public void setup() throws Exception { // Create HeadBucket and setClient to OzoneClientStub bucketEndpoint = new BucketEndpoint(); bucketEndpoint.setClient(clientStub); - bucketEndpoint.setSignatureProcessor(new SignatureProcessor() { - @Override - public String getStringToSign() throws Exception { - return null; - } - - @Override - public String getSignature() { - return null; - } - - @Override - public String getAwsAccessId() { - return OzoneConsts.OZONE; - } - - @Override - public Exception getException() { - return null; - } - }); } @Test diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestRootList.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestRootList.java index 7c5d2a59600f..19ab3bfb7e87 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestRootList.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestRootList.java @@ -20,13 +20,10 @@ package org.apache.hadoop.ozone.s3.endpoint; -import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.client.OzoneClient; import org.apache.hadoop.ozone.client.OzoneClientStub; import static org.junit.Assert.assertEquals; - -import org.apache.hadoop.ozone.s3.SignatureProcessor; import org.junit.Before; import org.junit.Test; @@ -54,27 +51,6 @@ public void setup() throws Exception { @Test public void testListBucket() throws Exception { - rootEndpoint.setSignatureProcessor(new SignatureProcessor() { - @Override - public String getStringToSign() { - return null; - } - - @Override - public String getSignature() { - return null; - } - - @Override - public String getAwsAccessId() { - return OzoneConsts.OZONE; - } - - @Override - public Exception getException() { - return null; - } - }); // List operation should succeed even there is no bucket. ListBucketResponse response = (ListBucketResponse) rootEndpoint.get().getEntity(); diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/header/TestAuthorizationHeaderV4.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/header/TestAuthorizationHeaderV4.java deleted file mode 100644 index 5ca1c4522414..000000000000 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/header/TestAuthorizationHeaderV4.java +++ /dev/null @@ -1,354 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.ozone.s3.header; - -import org.apache.hadoop.ozone.s3.exception.OS3Exception; -import org.apache.hadoop.test.LambdaTestUtils; -import org.junit.Before; -import org.junit.Test; - -import java.time.LocalDate; - -import static java.time.temporal.ChronoUnit.DAYS; -import static org.apache.hadoop.ozone.s3.AWSSignatureProcessor.DATE_FORMATTER; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.fail; - -/** - * This class tests Authorization header format v2. - */ - -public class TestAuthorizationHeaderV4 { - private String curDate; - - @Before - public void setup() { - LocalDate now = LocalDate.now(); - curDate = DATE_FORMATTER.format(now); - } - - @Test - public void testV4HeaderWellFormed() throws Exception { - String auth = "AWS4-HMAC-SHA256 " + - "Credential=ozone/" + curDate + "/us-east-1/s3/aws4_request, " + - "SignedHeaders=host;range;x-amz-date, " + - "Signature=fe5f80f77d5fa3beca038a248ff027"; - AuthorizationHeaderV4 v4 = new AuthorizationHeaderV4(auth); - assertEquals("AWS4-HMAC-SHA256", v4.getAlgorithm()); - assertEquals("ozone", v4.getAccessKeyID()); - assertEquals(curDate, v4.getDate()); - assertEquals("us-east-1", v4.getAwsRegion()); - assertEquals("aws4_request", v4.getAwsRequest()); - assertEquals("host;range;x-amz-date", v4.getSignedHeaderString()); - assertEquals("fe5f80f77d5fa3beca038a248ff027", v4.getSignature()); - } - - @Test - public void testV4HeaderMissingParts() { - try { - String auth = "AWS4-HMAC-SHA256 " + - "Credential=ozone/" + curDate + "/us-east-1/s3/aws4_request, " + - "SignedHeaders=host;range;x-amz-date,"; - AuthorizationHeaderV4 v4 = new AuthorizationHeaderV4(auth); - fail("Exception is expected in case of malformed header"); - } catch (OS3Exception ex) { - assertEquals("AuthorizationHeaderMalformed", ex.getCode()); - } - } - - @Test - public void testV4HeaderInvalidCredential() { - try { - String auth = "AWS4-HMAC-SHA256 " + - "Credential=" + curDate + "/us-east-1/s3/aws4_request, " + - "SignedHeaders=host;range;x-amz-date, " + - "Signature=fe5f80f77d5fa3beca038a248ff027"; - AuthorizationHeaderV4 v4 = new AuthorizationHeaderV4(auth); - fail("Exception is expected in case of malformed header"); - } catch (OS3Exception ex) { - assertEquals("AuthorizationHeaderMalformed", ex.getCode()); - } - } - - @Test - public void testV4HeaderWithoutSpace() throws OS3Exception { - - String auth = - "AWS4-HMAC-SHA256 Credential=ozone/" + curDate + "/us-east-1/s3" + - "/aws4_request," - + "SignedHeaders=host;x-amz-content-sha256;x-amz-date," - + "Signature" - + "=fe5f80f77d5fa3beca038a248ff027"; - AuthorizationHeaderV4 v4 = new AuthorizationHeaderV4(auth); - - assertEquals("AWS4-HMAC-SHA256", v4.getAlgorithm()); - assertEquals("ozone", v4.getAccessKeyID()); - assertEquals(curDate, v4.getDate()); - assertEquals("us-east-1", v4.getAwsRegion()); - assertEquals("aws4_request", v4.getAwsRequest()); - assertEquals("host;x-amz-content-sha256;x-amz-date", - v4.getSignedHeaderString()); - assertEquals("fe5f80f77d5fa3beca038a248ff027", v4.getSignature()); - - } - - @Test - public void testV4HeaderDateValidationSuccess() throws OS3Exception { - // Case 1: valid date within range. - LocalDate now = LocalDate.now(); - String dateStr = DATE_FORMATTER.format(now); - validateResponse(dateStr); - - // Case 2: Valid date with in range. - dateStr = DATE_FORMATTER.format(now.plus(1, DAYS)); - validateResponse(dateStr); - - // Case 3: Valid date with in range. - dateStr = DATE_FORMATTER.format(now.minus(1, DAYS)); - validateResponse(dateStr); - } - - @Test - public void testV4HeaderDateValidationFailure() throws Exception { - // Case 1: Empty date. - LocalDate now = LocalDate.now(); - String dateStr = ""; - LambdaTestUtils.intercept(OS3Exception.class, "", - () -> validateResponse(dateStr)); - - // Case 2: Date after yesterday. - String dateStr2 = DATE_FORMATTER.format(now.plus(2, DAYS)); - LambdaTestUtils.intercept(OS3Exception.class, "", - () -> validateResponse(dateStr2)); - - // Case 3: Date before yesterday. - String dateStr3 = DATE_FORMATTER.format(now.minus(2, DAYS)); - LambdaTestUtils.intercept(OS3Exception.class, "", - () -> validateResponse(dateStr3)); - } - - private void validateResponse(String dateStr) throws OS3Exception { - String auth = - "AWS4-HMAC-SHA256 Credential=ozone/" + dateStr + "/us-east-1/s3" + - "/aws4_request," - + "SignedHeaders=host;x-amz-content-sha256;x-amz-date," - + "Signature" - + "=fe5f80f77d5fa3beca038a248ff027"; - AuthorizationHeaderV4 v4 = new AuthorizationHeaderV4(auth); - - assertEquals("AWS4-HMAC-SHA256", v4.getAlgorithm()); - assertEquals("ozone", v4.getAccessKeyID()); - assertEquals(dateStr, v4.getDate()); - assertEquals("us-east-1", v4.getAwsRegion()); - assertEquals("aws4_request", v4.getAwsRequest()); - assertEquals("host;x-amz-content-sha256;x-amz-date", - v4.getSignedHeaderString()); - assertEquals("fe5f80f77d5fa3beca038a248ff027", v4.getSignature()); - } - - @Test - public void testV4HeaderRegionValidationFailure() throws Exception { - String auth = - "AWS4-HMAC-SHA256 Credential=ozone/" + curDate + "//s3/aws4_request," - + "SignedHeaders=host;x-amz-content-sha256;x-amz-date," - + "Signature" - + "=fe5f80f77d5fa3beca038a248ff027%"; - LambdaTestUtils.intercept(OS3Exception.class, "", - () -> new AuthorizationHeaderV4(auth)); - String auth2 = - "AWS4-HMAC-SHA256 Credential=ozone/" + curDate + "s3/aws4_request," - + "SignedHeaders=host;x-amz-content-sha256;x-amz-date," - + "Signature" - + "=fe5f80f77d5fa3beca038a248ff027%"; - LambdaTestUtils.intercept(OS3Exception.class, "", - () -> new AuthorizationHeaderV4(auth2)); - } - - @Test - public void testV4HeaderServiceValidationFailure() throws Exception { - String auth = - "AWS4-HMAC-SHA256 Credential=ozone/" + curDate + "/us-east-1" + - "//aws4_request," - + "SignedHeaders=host;x-amz-content-sha256;x-amz-date," - + "Signature" - + "=fe5f80f77d5fa3beca038a248ff027"; - LambdaTestUtils.intercept(OS3Exception.class, "", - () -> new AuthorizationHeaderV4(auth)); - - String auth2 = - "AWS4-HMAC-SHA256 Credential=ozone/" + curDate + "/us-east-1" + - "/aws4_request," - + "SignedHeaders=host;x-amz-content-sha256;x-amz-date," - + "Signature" - + "=fe5f80f77d5fa3beca038a248ff027"; - LambdaTestUtils.intercept(OS3Exception.class, "", - () -> new AuthorizationHeaderV4(auth2)); - } - - @Test - public void testV4HeaderRequestValidationFailure() throws Exception { - String auth = - "AWS4-HMAC-SHA256 Credential=ozone/" + curDate + "/us-east-1/s3" + - "/ ," - + "SignedHeaders=host;x-amz-content-sha256;x-amz-date," - + "Signature" - + "=fe5f80f77d5fa3beca038a248ff027"; - LambdaTestUtils.intercept(OS3Exception.class, "", - () -> new AuthorizationHeaderV4(auth)); - - String auth2 = - "AWS4-HMAC-SHA256 Credential=ozone/" + curDate + "/us-east-1/s3" + - "/," - + "SignedHeaders=host;x-amz-content-sha256;x-amz-date," - + "Signature" - + "=fe5f80f77d5fa3beca038a248ff027"; - LambdaTestUtils.intercept(OS3Exception.class, "", - () -> new AuthorizationHeaderV4(auth2)); - - String auth3 = - "AWS4-HMAC-SHA256 Credential=ozone/" + curDate + "/us-east-1/s3" + - "," - + "SignedHeaders=host;x-amz-content-sha256;x-amz-date," - + "Signature" - + "=fe5f80f77d5fa3beca038a248ff027"; - LambdaTestUtils.intercept(OS3Exception.class, "", - () -> new AuthorizationHeaderV4(auth3)); - } - - @Test - public void testV4HeaderSignedHeaderValidationFailure() throws Exception { - String auth = - "AWS4-HMAC-SHA256 Credential=ozone/" + curDate + "/us-east-1/s3" + - "/aws4_request," - + "SignedHeaders=;;," - + "Signature" - + "=fe5f80f77d5fa3beca038a248ff027"; - LambdaTestUtils.intercept(OS3Exception.class, "", - () -> new AuthorizationHeaderV4(auth)); - - String auth2 = - "AWS4-HMAC-SHA256 Credential=ozone/" + curDate + "/us-east-1/s3" + - "/aws4_request," - + "SignedHeaders=," - + "Signature" - + "=fe5f80f77d5fa3beca038a248ff027"; - LambdaTestUtils.intercept(OS3Exception.class, "", - () -> new AuthorizationHeaderV4(auth2)); - - String auth3 = - "AWS4-HMAC-SHA256 Credential=ozone/" + curDate + "/us-east-1/s3" + - "/aws4_request," - + "=x-amz-content-sha256;x-amz-date," - + "Signature" - + "=fe5f80f77d5fa3beca038a248ff027"; - LambdaTestUtils.intercept(OS3Exception.class, "", - () -> new AuthorizationHeaderV4(auth3)); - - String auth4 = - "AWS4-HMAC-SHA256 Credential=ozone/" + curDate + "/us-east-1/s3" + - "/aws4_request," - + "=," - + "Signature" - + "=fe5f80f77d5fa3beca038a248ff027"; - LambdaTestUtils.intercept(OS3Exception.class, "", - () -> new AuthorizationHeaderV4(auth4)); - } - - @Test - public void testV4HeaderSignatureValidationFailure() throws Exception { - String auth = - "AWS4-HMAC-SHA256 Credential=ozone/" + curDate + "/us-east-1/s3" + - "/aws4_request," - + "SignedHeaders=host;x-amz-content-sha256;x-amz-date," - + "Signature" - + "=fe5f80f77d5fa3beca038a248ff027%"; - LambdaTestUtils.intercept(OS3Exception.class, "", - () -> new AuthorizationHeaderV4(auth)); - - String auth2 = - "AWS4-HMAC-SHA256 Credential=ozone/" + curDate + "/us-east-1/s3" + - "/aws4_request," - + "SignedHeaders=host;x-amz-content-sha256;x-amz-date," - + "Signature" - + "="; - LambdaTestUtils.intercept(OS3Exception.class, "", - () -> new AuthorizationHeaderV4(auth2)); - - String auth3 = - "AWS4-HMAC-SHA256 Credential=ozone/" + curDate + "/us-east-1/s3" + - "/aws4_request," - + "SignedHeaders=host;x-amz-content-sha256;x-amz-date," - + "" - + "="; - LambdaTestUtils.intercept(OS3Exception.class, "", - () -> new AuthorizationHeaderV4(auth3)); - } - - @Test - public void testV4HeaderHashAlgoValidationFailure() throws Exception { - String auth = - "AWS4-HMAC-SHA Credential=ozone/" + curDate + "/us-east-1/s3" + - "/aws4_request," - + "SignedHeaders=host;x-amz-content-sha256;x-amz-date," - + "Signature" - + "=fe5f80f77d5fa3beca038a248ff027"; - LambdaTestUtils.intercept(OS3Exception.class, "", - () -> new AuthorizationHeaderV4(auth)); - - String auth2 = - "SHA-256 Credential=ozone/" + curDate + "/us-east-1/s3" + - "/aws4_request," - + "SignedHeaders=host;x-amz-content-sha256;x-amz-date," - + "Signature" - + "=fe5f80f77d5fa3beca038a248ff027"; - LambdaTestUtils.intercept(OS3Exception.class, "", - () -> new AuthorizationHeaderV4(auth2)); - - String auth3 = - " Credential=ozone/" + curDate + "/us-east-1/s3" + - "/aws4_request," - + "SignedHeaders=host;x-amz-content-sha256;x-amz-date," - + "Signature" - + "=fe5f80f77d5fa3beca038a248ff027"; - LambdaTestUtils.intercept(OS3Exception.class, "", - () -> new AuthorizationHeaderV4(auth3)); - } - - @Test - public void testV4HeaderCredentialValidationFailure() throws Exception { - String auth = - "AWS4-HMAC-SHA Credential=/" + curDate + "//" + - "/," - + "SignedHeaders=host;x-amz-content-sha256;x-amz-date," - + "Signature" - + "=fe5f80f77d5fa3beca038a248ff027"; - LambdaTestUtils.intercept(OS3Exception.class, "", - () -> new AuthorizationHeaderV4(auth)); - - String auth2 = - "AWS4-HMAC-SHA =/" + curDate + "//" + - "/," - + "SignedHeaders=host;x-amz-content-sha256;x-amz-date," - + "Signature" - + "=fe5f80f77d5fa3beca038a248ff027"; - LambdaTestUtils.intercept(OS3Exception.class, "", - () -> new AuthorizationHeaderV4(auth2)); - } - -} diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/signature/TestAWSSignatureProcessor.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/signature/TestAWSSignatureProcessor.java new file mode 100644 index 000000000000..f65c4b8ca20e --- /dev/null +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/signature/TestAWSSignatureProcessor.java @@ -0,0 +1,127 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.ozone.s3.signature; + +/** + * Test the Auth parser. + */ +public class TestAWSSignatureProcessor { +// +// @Test +// public void testV4Initialization() throws Exception { +// +// MultivaluedMap headers = new MultivaluedHashMap<>(); +// headers.putSingle("Content-Length", "123"); +// headers.putSingle("Host", "0.0.0.0:9878"); +// headers.putSingle("X-AMZ-Content-Sha256", "Content-SHA"); +// headers.putSingle("X-AMZ-Date", "123"); +// headers.putSingle("Content-Type", "ozone/mpu"); +// headers.putSingle(HeaderPreprocessor.ORIGINAL_CONTENT_TYPE, "streaming"); +// +// String authHeader = +// "AWS4-HMAC-SHA256 Credential=AKIAJWFJK62WUTKNFJJA/20181009/us-east-1" +// + "/s3/aws4_request, " +// + "SignedHeaders=host;x-amz-content-sha256;x-amz-date;" +// + "content-type, " +// + "Signature" +// + +// "=db81b057718d7c1b3b8dffa29933099551c51d787b3b13b9e0f9ebed45982bf2"; +// headers.putSingle("Authorization", +// authHeader); +// +// AuthorizationHeaderV4 parserAuthHeader = +// new AuthorizationHeaderV4(authHeader) { +// @Override +// public void validateDateRange() throws OS3Exception { +// } +// }; +// +// MultivaluedMap queryParameters = new MultivaluedHashMap<>(); +// +// UriInfo uriInfo = Mockito.mock(UriInfo.class); +// Mockito.when(uriInfo.getQueryParameters()).thenReturn(queryParameters); +// Mockito.when(uriInfo.getRequestUri()) +// .thenReturn(new URI("http://localhost/buckets")); +// +// ContainerRequestContext mock = Mockito.mock(ContainerRequestContext.class); +// Mockito.when(mock.getHeaders()).thenReturn(headers); +// Mockito.when(mock.getMethod()).thenReturn("GET"); +// Mockito.when(mock.getUriInfo()).thenReturn(uriInfo); +// +// AWSSignatureProcessor parser = new AWSSignatureProcessor() { +// @Override +// void validateSignedHeader(String header, String headerValue) +// throws OS3Exception { +// super.validateSignedHeader(header, headerValue); +// } +// }; +// parser.setV4Header(parserAuthHeader); +// parser.setContext(mock); +// parser.init(); +// +// Assert.assertTrue( +// "the ozone/mpu header is not changed back before signature processing", +// parser.buildCanonicalRequest().contains("content-type:streaming")); +// +// Assert.assertEquals( +// "String to sign is invalid", +// "AWS4-HMAC-SHA256\n" +// + "123\n" +// + "20181009/us-east-1/s3/aws4_request\n" +// + +// "f20d4de80af2271545385e8d4c7df608cae70a791c69b97aab1527ed93a0d665", +// parser.getStringToSign()); +// } +// +// @Test +// public void testV2Initialization() throws Exception { +// +// MultivaluedMap headers = new MultivaluedHashMap<>(); +// String authHeader = "AWS root:ixWQAgWvJDuqLUqgDG9o4b2HF7c="; +// headers.putSingle("Authorization", authHeader); +// +// AuthorizationHeaderV2 parserAuthHeader = +// new AuthorizationHeaderV2(authHeader); +// +// MultivaluedMap queryParameters = new MultivaluedHashMap<>(); +// +// UriInfo uriInfo = Mockito.mock(UriInfo.class); +// Mockito.when(uriInfo.getQueryParameters()).thenReturn(queryParameters); +// Mockito.when(uriInfo.getRequestUri()) +// .thenReturn(new URI("http://localhost/buckets")); +// +// ContainerRequestContext mock = Mockito.mock(ContainerRequestContext.class); +// Mockito.when(mock.getHeaders()).thenReturn(headers); +// Mockito.when(mock.getMethod()).thenReturn("GET"); +// Mockito.when(mock.getUriInfo()).thenReturn(uriInfo); +// +// AWSSignatureProcessor parser = new AWSSignatureProcessor() { +// @Override +// void validateSignedHeader(String header, String headerValue) +// throws OS3Exception { +// super.validateSignedHeader(header, headerValue); +// } +// }; +// parser.setV2Header(parserAuthHeader); +// parser.setContext(mock); +// parser.init(); +// +// Assert.assertEquals("root", parser.getAwsAccessId()); +// Assert.assertEquals("ixWQAgWvJDuqLUqgDG9o4b2HF7c=", parser.getSignature()); +// } +} \ No newline at end of file diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/header/TestAuthorizationHeaderV2.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/signature/TestAuthorizationHeaderV2.java similarity index 98% rename from hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/header/TestAuthorizationHeaderV2.java rename to hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/signature/TestAuthorizationHeaderV2.java index 97f7fb4cc094..6aa0fc84f0af 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/header/TestAuthorizationHeaderV2.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/signature/TestAuthorizationHeaderV2.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.hadoop.ozone.s3.header; +package org.apache.hadoop.ozone.s3.signature; import org.apache.hadoop.ozone.s3.exception.OS3Exception; import org.junit.Test; diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/signature/TestAuthorizationV4HeaderParser.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/signature/TestAuthorizationV4HeaderParser.java new file mode 100644 index 000000000000..b6010f60b0c7 --- /dev/null +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/signature/TestAuthorizationV4HeaderParser.java @@ -0,0 +1,354 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.s3.signature; + +import java.time.LocalDate; + +import org.apache.hadoop.ozone.s3.exception.OS3Exception; + +import static org.apache.hadoop.ozone.s3.signature.SignatureProcessor.DATE_FORMATTER; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; +import org.junit.Before; +import org.junit.Test; + +/** + * This class tests Authorization header format v2. + */ + +public class TestAuthorizationV4HeaderParser { + private String curDate; + + @Before + public void setup() { + LocalDate now = LocalDate.now(); + curDate = DATE_FORMATTER.format(now); + } + + @Test + public void testV4HeaderWellFormed() throws Exception { + String auth = "AWS4-HMAC-SHA256 " + + "Credential=ozone/" + curDate + "/us-east-1/s3/aws4_request, " + + "SignedHeaders=host;range;x-amz-date, " + + "Signature=fe5f80f77d5fa3beca038a248ff027"; + AuthorizationV4HeaderParser v4 = new AuthorizationV4HeaderParser(auth); + final SignatureInfo signatureInfo = v4.parseSignature(); + assertEquals("ozone", signatureInfo.getAwsAccessId()); + assertEquals(curDate, signatureInfo.getDate()); + assertEquals("host;range;x-amz-date", signatureInfo.getSignedHeaders()); + assertEquals("fe5f80f77d5fa3beca038a248ff027", + signatureInfo.getSignature()); + } + + @Test + public void testV4HeaderMissingParts() { + try { + String auth = "AWS4-HMAC-SHA256 " + + "Credential=ozone/" + curDate + "/us-east-1/s3/aws4_request, " + + "SignedHeaders=host;range;x-amz-date,"; + AuthorizationV4HeaderParser v4 = new AuthorizationV4HeaderParser(auth); + v4.parseSignature(); + fail("Exception is expected in case of malformed header"); + } catch (OS3Exception ex) { + assertEquals("AuthorizationHeaderMalformed", ex.getCode()); + } + } + + @Test + public void testV4HeaderInvalidCredential() { + try { + String auth = "AWS4-HMAC-SHA256 " + + "Credential=" + curDate + "/us-east-1/s3/aws4_request, " + + "SignedHeaders=host;range;x-amz-date, " + + "Signature=fe5f80f77d5fa3beca038a248ff027"; + AuthorizationV4HeaderParser v4 = new AuthorizationV4HeaderParser(auth); + v4.parseSignature(); + fail("Exception is expected in case of malformed header"); + } catch (OS3Exception ex) { + assertEquals("AuthorizationHeaderMalformed", ex.getCode()); + } + } + + // @Test + // public void testV4HeaderWithoutSpace() throws OS3Exception { + // + // String auth = + // "AWS4-HMAC-SHA256 Credential=ozone/" + curDate + "/us-east-1/s3" + + // "/aws4_request," + // + "SignedHeaders=host;x-amz-content-sha256;x-amz-date," + // + "Signature" + // + "=fe5f80f77d5fa3beca038a248ff027"; + // AuthorizationHeaderV4 v4 = new AuthorizationHeaderV4(auth); + // + // assertEquals("AWS4-HMAC-SHA256", v4.getAlgorithm()); + // assertEquals("ozone", v4.getAccessKeyID()); + // assertEquals(curDate, v4.getDate()); + // assertEquals("us-east-1", v4.getAwsRegion()); + // assertEquals("aws4_request", v4.getAwsRequest()); + // assertEquals("host;x-amz-content-sha256;x-amz-date", + // v4.getSignedHeaderString()); + // assertEquals("fe5f80f77d5fa3beca038a248ff027", v4.getSignature()); + // + // } + // + // @Test + // public void testV4HeaderDateValidationSuccess() throws OS3Exception { + // // Case 1: valid date within range. + // LocalDate now = LocalDate.now(); + // String dateStr = DATE_FORMATTER.format(now); + // validateResponse(dateStr); + // + // // Case 2: Valid date with in range. + // dateStr = DATE_FORMATTER.format(now.plus(1, DAYS)); + // validateResponse(dateStr); + // + // // Case 3: Valid date with in range. + // dateStr = DATE_FORMATTER.format(now.minus(1, DAYS)); + // validateResponse(dateStr); + // } + // + // @Test + // public void testV4HeaderDateValidationFailure() throws Exception { + // // Case 1: Empty date. + // LocalDate now = LocalDate.now(); + // String dateStr = ""; + // LambdaTestUtils.intercept(OS3Exception.class, "", + // () -> validateResponse(dateStr)); + // + // // Case 2: Date after yesterday. + // String dateStr2 = DATE_FORMATTER.format(now.plus(2, DAYS)); + // LambdaTestUtils.intercept(OS3Exception.class, "", + // () -> validateResponse(dateStr2)); + // + // // Case 3: Date before yesterday. + // String dateStr3 = DATE_FORMATTER.format(now.minus(2, DAYS)); + // LambdaTestUtils.intercept(OS3Exception.class, "", + // () -> validateResponse(dateStr3)); + // } + // + // private void validateResponse(String dateStr) throws OS3Exception { + // String auth = + // "AWS4-HMAC-SHA256 Credential=ozone/" + dateStr + "/us-east-1/s3" + + // "/aws4_request," + // + "SignedHeaders=host;x-amz-content-sha256;x-amz-date," + // + "Signature" + // + "=fe5f80f77d5fa3beca038a248ff027"; + // AuthorizationHeaderV4 v4 = new AuthorizationHeaderV4(auth); + // + // assertEquals("AWS4-HMAC-SHA256", v4.getAlgorithm()); + // assertEquals("ozone", v4.getAccessKeyID()); + // assertEquals(dateStr, v4.getDate()); + // assertEquals("us-east-1", v4.getAwsRegion()); + // assertEquals("aws4_request", v4.getAwsRequest()); + // assertEquals("host;x-amz-content-sha256;x-amz-date", + // v4.getSignedHeaderString()); + // assertEquals("fe5f80f77d5fa3beca038a248ff027", v4.getSignature()); + // } + // + // @Test + // public void testV4HeaderRegionValidationFailure() throws Exception { + // String auth = + // "AWS4-HMAC-SHA256 Credential=ozone/" + curDate + + // "//s3/aws4_request," + // + "SignedHeaders=host;x-amz-content-sha256;x-amz-date," + // + "Signature" + // + "=fe5f80f77d5fa3beca038a248ff027%"; + // LambdaTestUtils.intercept(OS3Exception.class, "", + // () -> new AuthorizationHeaderV4(auth)); + // String auth2 = + // "AWS4-HMAC-SHA256 Credential=ozone/" + curDate + "s3/aws4_request," + // + "SignedHeaders=host;x-amz-content-sha256;x-amz-date," + // + "Signature" + // + "=fe5f80f77d5fa3beca038a248ff027%"; + // LambdaTestUtils.intercept(OS3Exception.class, "", + // () -> new AuthorizationHeaderV4(auth2)); + // } + // + // @Test + // public void testV4HeaderServiceValidationFailure() throws Exception { + // String auth = + // "AWS4-HMAC-SHA256 Credential=ozone/" + curDate + "/us-east-1" + + // "//aws4_request," + // + "SignedHeaders=host;x-amz-content-sha256;x-amz-date," + // + "Signature" + // + "=fe5f80f77d5fa3beca038a248ff027"; + // LambdaTestUtils.intercept(OS3Exception.class, "", + // () -> new AuthorizationHeaderV4(auth)); + // + // String auth2 = + // "AWS4-HMAC-SHA256 Credential=ozone/" + curDate + "/us-east-1" + + // "/aws4_request," + // + "SignedHeaders=host;x-amz-content-sha256;x-amz-date," + // + "Signature" + // + "=fe5f80f77d5fa3beca038a248ff027"; + // LambdaTestUtils.intercept(OS3Exception.class, "", + // () -> new AuthorizationHeaderV4(auth2)); + // } + // + // @Test + // public void testV4HeaderRequestValidationFailure() throws Exception { + // String auth = + // "AWS4-HMAC-SHA256 Credential=ozone/" + curDate + "/us-east-1/s3" + + // "/ ," + // + "SignedHeaders=host;x-amz-content-sha256;x-amz-date," + // + "Signature" + // + "=fe5f80f77d5fa3beca038a248ff027"; + // LambdaTestUtils.intercept(OS3Exception.class, "", + // () -> new AuthorizationHeaderV4(auth)); + // + // String auth2 = + // "AWS4-HMAC-SHA256 Credential=ozone/" + curDate + "/us-east-1/s3" + + // "/," + // + "SignedHeaders=host;x-amz-content-sha256;x-amz-date," + // + "Signature" + // + "=fe5f80f77d5fa3beca038a248ff027"; + // LambdaTestUtils.intercept(OS3Exception.class, "", + // () -> new AuthorizationHeaderV4(auth2)); + // + // String auth3 = + // "AWS4-HMAC-SHA256 Credential=ozone/" + curDate + "/us-east-1/s3" + + // "," + // + "SignedHeaders=host;x-amz-content-sha256;x-amz-date," + // + "Signature" + // + "=fe5f80f77d5fa3beca038a248ff027"; + // LambdaTestUtils.intercept(OS3Exception.class, "", + // () -> new AuthorizationHeaderV4(auth3)); + // } + // + // @Test + // public void testV4HeaderSignedHeaderValidationFailure() throws Exception { + // String auth = + // "AWS4-HMAC-SHA256 Credential=ozone/" + curDate + "/us-east-1/s3" + + // "/aws4_request," + // + "SignedHeaders=;;," + // + "Signature" + // + "=fe5f80f77d5fa3beca038a248ff027"; + // LambdaTestUtils.intercept(OS3Exception.class, "", + // () -> new AuthorizationHeaderV4(auth)); + // + // String auth2 = + // "AWS4-HMAC-SHA256 Credential=ozone/" + curDate + "/us-east-1/s3" + + // "/aws4_request," + // + "SignedHeaders=," + // + "Signature" + // + "=fe5f80f77d5fa3beca038a248ff027"; + // LambdaTestUtils.intercept(OS3Exception.class, "", + // () -> new AuthorizationHeaderV4(auth2)); + // + // String auth3 = + // "AWS4-HMAC-SHA256 Credential=ozone/" + curDate + "/us-east-1/s3" + + // "/aws4_request," + // + "=x-amz-content-sha256;x-amz-date," + // + "Signature" + // + "=fe5f80f77d5fa3beca038a248ff027"; + // LambdaTestUtils.intercept(OS3Exception.class, "", + // () -> new AuthorizationHeaderV4(auth3)); + // + // String auth4 = + // "AWS4-HMAC-SHA256 Credential=ozone/" + curDate + "/us-east-1/s3" + + // "/aws4_request," + // + "=," + // + "Signature" + // + "=fe5f80f77d5fa3beca038a248ff027"; + // LambdaTestUtils.intercept(OS3Exception.class, "", + // () -> new AuthorizationHeaderV4(auth4)); + // } + // + // @Test + // public void testV4HeaderSignatureValidationFailure() throws Exception { + // String auth = + // "AWS4-HMAC-SHA256 Credential=ozone/" + curDate + "/us-east-1/s3" + + // "/aws4_request," + // + "SignedHeaders=host;x-amz-content-sha256;x-amz-date," + // + "Signature" + // + "=fe5f80f77d5fa3beca038a248ff027%"; + // LambdaTestUtils.intercept(OS3Exception.class, "", + // () -> new AuthorizationHeaderV4(auth)); + // + // String auth2 = + // "AWS4-HMAC-SHA256 Credential=ozone/" + curDate + "/us-east-1/s3" + + // "/aws4_request," + // + "SignedHeaders=host;x-amz-content-sha256;x-amz-date," + // + "Signature" + // + "="; + // LambdaTestUtils.intercept(OS3Exception.class, "", + // () -> new AuthorizationHeaderV4(auth2)); + // + // String auth3 = + // "AWS4-HMAC-SHA256 Credential=ozone/" + curDate + "/us-east-1/s3" + + // "/aws4_request," + // + "SignedHeaders=host;x-amz-content-sha256;x-amz-date," + // + "" + // + "="; + // LambdaTestUtils.intercept(OS3Exception.class, "", + // () -> new AuthorizationHeaderV4(auth3)); + // } + // + // @Test + // public void testV4HeaderHashAlgoValidationFailure() throws Exception { + // String auth = + // "AWS4-HMAC-SHA Credential=ozone/" + curDate + "/us-east-1/s3" + + // "/aws4_request," + // + "SignedHeaders=host;x-amz-content-sha256;x-amz-date," + // + "Signature" + // + "=fe5f80f77d5fa3beca038a248ff027"; + // LambdaTestUtils.intercept(OS3Exception.class, "", + // () -> new AuthorizationHeaderV4(auth)); + // + // String auth2 = + // "SHA-256 Credential=ozone/" + curDate + "/us-east-1/s3" + + // "/aws4_request," + // + "SignedHeaders=host;x-amz-content-sha256;x-amz-date," + // + "Signature" + // + "=fe5f80f77d5fa3beca038a248ff027"; + // LambdaTestUtils.intercept(OS3Exception.class, "", + // () -> new AuthorizationHeaderV4(auth2)); + // + // String auth3 = + // " Credential=ozone/" + curDate + "/us-east-1/s3" + + // "/aws4_request," + // + "SignedHeaders=host;x-amz-content-sha256;x-amz-date," + // + "Signature" + // + "=fe5f80f77d5fa3beca038a248ff027"; + // LambdaTestUtils.intercept(OS3Exception.class, "", + // () -> new AuthorizationHeaderV4(auth3)); + // } + // + // @Test + // public void testV4HeaderCredentialValidationFailure() throws Exception { + // String auth = + // "AWS4-HMAC-SHA Credential=/" + curDate + "//" + + // "/," + // + "SignedHeaders=host;x-amz-content-sha256;x-amz-date," + // + "Signature" + // + "=fe5f80f77d5fa3beca038a248ff027"; + // LambdaTestUtils.intercept(OS3Exception.class, "", + // () -> new AuthorizationHeaderV4(auth)); + // + // String auth2 = + // "AWS4-HMAC-SHA =/" + curDate + "//" + + // "/," + // + "SignedHeaders=host;x-amz-content-sha256;x-amz-date," + // + "Signature" + // + "=fe5f80f77d5fa3beca038a248ff027"; + // LambdaTestUtils.intercept(OS3Exception.class, "", + // () -> new AuthorizationHeaderV4(auth2)); + // } + +} diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/signature/TestStringToSignProducer.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/signature/TestStringToSignProducer.java new file mode 100644 index 000000000000..1e0751b71f09 --- /dev/null +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/signature/TestStringToSignProducer.java @@ -0,0 +1,76 @@ +package org.apache.hadoop.ozone.s3.signature; + +import javax.ws.rs.container.ContainerRequestContext; +import javax.ws.rs.core.MultivaluedHashMap; +import javax.ws.rs.core.MultivaluedMap; +import javax.ws.rs.core.UriInfo; +import java.net.URI; + +import org.apache.hadoop.ozone.s3.HeaderPreprocessor; +import org.apache.hadoop.ozone.s3.exception.OS3Exception; + +import org.junit.Assert; +import org.junit.Test; +import org.mockito.Mockito; + +public class TestStringToSignProducer { + + @Test + public void test() throws Exception { + + MultivaluedMap headers = new MultivaluedHashMap<>(); + headers.putSingle("Content-Length", "123"); + headers.putSingle("Host", "0.0.0.0:9878"); + headers.putSingle("X-AMZ-Content-Sha256", "Content-SHA"); + headers.putSingle("X-AMZ-Date", "123"); + headers.putSingle("Content-Type", "ozone/mpu"); + headers.putSingle(HeaderPreprocessor.ORIGINAL_CONTENT_TYPE, "streaming"); + + String authHeader = + "AWS4-HMAC-SHA256 Credential=AKIAJWFJK62WUTKNFJJA/20181009/us-east-1" + + "/s3/aws4_request, " + + "SignedHeaders=host;x-amz-content-sha256;x-amz-date;" + + "content-type, " + + "Signature" + + + "=db81b057718d7c1b3b8dffa29933099551c51d787b3b13b9e0f9ebed45982bf2"; + + headers.putSingle("Authorization", + authHeader); + + MultivaluedMap queryParameters = new MultivaluedHashMap<>(); + + UriInfo uriInfo = Mockito.mock(UriInfo.class); + Mockito.when(uriInfo.getQueryParameters()).thenReturn(queryParameters); + Mockito.when(uriInfo.getRequestUri()) + .thenReturn(new URI("http://localhost/buckets")); + + ContainerRequestContext context = + Mockito.mock(ContainerRequestContext.class); + Mockito.when(context.getHeaders()).thenReturn(headers); + Mockito.when(context.getMethod()).thenReturn("GET"); + Mockito.when(context.getUriInfo()).thenReturn(uriInfo); + + final SignatureInfo signatureInfo = + new AuthorizationV4HeaderParser(authHeader) { + @Override + public void validateDateRange(Credential credentialObj) + throws OS3Exception { + //NOOP + } + }.parseSignature(); + + final String signatureBase = + StringToSignProducer.createSignatureBase(signatureInfo, context); + + Assert.assertEquals( + "String to sign is invalid", + "AWS4-HMAC-SHA256\n" + + "123\n" + + "20181009/us-east-1/s3/aws4_request\n" + + + "f20d4de80af2271545385e8d4c7df608cae70a791c69b97aab1527ed93a0d665", + signatureBase); + } + +} \ No newline at end of file From 675b49abf493ab6ee43a707de0f956469bc469c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Elek=20M=C3=A1rton?= Date: Wed, 25 Nov 2020 11:32:08 +0100 Subject: [PATCH 02/11] fix unit tests and v2 parser --- .../hadoop/ozone/s3/OzoneClientProducer.java | 29 +- .../s3/signature/AWSSignatureProcessor.java | 13 +- ....java => AuthorizationV2HeaderParser.java} | 68 +-- .../AuthorizationV4HeaderParser.java | 38 +- .../signature/AuthorizationV4QueryParser.java | 27 +- .../hadoop/ozone/s3/signature/Credential.java | 6 + .../ozone/s3/signature/SignatureInfo.java | 34 ++ .../ozone/s3/signature/SignatureParser.java | 20 + .../s3/signature/StringToSignProducer.java | 28 +- .../ozone/s3/TestOzoneClientProducer.java | 85 ++- .../signature/TestAWSSignatureProcessor.java | 127 ---- ...a => TestAuthorizationV2HeaderParser.java} | 33 +- .../TestAuthorizationV4HeaderParser.java | 559 +++++++++--------- 13 files changed, 526 insertions(+), 541 deletions(-) rename hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/{AuthorizationHeaderV2.java => AuthorizationV2HeaderParser.java} (67%) delete mode 100644 hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/signature/TestAWSSignatureProcessor.java rename hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/signature/{TestAuthorizationHeaderV2.java => TestAuthorizationV2HeaderParser.java} (70%) diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/OzoneClientProducer.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/OzoneClientProducer.java index 1fb396a79c29..dc9c3e43385e 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/OzoneClientProducer.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/OzoneClientProducer.java @@ -34,6 +34,7 @@ import org.apache.hadoop.ozone.client.OzoneClientFactory; import org.apache.hadoop.ozone.s3.exception.OS3Exception; import org.apache.hadoop.ozone.s3.signature.SignatureInfo; +import org.apache.hadoop.ozone.s3.signature.SignatureInfo.Version; import org.apache.hadoop.ozone.s3.signature.SignatureProcessor; import org.apache.hadoop.ozone.s3.signature.StringToSignProducer; import org.apache.hadoop.ozone.security.OzoneTokenIdentifier; @@ -44,6 +45,7 @@ import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMTokenProto.Type.S3AUTHINFO; import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.INTERNAL_ERROR; import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.MALFORMED_HEADER; +import org.jetbrains.annotations.NotNull; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -92,8 +94,7 @@ private OzoneClient getClient(OzoneConfiguration config) SignatureInfo signatureInfo = signatureProcessor.parseSignature(); String stringToSign = ""; - if (signatureInfo.getAwsAccessId() != null - && signatureInfo.getAwsAccessId().length() > 0) { + if (signatureInfo.getVersion() == Version.V4) { stringToSign = StringToSignProducer.createSignatureBase(signatureInfo, context); } @@ -106,6 +107,10 @@ private OzoneClient getClient(OzoneConfiguration config) if (OzoneSecurityUtil.isSecurityEnabled(config)) { LOG.debug("Creating s3 auth info for client."); + if (signatureInfo.getVersion() == Version.NONE) { + throw MALFORMED_HEADER; + } + OzoneTokenIdentifier identifier = new OzoneTokenIdentifier(); identifier.setTokenType(S3AUTHINFO); identifier.setStrToSign(stringToSign); @@ -124,13 +129,7 @@ private OzoneClient getClient(OzoneConfiguration config) } ozoneClient = remoteUser.doAs((PrivilegedExceptionAction)() -> { - if (omServiceID == null) { - return OzoneClientFactory.getRpcClient(ozoneConfiguration); - } else { - // As in HA case, we need to pass om service ID. - return OzoneClientFactory.getRpcClient(omServiceID, - ozoneConfiguration); - } + return createOzoneClient(); }); } catch (OS3Exception ex) { if (LOG.isDebugEnabled()) { @@ -148,6 +147,18 @@ private OzoneClient getClient(OzoneConfiguration config) return ozoneClient; } + @NotNull + @VisibleForTesting + OzoneClient createOzoneClient() throws IOException { + if (omServiceID == null) { + return OzoneClientFactory.getRpcClient(ozoneConfiguration); + } else { + // As in HA case, we need to pass om service ID. + return OzoneClientFactory.getRpcClient(omServiceID, + ozoneConfiguration); + } + } + private void getSignatureInfo() { } diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/AWSSignatureProcessor.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/AWSSignatureProcessor.java index 8bb3c28ab2c4..d4b818d0a4dd 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/AWSSignatureProcessor.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/AWSSignatureProcessor.java @@ -30,6 +30,7 @@ import org.apache.hadoop.ozone.s3.HeaderPreprocessor; import org.apache.hadoop.ozone.s3.exception.OS3Exception; +import org.apache.hadoop.ozone.s3.signature.SignatureInfo.Version; import com.google.common.annotations.VisibleForTesting; import org.slf4j.Logger; @@ -45,25 +46,24 @@ public class AWSSignatureProcessor implements SignatureProcessor { private final static Logger LOG = LoggerFactory.getLogger(AWSSignatureProcessor.class); - String AUTHORIZATION_HEADER = "Authorization"; @Context private ContainerRequestContext context; - private SignatureInfo signatureInfo; - public SignatureInfo parseSignature() throws OS3Exception { LowerCaseKeyStringMap headers = LowerCaseKeyStringMap.fromHeaderMap(context.getHeaders()); - String authHeader = headers.get(AUTHORIZATION_HEADER); + String authHeader = headers.get("Authorization"); List signatureParsers = new ArrayList<>(); signatureParsers.add(new AuthorizationV4HeaderParser(authHeader)); - signatureParsers.add(new AuthorizationV4QueryParser(context.getUriInfo().getQueryParameters())); + signatureParsers.add(new AuthorizationV4QueryParser( + context.getUriInfo().getQueryParameters())); + signatureParsers.add(new AuthorizationV2HeaderParser(authHeader)); - signatureInfo = null; + SignatureInfo signatureInfo = null; for (SignatureParser parser : signatureParsers) { signatureInfo = parser.parseSignature(); if (signatureInfo != null) { @@ -72,6 +72,7 @@ public SignatureInfo parseSignature() throws OS3Exception { } if (signatureInfo == null) { signatureInfo = new SignatureInfo( + Version.NONE, "", "", "", "", "", "" ); } diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/AuthorizationHeaderV2.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/AuthorizationV2HeaderParser.java similarity index 67% rename from hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/AuthorizationHeaderV2.java rename to hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/AuthorizationV2HeaderParser.java index 5f7a2e46138d..3b3f7b76c7cc 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/AuthorizationHeaderV2.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/AuthorizationV2HeaderParser.java @@ -15,52 +15,44 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.hadoop.ozone.s3.signature; import org.apache.hadoop.ozone.s3.exception.OS3Exception; import org.apache.hadoop.ozone.s3.exception.S3ErrorTable; +import org.apache.hadoop.ozone.s3.signature.SignatureInfo.Version; -import com.google.common.base.Preconditions; import static org.apache.commons.lang3.StringUtils.isBlank; /** - * Authorization Header v2. + * Class to parse V2 auth information from header. */ -public class AuthorizationHeaderV2 { +public class AuthorizationV2HeaderParser implements SignatureParser { public final static String IDENTIFIER = "AWS"; - private String authHeader; - - private String identifier; - - private String accessKeyID; + private final String authHeader; - private String signature; - - public AuthorizationHeaderV2(String auth) throws OS3Exception { - Preconditions.checkNotNull(auth); - this.authHeader = auth; - parseHeader(); + public AuthorizationV2HeaderParser(String authHeader) { + this.authHeader = authHeader; } /** * This method parses the authorization header. - * + *

* Authorization header sample: * AWS AKIAIOSFODNN7EXAMPLE:frJIUN8DYpKDtOLCwo//yllqDzg= - * - * @throws OS3Exception */ - @SuppressWarnings("StringSplitter") - public void parseHeader() throws OS3Exception { + @Override + public SignatureInfo parseSignature() throws OS3Exception { + if (authHeader == null || !authHeader.startsWith(IDENTIFIER)) { + return null; + } String[] split = authHeader.split(" "); if (split.length != 2) { throw S3ErrorTable.newError(S3ErrorTable.MALFORMED_HEADER, authHeader); } - identifier = split[0]; + String identifier = split[0]; if (!IDENTIFIER.equals(identifier)) { throw S3ErrorTable.newError(S3ErrorTable.MALFORMED_HEADER, authHeader); } @@ -71,31 +63,19 @@ public void parseHeader() throws OS3Exception { throw S3ErrorTable.newError(S3ErrorTable.MALFORMED_HEADER, authHeader); } - accessKeyID = remainingSplit[0]; - signature = remainingSplit[1]; + String accessKeyID = remainingSplit[0]; + String signature = remainingSplit[1]; if (isBlank(accessKeyID) || isBlank(signature)) { throw S3ErrorTable.newError(S3ErrorTable.MALFORMED_HEADER, authHeader); } + return new SignatureInfo( + Version.V2, + "", + accessKeyID, + signature, + "", + "", + "" + ); } - - public String getAuthHeader() { - return authHeader; - } - - public void setAuthHeader(String authHeader) { - this.authHeader = authHeader; - } - - public String getIdentifier() { - return identifier; - } - - public String getAccessKeyID() { - return accessKeyID; - } - - public String getSignature() { - return signature; - } - } diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/AuthorizationV4HeaderParser.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/AuthorizationV4HeaderParser.java index ec10ad6c95c4..d644e0535d5f 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/AuthorizationV4HeaderParser.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/AuthorizationV4HeaderParser.java @@ -1,3 +1,20 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.hadoop.ozone.s3.signature; import java.time.LocalDate; @@ -5,6 +22,7 @@ import org.apache.hadoop.ozone.s3.exception.OS3Exception; import org.apache.hadoop.ozone.s3.exception.S3ErrorTable; +import org.apache.hadoop.ozone.s3.signature.SignatureInfo.Version; import org.apache.hadoop.util.StringUtils; import com.google.common.annotations.VisibleForTesting; @@ -19,6 +37,9 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +/** + * Class to parse v4 auth information from header. + */ public class AuthorizationV4HeaderParser implements SignatureParser { private static final Logger LOG = @@ -31,12 +52,7 @@ public class AuthorizationV4HeaderParser implements SignatureParser { private String authHeader; public AuthorizationV4HeaderParser(String authHeader) { - if (authHeader != null) { - String[] split = authHeader.split(" "); - if (!split[0].equals(AuthorizationHeaderV2.IDENTIFIER)) { - this.authHeader = authHeader; - } - } + this.authHeader = authHeader; } /** @@ -53,7 +69,8 @@ public AuthorizationV4HeaderParser(String authHeader) { @SuppressWarnings("StringSplitter") @Override public SignatureInfo parseSignature() throws OS3Exception { - if (authHeader == null) { + if (authHeader == null || authHeader + .startsWith(AuthorizationV2HeaderParser.IDENTIFIER + " ")) { return null; } int firstSep = authHeader.indexOf(' '); @@ -73,13 +90,12 @@ public SignatureInfo parseSignature() throws OS3Exception { String signedHeaders = parseSignedHeaders(split[1]); String signature = parseSignature(split[2]); return new SignatureInfo( + Version.V4, credentialObj.getDate(), credentialObj.getAccessKeyID(), signature, signedHeaders, - String.format("%s/%s/%s/%s", credentialObj.getDate(), - credentialObj.getAwsRegion(), credentialObj.getAwsService(), - credentialObj.getAwsRequest()), + credentialObj.createScope(), algorithm ); } @@ -94,7 +110,7 @@ private String parseSignedHeaders(String signedHeadersStr) String parsedSignedHeaders = signedHeadersStr.substring(SIGNEDHEADERS.length()); Collection signedHeaders = - StringUtils.getStringCollection(signedHeadersStr, ";"); + StringUtils.getStringCollection(parsedSignedHeaders, ";"); if (signedHeaders.size() == 0) { LOG.error("No signed headers found. Authheader:{}", authHeader); throw S3ErrorTable.newError(MALFORMED_HEADER, authHeader); diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/AuthorizationV4QueryParser.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/AuthorizationV4QueryParser.java index d0fe4e82548a..7106c36d0bcf 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/AuthorizationV4QueryParser.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/AuthorizationV4QueryParser.java @@ -1,8 +1,26 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.hadoop.ozone.s3.signature; import javax.ws.rs.core.MultivaluedMap; import org.apache.hadoop.ozone.s3.exception.OS3Exception; +import org.apache.hadoop.ozone.s3.signature.SignatureInfo.Version; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -20,7 +38,9 @@ public class AuthorizationV4QueryParser implements SignatureParser { private final MultivaluedMap queryParameters; - public AuthorizationV4QueryParser(MultivaluedMap queryParameters) { + public AuthorizationV4QueryParser( + MultivaluedMap queryParameters + ) { this.queryParameters = queryParameters; } @@ -35,13 +55,12 @@ public SignatureInfo parseSignature() throws OS3Exception { new Credential(queryParameters.getFirst("X-Amz-Credential")); return new SignatureInfo( + Version.V4, queryParameters.getFirst("X-Amz-Date"), credential.getAccessKeyID(), queryParameters.getFirst("X-Amz-Signature"), queryParameters.getFirst("X-Amz-SignedHeaders"), - String.format("%s/%s/%s/%s", credential.getDate(), - credential.getAwsRegion(), credential.getAwsService(), - credential.getAwsRequest()), + credential.createScope(), queryParameters.getFirst("X-Amz-Algorithm") ); } diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/Credential.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/Credential.java index 504e42b8357c..14bf2a23cf4c 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/Credential.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/Credential.java @@ -107,4 +107,10 @@ public String getAwsRequest() { public String getCredential() { return credential; } + + public String createScope() { + return String.format("%s/%s/%s/%s", getDate(), + getAwsRegion(), getAwsService(), + getAwsRequest()); + } } diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/SignatureInfo.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/SignatureInfo.java index fb29e7699a68..6ba5675d503a 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/SignatureInfo.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/SignatureInfo.java @@ -1,7 +1,31 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.hadoop.ozone.s3.signature; +/** + * Signature and related information. + *

+ * Required to create stringToSign and token. + */ public class SignatureInfo { + private Version version; + private String date; private String awsAccessId; @@ -15,6 +39,7 @@ public class SignatureInfo { private String algorithm; public SignatureInfo( + Version version, String date, String awsAccessId, String signature, @@ -22,6 +47,7 @@ public SignatureInfo( String credentialScope, String algorithm ) { + this.version = version; this.date = date; this.awsAccessId = awsAccessId; this.signature = signature; @@ -53,4 +79,12 @@ public String getCredentialScope() { public String getAlgorithm() { return algorithm; } + + public Version getVersion() { + return version; + } + + public enum Version { + NONE, V4, V2; + } } diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/SignatureParser.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/SignatureParser.java index f6805257318b..46595738206d 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/SignatureParser.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/SignatureParser.java @@ -1,7 +1,27 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.hadoop.ozone.s3.signature; import org.apache.hadoop.ozone.s3.exception.OS3Exception; +/** + * Parser contract to extract signature information from header or query. + */ public interface SignatureParser { String AUTHORIZATION_HEADER = "Authorization"; diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/StringToSignProducer.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/StringToSignProducer.java index 896146263c26..4f8ad50b74fd 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/StringToSignProducer.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/StringToSignProducer.java @@ -1,3 +1,20 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.hadoop.ozone.s3.signature; import javax.ws.rs.container.ContainerRequestContext; @@ -9,6 +26,7 @@ import java.net.URLEncoder; import java.net.UnknownHostException; import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; import java.time.LocalDate; @@ -32,13 +50,16 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class StringToSignProducer { +/** + * Stateless utility to create stringToSign, the base of the signature. + */ +public final class StringToSignProducer { public static final String X_AMZ_CONTENT_SHA256 = "X-Amz-Content-SHA256"; public static final String X_AMAZ_DATE = "X-Amz-Date"; private static final Logger LOG = LoggerFactory.getLogger(StringToSignProducer.class); - private static final Charset UTF_8 = Charset.forName("utf-8"); + private static final Charset UTF_8 = StandardCharsets.UTF_8; private static final String NEWLINE = "\n"; private static final String HOST = "host"; private static final String UNSIGNED_PAYLOAD = "UNSIGNED-PAYLOAD"; @@ -51,6 +72,9 @@ public class StringToSignProducer { DateTimeFormatter.ofPattern("yyyyMMdd'T'HHmmss'Z'") .withZone(ZoneOffset.UTC); + private StringToSignProducer() { + } + public static String createSignatureBase( SignatureInfo signatureInfo, ContainerRequestContext context diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/TestOzoneClientProducer.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/TestOzoneClientProducer.java index f0f0581e2957..127bd4e4a29d 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/TestOzoneClientProducer.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/TestOzoneClientProducer.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + *

* http://www.apache.org/licenses/LICENSE-2.0 - * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -31,7 +31,6 @@ import org.apache.hadoop.ozone.s3.exception.OS3Exception; import org.apache.hadoop.ozone.s3.signature.AWSSignatureProcessor; -//import static org.apache.hadoop.ozone.s3.signature.SignatureProcessor.AUTHORIZATION_HEADER; import static org.apache.hadoop.ozone.s3.signature.SignatureParser.AUTHORIZATION_HEADER; import static org.apache.hadoop.ozone.s3.signature.SignatureProcessor.CONTENT_MD5; import static org.apache.hadoop.ozone.s3.signature.SignatureProcessor.CONTENT_TYPE; @@ -48,7 +47,7 @@ /** * Test class for @{@link OzoneClientProducer}. - * */ + */ @RunWith(Parameterized.class) public class TestOzoneClientProducer { @@ -61,13 +60,13 @@ public class TestOzoneClientProducer { private String amzContentSha256; private String date; private String contentType; - - private ContainerRequestContext context; private UriInfo uriInfo; - public TestOzoneClientProducer(String authHeader, String contentMd5, - String host, String amzContentSha256, String date, String contentType) + public TestOzoneClientProducer( + String authHeader, String contentMd5, + String host, String amzContentSha256, String date, String contentType + ) throws Exception { this.authHeader = authHeader; this.contentMd5 = contentMd5; @@ -87,43 +86,9 @@ public TestOzoneClientProducer(String authHeader, String contentMd5, producer.setOzoneConfiguration(config); } - @Test - public void testGetClientFailure() { - try { - producer.createClient(); - fail("testGetClientFailure"); - } catch (Exception ex) { - Assert.assertTrue(ex instanceof OS3Exception); - } - } - - private void setupContext() throws Exception { - headerMap.putSingle(AUTHORIZATION_HEADER, authHeader); - headerMap.putSingle(CONTENT_MD5, contentMd5); - headerMap.putSingle(HOST_HEADER, host); - headerMap.putSingle(X_AMZ_CONTENT_SHA256, amzContentSha256); - headerMap.putSingle(X_AMAZ_DATE, date); - headerMap.putSingle(CONTENT_TYPE, contentType); - - Mockito.when(uriInfo.getQueryParameters()).thenReturn(queryMap); - Mockito.when(uriInfo.getRequestUri()).thenReturn(new URI("")); - - Mockito.when(context.getUriInfo()).thenReturn(uriInfo); - Mockito.when(context.getHeaders()).thenReturn(headerMap); - Mockito.when(context.getHeaderString(AUTHORIZATION_HEADER)) - .thenReturn(authHeader); - Mockito.when(context.getUriInfo().getQueryParameters()) - .thenReturn(queryMap); - - AWSSignatureProcessor awsSignatureProcessor = new AWSSignatureProcessor(); - awsSignatureProcessor.setContext(context); - - producer.setSignatureParser(awsSignatureProcessor); - } - @Parameterized.Parameters public static Collection data() { - return Arrays.asList(new Object[][]{ + return Arrays.asList(new Object[][] { { "AWS4-HMAC-SHA256 Credential=testuser1/20190221/us-west-1/s3" + "/aws4_request, SignedHeaders=content-md5;host;" + @@ -157,4 +122,38 @@ public static Collection data() { }); } + @Test + public void testGetClientFailure() { + try { + producer.createClient(); + fail("testGetClientFailure"); + } catch (Exception ex) { + Assert.assertTrue(ex instanceof OS3Exception); + } + } + + private void setupContext() throws Exception { + headerMap.putSingle(AUTHORIZATION_HEADER, authHeader); + headerMap.putSingle(CONTENT_MD5, contentMd5); + headerMap.putSingle(HOST_HEADER, host); + headerMap.putSingle(X_AMZ_CONTENT_SHA256, amzContentSha256); + headerMap.putSingle(X_AMAZ_DATE, date); + headerMap.putSingle(CONTENT_TYPE, contentType); + + Mockito.when(uriInfo.getQueryParameters()).thenReturn(queryMap); + Mockito.when(uriInfo.getRequestUri()).thenReturn(new URI("")); + + Mockito.when(context.getUriInfo()).thenReturn(uriInfo); + Mockito.when(context.getHeaders()).thenReturn(headerMap); + Mockito.when(context.getHeaderString(AUTHORIZATION_HEADER)) + .thenReturn(authHeader); + Mockito.when(context.getUriInfo().getQueryParameters()) + .thenReturn(queryMap); + + AWSSignatureProcessor awsSignatureProcessor = new AWSSignatureProcessor(); + awsSignatureProcessor.setContext(context); + + producer.setSignatureParser(awsSignatureProcessor); + } + } diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/signature/TestAWSSignatureProcessor.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/signature/TestAWSSignatureProcessor.java deleted file mode 100644 index f65c4b8ca20e..000000000000 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/signature/TestAWSSignatureProcessor.java +++ /dev/null @@ -1,127 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.ozone.s3.signature; - -/** - * Test the Auth parser. - */ -public class TestAWSSignatureProcessor { -// -// @Test -// public void testV4Initialization() throws Exception { -// -// MultivaluedMap headers = new MultivaluedHashMap<>(); -// headers.putSingle("Content-Length", "123"); -// headers.putSingle("Host", "0.0.0.0:9878"); -// headers.putSingle("X-AMZ-Content-Sha256", "Content-SHA"); -// headers.putSingle("X-AMZ-Date", "123"); -// headers.putSingle("Content-Type", "ozone/mpu"); -// headers.putSingle(HeaderPreprocessor.ORIGINAL_CONTENT_TYPE, "streaming"); -// -// String authHeader = -// "AWS4-HMAC-SHA256 Credential=AKIAJWFJK62WUTKNFJJA/20181009/us-east-1" -// + "/s3/aws4_request, " -// + "SignedHeaders=host;x-amz-content-sha256;x-amz-date;" -// + "content-type, " -// + "Signature" -// + -// "=db81b057718d7c1b3b8dffa29933099551c51d787b3b13b9e0f9ebed45982bf2"; -// headers.putSingle("Authorization", -// authHeader); -// -// AuthorizationHeaderV4 parserAuthHeader = -// new AuthorizationHeaderV4(authHeader) { -// @Override -// public void validateDateRange() throws OS3Exception { -// } -// }; -// -// MultivaluedMap queryParameters = new MultivaluedHashMap<>(); -// -// UriInfo uriInfo = Mockito.mock(UriInfo.class); -// Mockito.when(uriInfo.getQueryParameters()).thenReturn(queryParameters); -// Mockito.when(uriInfo.getRequestUri()) -// .thenReturn(new URI("http://localhost/buckets")); -// -// ContainerRequestContext mock = Mockito.mock(ContainerRequestContext.class); -// Mockito.when(mock.getHeaders()).thenReturn(headers); -// Mockito.when(mock.getMethod()).thenReturn("GET"); -// Mockito.when(mock.getUriInfo()).thenReturn(uriInfo); -// -// AWSSignatureProcessor parser = new AWSSignatureProcessor() { -// @Override -// void validateSignedHeader(String header, String headerValue) -// throws OS3Exception { -// super.validateSignedHeader(header, headerValue); -// } -// }; -// parser.setV4Header(parserAuthHeader); -// parser.setContext(mock); -// parser.init(); -// -// Assert.assertTrue( -// "the ozone/mpu header is not changed back before signature processing", -// parser.buildCanonicalRequest().contains("content-type:streaming")); -// -// Assert.assertEquals( -// "String to sign is invalid", -// "AWS4-HMAC-SHA256\n" -// + "123\n" -// + "20181009/us-east-1/s3/aws4_request\n" -// + -// "f20d4de80af2271545385e8d4c7df608cae70a791c69b97aab1527ed93a0d665", -// parser.getStringToSign()); -// } -// -// @Test -// public void testV2Initialization() throws Exception { -// -// MultivaluedMap headers = new MultivaluedHashMap<>(); -// String authHeader = "AWS root:ixWQAgWvJDuqLUqgDG9o4b2HF7c="; -// headers.putSingle("Authorization", authHeader); -// -// AuthorizationHeaderV2 parserAuthHeader = -// new AuthorizationHeaderV2(authHeader); -// -// MultivaluedMap queryParameters = new MultivaluedHashMap<>(); -// -// UriInfo uriInfo = Mockito.mock(UriInfo.class); -// Mockito.when(uriInfo.getQueryParameters()).thenReturn(queryParameters); -// Mockito.when(uriInfo.getRequestUri()) -// .thenReturn(new URI("http://localhost/buckets")); -// -// ContainerRequestContext mock = Mockito.mock(ContainerRequestContext.class); -// Mockito.when(mock.getHeaders()).thenReturn(headers); -// Mockito.when(mock.getMethod()).thenReturn("GET"); -// Mockito.when(mock.getUriInfo()).thenReturn(uriInfo); -// -// AWSSignatureProcessor parser = new AWSSignatureProcessor() { -// @Override -// void validateSignedHeader(String header, String headerValue) -// throws OS3Exception { -// super.validateSignedHeader(header, headerValue); -// } -// }; -// parser.setV2Header(parserAuthHeader); -// parser.setContext(mock); -// parser.init(); -// -// Assert.assertEquals("root", parser.getAwsAccessId()); -// Assert.assertEquals("ixWQAgWvJDuqLUqgDG9o4b2HF7c=", parser.getSignature()); -// } -} \ No newline at end of file diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/signature/TestAuthorizationHeaderV2.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/signature/TestAuthorizationV2HeaderParser.java similarity index 70% rename from hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/signature/TestAuthorizationHeaderV2.java rename to hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/signature/TestAuthorizationV2HeaderParser.java index 6aa0fc84f0af..3e2373ba8d6a 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/signature/TestAuthorizationHeaderV2.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/signature/TestAuthorizationV2HeaderParser.java @@ -19,24 +19,25 @@ package org.apache.hadoop.ozone.s3.signature; import org.apache.hadoop.ozone.s3.exception.OS3Exception; -import org.junit.Test; - +import org.junit.Assert; import static org.junit.Assert.assertEquals; import static org.junit.Assert.fail; +import org.junit.Test; /** * This class tests Authorization header format v2. */ -public class TestAuthorizationHeaderV2 { +public class TestAuthorizationV2HeaderParser { @Test public void testAuthHeaderV2() throws OS3Exception { try { String auth = "AWS accessKey:signature"; - AuthorizationHeaderV2 v2 = new AuthorizationHeaderV2(auth); - assertEquals(v2.getAccessKeyID(), "accessKey"); - assertEquals(v2.getSignature(), "signature"); + AuthorizationV2HeaderParser v2 = new AuthorizationV2HeaderParser(auth); + final SignatureInfo signatureInfo = v2.parseSignature(); + assertEquals(signatureInfo.getAwsAccessId(), "accessKey"); + assertEquals(signatureInfo.getSignature(), "signature"); } catch (OS3Exception ex) { fail("testAuthHeaderV2 failed"); } @@ -44,20 +45,18 @@ public void testAuthHeaderV2() throws OS3Exception { @Test public void testIncorrectHeader1() throws OS3Exception { - try { - String auth = "AAA accessKey:signature"; - new AuthorizationHeaderV2(auth); - fail("testIncorrectHeader"); - } catch (OS3Exception ex) { - assertEquals("AuthorizationHeaderMalformed", ex.getCode()); - } + String auth = "AAA accessKey:signature"; + AuthorizationV2HeaderParser v2 = new AuthorizationV2HeaderParser(auth); + Assert.assertNull(v2.parseSignature()); + } @Test public void testIncorrectHeader2() throws OS3Exception { try { String auth = "AWS :accessKey"; - new AuthorizationHeaderV2(auth); + AuthorizationV2HeaderParser v2 = new AuthorizationV2HeaderParser(auth); + Assert.assertNull(v2.parseSignature()); fail("testIncorrectHeader"); } catch (OS3Exception ex) { assertEquals("AuthorizationHeaderMalformed", ex.getCode()); @@ -68,7 +67,8 @@ public void testIncorrectHeader2() throws OS3Exception { public void testIncorrectHeader3() throws OS3Exception { try { String auth = "AWS :signature"; - new AuthorizationHeaderV2(auth); + AuthorizationV2HeaderParser v2 = new AuthorizationV2HeaderParser(auth); + Assert.assertNull(v2.parseSignature()); fail("testIncorrectHeader"); } catch (OS3Exception ex) { assertEquals("AuthorizationHeaderMalformed", ex.getCode()); @@ -79,7 +79,8 @@ public void testIncorrectHeader3() throws OS3Exception { public void testIncorrectHeader4() throws OS3Exception { try { String auth = "AWS accessKey:"; - new AuthorizationHeaderV2(auth); + AuthorizationV2HeaderParser v2 = new AuthorizationV2HeaderParser(auth); + Assert.assertNull(v2.parseSignature()); fail("testIncorrectHeader"); } catch (OS3Exception ex) { assertEquals("AuthorizationHeaderMalformed", ex.getCode()); diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/signature/TestAuthorizationV4HeaderParser.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/signature/TestAuthorizationV4HeaderParser.java index b6010f60b0c7..953e9981633f 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/signature/TestAuthorizationV4HeaderParser.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/signature/TestAuthorizationV4HeaderParser.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -21,7 +21,9 @@ import java.time.LocalDate; import org.apache.hadoop.ozone.s3.exception.OS3Exception; +import org.apache.hadoop.test.LambdaTestUtils; +import static java.time.temporal.ChronoUnit.DAYS; import static org.apache.hadoop.ozone.s3.signature.SignatureProcessor.DATE_FORMATTER; import static org.junit.Assert.assertEquals; import static org.junit.Assert.fail; @@ -70,285 +72,284 @@ public void testV4HeaderMissingParts() { } } - @Test - public void testV4HeaderInvalidCredential() { - try { - String auth = "AWS4-HMAC-SHA256 " + - "Credential=" + curDate + "/us-east-1/s3/aws4_request, " + - "SignedHeaders=host;range;x-amz-date, " + - "Signature=fe5f80f77d5fa3beca038a248ff027"; - AuthorizationV4HeaderParser v4 = new AuthorizationV4HeaderParser(auth); - v4.parseSignature(); - fail("Exception is expected in case of malformed header"); - } catch (OS3Exception ex) { - assertEquals("AuthorizationHeaderMalformed", ex.getCode()); - } + @Test + public void testV4HeaderInvalidCredential() { + try { + String auth = "AWS4-HMAC-SHA256 " + + "Credential=" + curDate + "/us-east-1/s3/aws4_request, " + + "SignedHeaders=host;range;x-amz-date, " + + "Signature=fe5f80f77d5fa3beca038a248ff027"; + AuthorizationV4HeaderParser v4 = new AuthorizationV4HeaderParser(auth); + v4.parseSignature(); + fail("Exception is expected in case of malformed header"); + } catch (OS3Exception ex) { + assertEquals("AuthorizationHeaderMalformed", ex.getCode()); } + } + + @Test + public void testV4HeaderWithoutSpace() throws OS3Exception { + + String auth = + "AWS4-HMAC-SHA256 Credential=ozone/" + curDate + "/us-east-1/s3" + + "/aws4_request," + + "SignedHeaders=host;x-amz-content-sha256;x-amz-date," + + "Signature" + + "=fe5f80f77d5fa3beca038a248ff027"; - // @Test - // public void testV4HeaderWithoutSpace() throws OS3Exception { - // - // String auth = - // "AWS4-HMAC-SHA256 Credential=ozone/" + curDate + "/us-east-1/s3" + - // "/aws4_request," - // + "SignedHeaders=host;x-amz-content-sha256;x-amz-date," - // + "Signature" - // + "=fe5f80f77d5fa3beca038a248ff027"; - // AuthorizationHeaderV4 v4 = new AuthorizationHeaderV4(auth); - // - // assertEquals("AWS4-HMAC-SHA256", v4.getAlgorithm()); - // assertEquals("ozone", v4.getAccessKeyID()); - // assertEquals(curDate, v4.getDate()); - // assertEquals("us-east-1", v4.getAwsRegion()); - // assertEquals("aws4_request", v4.getAwsRequest()); - // assertEquals("host;x-amz-content-sha256;x-amz-date", - // v4.getSignedHeaderString()); - // assertEquals("fe5f80f77d5fa3beca038a248ff027", v4.getSignature()); - // - // } - // - // @Test - // public void testV4HeaderDateValidationSuccess() throws OS3Exception { - // // Case 1: valid date within range. - // LocalDate now = LocalDate.now(); - // String dateStr = DATE_FORMATTER.format(now); - // validateResponse(dateStr); - // - // // Case 2: Valid date with in range. - // dateStr = DATE_FORMATTER.format(now.plus(1, DAYS)); - // validateResponse(dateStr); - // - // // Case 3: Valid date with in range. - // dateStr = DATE_FORMATTER.format(now.minus(1, DAYS)); - // validateResponse(dateStr); - // } - // - // @Test - // public void testV4HeaderDateValidationFailure() throws Exception { - // // Case 1: Empty date. - // LocalDate now = LocalDate.now(); - // String dateStr = ""; - // LambdaTestUtils.intercept(OS3Exception.class, "", - // () -> validateResponse(dateStr)); - // - // // Case 2: Date after yesterday. - // String dateStr2 = DATE_FORMATTER.format(now.plus(2, DAYS)); - // LambdaTestUtils.intercept(OS3Exception.class, "", - // () -> validateResponse(dateStr2)); - // - // // Case 3: Date before yesterday. - // String dateStr3 = DATE_FORMATTER.format(now.minus(2, DAYS)); - // LambdaTestUtils.intercept(OS3Exception.class, "", - // () -> validateResponse(dateStr3)); - // } - // - // private void validateResponse(String dateStr) throws OS3Exception { - // String auth = - // "AWS4-HMAC-SHA256 Credential=ozone/" + dateStr + "/us-east-1/s3" + - // "/aws4_request," - // + "SignedHeaders=host;x-amz-content-sha256;x-amz-date," - // + "Signature" - // + "=fe5f80f77d5fa3beca038a248ff027"; - // AuthorizationHeaderV4 v4 = new AuthorizationHeaderV4(auth); - // - // assertEquals("AWS4-HMAC-SHA256", v4.getAlgorithm()); - // assertEquals("ozone", v4.getAccessKeyID()); - // assertEquals(dateStr, v4.getDate()); - // assertEquals("us-east-1", v4.getAwsRegion()); - // assertEquals("aws4_request", v4.getAwsRequest()); - // assertEquals("host;x-amz-content-sha256;x-amz-date", - // v4.getSignedHeaderString()); - // assertEquals("fe5f80f77d5fa3beca038a248ff027", v4.getSignature()); - // } - // - // @Test - // public void testV4HeaderRegionValidationFailure() throws Exception { - // String auth = - // "AWS4-HMAC-SHA256 Credential=ozone/" + curDate + - // "//s3/aws4_request," - // + "SignedHeaders=host;x-amz-content-sha256;x-amz-date," - // + "Signature" - // + "=fe5f80f77d5fa3beca038a248ff027%"; - // LambdaTestUtils.intercept(OS3Exception.class, "", - // () -> new AuthorizationHeaderV4(auth)); - // String auth2 = - // "AWS4-HMAC-SHA256 Credential=ozone/" + curDate + "s3/aws4_request," - // + "SignedHeaders=host;x-amz-content-sha256;x-amz-date," - // + "Signature" - // + "=fe5f80f77d5fa3beca038a248ff027%"; - // LambdaTestUtils.intercept(OS3Exception.class, "", - // () -> new AuthorizationHeaderV4(auth2)); - // } - // - // @Test - // public void testV4HeaderServiceValidationFailure() throws Exception { - // String auth = - // "AWS4-HMAC-SHA256 Credential=ozone/" + curDate + "/us-east-1" + - // "//aws4_request," - // + "SignedHeaders=host;x-amz-content-sha256;x-amz-date," - // + "Signature" - // + "=fe5f80f77d5fa3beca038a248ff027"; - // LambdaTestUtils.intercept(OS3Exception.class, "", - // () -> new AuthorizationHeaderV4(auth)); - // - // String auth2 = - // "AWS4-HMAC-SHA256 Credential=ozone/" + curDate + "/us-east-1" + - // "/aws4_request," - // + "SignedHeaders=host;x-amz-content-sha256;x-amz-date," - // + "Signature" - // + "=fe5f80f77d5fa3beca038a248ff027"; - // LambdaTestUtils.intercept(OS3Exception.class, "", - // () -> new AuthorizationHeaderV4(auth2)); - // } - // - // @Test - // public void testV4HeaderRequestValidationFailure() throws Exception { - // String auth = - // "AWS4-HMAC-SHA256 Credential=ozone/" + curDate + "/us-east-1/s3" + - // "/ ," - // + "SignedHeaders=host;x-amz-content-sha256;x-amz-date," - // + "Signature" - // + "=fe5f80f77d5fa3beca038a248ff027"; - // LambdaTestUtils.intercept(OS3Exception.class, "", - // () -> new AuthorizationHeaderV4(auth)); - // - // String auth2 = - // "AWS4-HMAC-SHA256 Credential=ozone/" + curDate + "/us-east-1/s3" + - // "/," - // + "SignedHeaders=host;x-amz-content-sha256;x-amz-date," - // + "Signature" - // + "=fe5f80f77d5fa3beca038a248ff027"; - // LambdaTestUtils.intercept(OS3Exception.class, "", - // () -> new AuthorizationHeaderV4(auth2)); - // - // String auth3 = - // "AWS4-HMAC-SHA256 Credential=ozone/" + curDate + "/us-east-1/s3" + - // "," - // + "SignedHeaders=host;x-amz-content-sha256;x-amz-date," - // + "Signature" - // + "=fe5f80f77d5fa3beca038a248ff027"; - // LambdaTestUtils.intercept(OS3Exception.class, "", - // () -> new AuthorizationHeaderV4(auth3)); - // } - // - // @Test - // public void testV4HeaderSignedHeaderValidationFailure() throws Exception { - // String auth = - // "AWS4-HMAC-SHA256 Credential=ozone/" + curDate + "/us-east-1/s3" + - // "/aws4_request," - // + "SignedHeaders=;;," - // + "Signature" - // + "=fe5f80f77d5fa3beca038a248ff027"; - // LambdaTestUtils.intercept(OS3Exception.class, "", - // () -> new AuthorizationHeaderV4(auth)); - // - // String auth2 = - // "AWS4-HMAC-SHA256 Credential=ozone/" + curDate + "/us-east-1/s3" + - // "/aws4_request," - // + "SignedHeaders=," - // + "Signature" - // + "=fe5f80f77d5fa3beca038a248ff027"; - // LambdaTestUtils.intercept(OS3Exception.class, "", - // () -> new AuthorizationHeaderV4(auth2)); - // - // String auth3 = - // "AWS4-HMAC-SHA256 Credential=ozone/" + curDate + "/us-east-1/s3" + - // "/aws4_request," - // + "=x-amz-content-sha256;x-amz-date," - // + "Signature" - // + "=fe5f80f77d5fa3beca038a248ff027"; - // LambdaTestUtils.intercept(OS3Exception.class, "", - // () -> new AuthorizationHeaderV4(auth3)); - // - // String auth4 = - // "AWS4-HMAC-SHA256 Credential=ozone/" + curDate + "/us-east-1/s3" + - // "/aws4_request," - // + "=," - // + "Signature" - // + "=fe5f80f77d5fa3beca038a248ff027"; - // LambdaTestUtils.intercept(OS3Exception.class, "", - // () -> new AuthorizationHeaderV4(auth4)); - // } - // - // @Test - // public void testV4HeaderSignatureValidationFailure() throws Exception { - // String auth = - // "AWS4-HMAC-SHA256 Credential=ozone/" + curDate + "/us-east-1/s3" + - // "/aws4_request," - // + "SignedHeaders=host;x-amz-content-sha256;x-amz-date," - // + "Signature" - // + "=fe5f80f77d5fa3beca038a248ff027%"; - // LambdaTestUtils.intercept(OS3Exception.class, "", - // () -> new AuthorizationHeaderV4(auth)); - // - // String auth2 = - // "AWS4-HMAC-SHA256 Credential=ozone/" + curDate + "/us-east-1/s3" + - // "/aws4_request," - // + "SignedHeaders=host;x-amz-content-sha256;x-amz-date," - // + "Signature" - // + "="; - // LambdaTestUtils.intercept(OS3Exception.class, "", - // () -> new AuthorizationHeaderV4(auth2)); - // - // String auth3 = - // "AWS4-HMAC-SHA256 Credential=ozone/" + curDate + "/us-east-1/s3" + - // "/aws4_request," - // + "SignedHeaders=host;x-amz-content-sha256;x-amz-date," - // + "" - // + "="; - // LambdaTestUtils.intercept(OS3Exception.class, "", - // () -> new AuthorizationHeaderV4(auth3)); - // } - // - // @Test - // public void testV4HeaderHashAlgoValidationFailure() throws Exception { - // String auth = - // "AWS4-HMAC-SHA Credential=ozone/" + curDate + "/us-east-1/s3" + - // "/aws4_request," - // + "SignedHeaders=host;x-amz-content-sha256;x-amz-date," - // + "Signature" - // + "=fe5f80f77d5fa3beca038a248ff027"; - // LambdaTestUtils.intercept(OS3Exception.class, "", - // () -> new AuthorizationHeaderV4(auth)); - // - // String auth2 = - // "SHA-256 Credential=ozone/" + curDate + "/us-east-1/s3" + - // "/aws4_request," - // + "SignedHeaders=host;x-amz-content-sha256;x-amz-date," - // + "Signature" - // + "=fe5f80f77d5fa3beca038a248ff027"; - // LambdaTestUtils.intercept(OS3Exception.class, "", - // () -> new AuthorizationHeaderV4(auth2)); - // - // String auth3 = - // " Credential=ozone/" + curDate + "/us-east-1/s3" + - // "/aws4_request," - // + "SignedHeaders=host;x-amz-content-sha256;x-amz-date," - // + "Signature" - // + "=fe5f80f77d5fa3beca038a248ff027"; - // LambdaTestUtils.intercept(OS3Exception.class, "", - // () -> new AuthorizationHeaderV4(auth3)); - // } - // - // @Test - // public void testV4HeaderCredentialValidationFailure() throws Exception { - // String auth = - // "AWS4-HMAC-SHA Credential=/" + curDate + "//" + - // "/," - // + "SignedHeaders=host;x-amz-content-sha256;x-amz-date," - // + "Signature" - // + "=fe5f80f77d5fa3beca038a248ff027"; - // LambdaTestUtils.intercept(OS3Exception.class, "", - // () -> new AuthorizationHeaderV4(auth)); - // - // String auth2 = - // "AWS4-HMAC-SHA =/" + curDate + "//" + - // "/," - // + "SignedHeaders=host;x-amz-content-sha256;x-amz-date," - // + "Signature" - // + "=fe5f80f77d5fa3beca038a248ff027"; - // LambdaTestUtils.intercept(OS3Exception.class, "", - // () -> new AuthorizationHeaderV4(auth2)); - // } + AuthorizationV4HeaderParser v4 = new AuthorizationV4HeaderParser(auth); + SignatureInfo signature = v4.parseSignature(); + + assertEquals("AWS4-HMAC-SHA256", signature.getAlgorithm()); + assertEquals("ozone", signature.getAwsAccessId()); + assertEquals(curDate, signature.getDate()); + assertEquals("host;x-amz-content-sha256;x-amz-date", + signature.getSignedHeaders()); + assertEquals("fe5f80f77d5fa3beca038a248ff027", signature.getSignature()); + + } + + @Test + public void testV4HeaderDateValidationSuccess() throws OS3Exception { + // Case 1: valid date within range. + LocalDate now = LocalDate.now(); + String dateStr = DATE_FORMATTER.format(now); + testRequestWithSpecificDate(dateStr); + + // Case 2: Valid date with in range. + dateStr = DATE_FORMATTER.format(now.plus(1, DAYS)); + testRequestWithSpecificDate(dateStr); + + // Case 3: Valid date with in range. + dateStr = DATE_FORMATTER.format(now.minus(1, DAYS)); + testRequestWithSpecificDate(dateStr); + } + + @Test + public void testV4HeaderDateValidationFailure() throws Exception { + // Case 1: Empty date. + LocalDate now = LocalDate.now(); + String dateStr = ""; + LambdaTestUtils.intercept(OS3Exception.class, "", + () -> testRequestWithSpecificDate(dateStr)); + + // Case 2: Date after yesterday. + String dateStr2 = DATE_FORMATTER.format(now.plus(2, DAYS)); + LambdaTestUtils.intercept(OS3Exception.class, "", + () -> testRequestWithSpecificDate(dateStr2)); + + // Case 3: Date before yesterday. + String dateStr3 = DATE_FORMATTER.format(now.minus(2, DAYS)); + LambdaTestUtils.intercept(OS3Exception.class, "", + () -> testRequestWithSpecificDate(dateStr3)); + } + + private void testRequestWithSpecificDate(String dateStr) throws OS3Exception { + String auth = + "AWS4-HMAC-SHA256 Credential=ozone/" + dateStr + "/us-east-1/s3" + + "/aws4_request," + + "SignedHeaders=host;x-amz-content-sha256;x-amz-date," + + "Signature" + + "=fe5f80f77d5fa3beca038a248ff027"; + AuthorizationV4HeaderParser v4 = new AuthorizationV4HeaderParser(auth); + SignatureInfo signature = v4.parseSignature(); + + assertEquals("AWS4-HMAC-SHA256", signature.getAlgorithm()); + assertEquals("ozone", signature.getAwsAccessId()); + assertEquals(dateStr, signature.getDate()); + assertEquals("host;x-amz-content-sha256;x-amz-date", + signature.getSignedHeaders()); + assertEquals("fe5f80f77d5fa3beca038a248ff027", signature.getSignature()); + } + + @Test + public void testV4HeaderRegionValidationFailure() throws Exception { + String auth = + "AWS4-HMAC-SHA256 Credential=ozone/" + curDate + + "//s3/aws4_request," + + "SignedHeaders=host;x-amz-content-sha256;x-amz-date," + + "Signature" + + "=fe5f80f77d5fa3beca038a248ff027%"; + LambdaTestUtils.intercept(OS3Exception.class, "", + () -> new AuthorizationV4HeaderParser(auth).parseSignature()); + String auth2 = + "AWS4-HMAC-SHA256 Credential=ozone/" + curDate + "s3/aws4_request," + + "SignedHeaders=host;x-amz-content-sha256;x-amz-date," + + "Signature" + + "=fe5f80f77d5fa3beca038a248ff027%"; + LambdaTestUtils.intercept(OS3Exception.class, "", + () -> new AuthorizationV4HeaderParser(auth2).parseSignature()); + } + + @Test + public void testV4HeaderServiceValidationFailure() throws Exception { + String auth = + "AWS4-HMAC-SHA256 Credential=ozone/" + curDate + "/us-east-1" + + "//aws4_request," + + "SignedHeaders=host;x-amz-content-sha256;x-amz-date," + + "Signature" + + "=fe5f80f77d5fa3beca038a248ff027"; + LambdaTestUtils.intercept(OS3Exception.class, "", + () -> new AuthorizationV4HeaderParser(auth).parseSignature()); + + String auth2 = + "AWS4-HMAC-SHA256 Credential=ozone/" + curDate + "/us-east-1" + + "/aws4_request," + + "SignedHeaders=host;x-amz-content-sha256;x-amz-date," + + "Signature" + + "=fe5f80f77d5fa3beca038a248ff027"; + LambdaTestUtils.intercept(OS3Exception.class, "", + () -> new AuthorizationV4HeaderParser(auth2).parseSignature()); + } + + @Test + public void testV4HeaderRequestValidationFailure() throws Exception { + String auth = + "AWS4-HMAC-SHA256 Credential=ozone/" + curDate + "/us-east-1/s3" + + "/ ," + + "SignedHeaders=host;x-amz-content-sha256;x-amz-date," + + "Signature" + + "=fe5f80f77d5fa3beca038a248ff027"; + LambdaTestUtils.intercept(OS3Exception.class, "", + () -> new AuthorizationV4HeaderParser(auth).parseSignature()); + + String auth2 = + "AWS4-HMAC-SHA256 Credential=ozone/" + curDate + "/us-east-1/s3" + + "/," + + "SignedHeaders=host;x-amz-content-sha256;x-amz-date," + + "Signature" + + "=fe5f80f77d5fa3beca038a248ff027"; + LambdaTestUtils.intercept(OS3Exception.class, "", + () -> new AuthorizationV4HeaderParser(auth2).parseSignature()); + + String auth3 = + "AWS4-HMAC-SHA256 Credential=ozone/" + curDate + "/us-east-1/s3" + + "," + + "SignedHeaders=host;x-amz-content-sha256;x-amz-date," + + "Signature" + + "=fe5f80f77d5fa3beca038a248ff027"; + LambdaTestUtils.intercept(OS3Exception.class, "", + () -> new AuthorizationV4HeaderParser(auth3).parseSignature()); + } + + @Test + public void testV4HeaderSignedHeaderValidationFailure() throws Exception { + String auth = + "AWS4-HMAC-SHA256 Credential=ozone/" + curDate + "/us-east-1/s3" + + "/aws4_request," + + "SignedHeaders=;;," + + "Signature" + + "=fe5f80f77d5fa3beca038a248ff027"; + LambdaTestUtils.intercept(OS3Exception.class, "", + () -> new AuthorizationV4HeaderParser(auth).parseSignature()); + + String auth2 = + "AWS4-HMAC-SHA256 Credential=ozone/" + curDate + "/us-east-1/s3" + + "/aws4_request," + + "SignedHeaders=," + + "Signature" + + "=fe5f80f77d5fa3beca038a248ff027"; + LambdaTestUtils.intercept(OS3Exception.class, "", + () -> new AuthorizationV4HeaderParser(auth2).parseSignature()); + + String auth3 = + "AWS4-HMAC-SHA256 Credential=ozone/" + curDate + "/us-east-1/s3" + + "/aws4_request," + + "=x-amz-content-sha256;x-amz-date," + + "Signature" + + "=fe5f80f77d5fa3beca038a248ff027"; + LambdaTestUtils.intercept(OS3Exception.class, "", + () -> new AuthorizationV4HeaderParser(auth3).parseSignature()); + + String auth4 = + "AWS4-HMAC-SHA256 Credential=ozone/" + curDate + "/us-east-1/s3" + + "/aws4_request," + + "=," + + "Signature" + + "=fe5f80f77d5fa3beca038a248ff027"; + LambdaTestUtils.intercept(OS3Exception.class, "", + () -> new AuthorizationV4HeaderParser(auth4).parseSignature()); + } + + @Test + public void testV4HeaderSignatureValidationFailure() throws Exception { + String auth = + "AWS4-HMAC-SHA256 Credential=ozone/" + curDate + "/us-east-1/s3" + + "/aws4_request," + + "SignedHeaders=host;x-amz-content-sha256;x-amz-date," + + "Signature" + + "=fe5f80f77d5fa3beca038a248ff027%"; + LambdaTestUtils.intercept(OS3Exception.class, "", + () -> new AuthorizationV4HeaderParser(auth).parseSignature()); + + String auth2 = + "AWS4-HMAC-SHA256 Credential=ozone/" + curDate + "/us-east-1/s3" + + "/aws4_request," + + "SignedHeaders=host;x-amz-content-sha256;x-amz-date," + + "Signature" + + "="; + LambdaTestUtils.intercept(OS3Exception.class, "", + () -> new AuthorizationV4HeaderParser(auth2).parseSignature()); + + String auth3 = + "AWS4-HMAC-SHA256 Credential=ozone/" + curDate + "/us-east-1/s3" + + "/aws4_request," + + "SignedHeaders=host;x-amz-content-sha256;x-amz-date," + + "" + + "="; + LambdaTestUtils.intercept(OS3Exception.class, "", + () -> new AuthorizationV4HeaderParser(auth3).parseSignature()); + } + + @Test + public void testV4HeaderHashAlgoValidationFailure() throws Exception { + String auth = + "AWS4-HMAC-SHA Credential=ozone/" + curDate + "/us-east-1/s3" + + "/aws4_request," + + "SignedHeaders=host;x-amz-content-sha256;x-amz-date," + + "Signature" + + "=fe5f80f77d5fa3beca038a248ff027"; + LambdaTestUtils.intercept(OS3Exception.class, "", + () -> new AuthorizationV4HeaderParser(auth).parseSignature()); + + String auth2 = + "SHA-256 Credential=ozone/" + curDate + "/us-east-1/s3" + + "/aws4_request," + + "SignedHeaders=host;x-amz-content-sha256;x-amz-date," + + "Signature" + + "=fe5f80f77d5fa3beca038a248ff027"; + LambdaTestUtils.intercept(OS3Exception.class, "", + () -> new AuthorizationV4HeaderParser(auth2).parseSignature()); + + String auth3 = + " Credential=ozone/" + curDate + "/us-east-1/s3" + + "/aws4_request," + + "SignedHeaders=host;x-amz-content-sha256;x-amz-date," + + "Signature" + + "=fe5f80f77d5fa3beca038a248ff027"; + LambdaTestUtils.intercept(OS3Exception.class, "", + () -> new AuthorizationV4HeaderParser(auth3).parseSignature()); + } + + @Test + public void testV4HeaderCredentialValidationFailure() throws Exception { + String auth = + "AWS4-HMAC-SHA Credential=/" + curDate + "//" + + "/," + + "SignedHeaders=host;x-amz-content-sha256;x-amz-date," + + "Signature" + + "=fe5f80f77d5fa3beca038a248ff027"; + LambdaTestUtils.intercept(OS3Exception.class, "", + () -> new AuthorizationV4HeaderParser(auth).parseSignature()); + + String auth2 = + "AWS4-HMAC-SHA =/" + curDate + "//" + + "/," + + "SignedHeaders=host;x-amz-content-sha256;x-amz-date," + + "Signature" + + "=fe5f80f77d5fa3beca038a248ff027"; + LambdaTestUtils.intercept(OS3Exception.class, "", + () -> new AuthorizationV4HeaderParser(auth2).parseSignature()); + } } From 04b230dfae488ac8f592398fe9fa6fe82b461331 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Elek=20M=C3=A1rton?= Date: Wed, 25 Nov 2020 15:53:35 +0100 Subject: [PATCH 03/11] fix rat and checkstyle problems --- .../s3/signature/StringToSignProducer.java | 4 +++- .../s3/signature/TestStringToSignProducer.java | 17 +++++++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/StringToSignProducer.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/StringToSignProducer.java index 4f8ad50b74fd..2ce89f50d3f3 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/StringToSignProducer.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/StringToSignProducer.java @@ -243,7 +243,9 @@ private static String urlEncode(String str) { } } - private static String getQueryParamString(MultivaluedMap queryMap) { + private static String getQueryParamString( + MultivaluedMap queryMap + ) { List params = new ArrayList<>(queryMap.keySet()); // Sort by name, then by value diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/signature/TestStringToSignProducer.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/signature/TestStringToSignProducer.java index 1e0751b71f09..44c6ed3ebcc2 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/signature/TestStringToSignProducer.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/signature/TestStringToSignProducer.java @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.hadoop.ozone.s3.signature; import javax.ws.rs.container.ContainerRequestContext; From fc481a313b5374d47d16e83a09db95d71abde81e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Elek=20M=C3=A1rton?= Date: Wed, 9 Dec 2020 11:07:09 +0100 Subject: [PATCH 04/11] Simplify test without mockito --- .../s3/signature/AWSSignatureProcessor.java | 27 ++++++----- .../s3/signature/StringToSignProducer.java | 3 +- .../signature/TestStringToSignProducer.java | 45 +++++++++---------- 3 files changed, 38 insertions(+), 37 deletions(-) diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/AWSSignatureProcessor.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/AWSSignatureProcessor.java index d4b818d0a4dd..a5c0afc2dc1c 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/AWSSignatureProcessor.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/AWSSignatureProcessor.java @@ -89,12 +89,10 @@ public void setContext(ContainerRequestContext context) { */ public static class LowerCaseKeyStringMap implements Map { - private HashMap delegate; + private Map delegate; - public LowerCaseKeyStringMap( - HashMap delegate - ) { - this.delegate = delegate; + public LowerCaseKeyStringMap() { + this.delegate = new HashMap<>(); } public static LowerCaseKeyStringMap fromHeaderMap( @@ -105,7 +103,7 @@ public static LowerCaseKeyStringMap fromHeaderMap( //header map is MUTABLE. It's better to save it here. (with lower case // keys!!!) final LowerCaseKeyStringMap headers = - new LowerCaseKeyStringMap(new HashMap<>()); + new LowerCaseKeyStringMap(); for (Entry> headerEntry : rawHeaders.entrySet()) { if (0 < headerEntry.getValue().size()) { @@ -120,12 +118,7 @@ public static LowerCaseKeyStringMap fromHeaderMap( } } - //in case of the HeaderPreprocessor executed before us, let's restore the - // original content type. - if (headers.containsKey(HeaderPreprocessor.ORIGINAL_CONTENT_TYPE)) { - headers.put(HeaderPreprocessor.CONTENT_TYPE, - headers.get(HeaderPreprocessor.ORIGINAL_CONTENT_TYPE)); - } + headers.fixContentType(); if (LOG.isTraceEnabled()) { headers.keySet().forEach(k -> LOG.trace("Header:{},value:{}", k, @@ -134,6 +127,16 @@ public static LowerCaseKeyStringMap fromHeaderMap( return headers; } + @VisibleForTesting + protected void fixContentType() { + //in case of the HeaderPreprocessor executed before us, let's restore the + // original content type. + if (containsKey(HeaderPreprocessor.ORIGINAL_CONTENT_TYPE)) { + put(HeaderPreprocessor.CONTENT_TYPE, + get(HeaderPreprocessor.ORIGINAL_CONTENT_TYPE)); + } + } + @Override public int size() { return delegate.size(); diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/StringToSignProducer.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/StringToSignProducer.java index 2ce89f50d3f3..f982a0aff054 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/StringToSignProducer.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/StringToSignProducer.java @@ -87,12 +87,13 @@ public static String createSignatureBase( context.getUriInfo().getQueryParameters()); } + @VisibleForTesting public static String createSignatureBase( SignatureInfo signatureInfo, String scheme, String method, String uri, - Map headers, + LowerCaseKeyStringMap headers, MultivaluedMap queryParams ) throws Exception { StringBuilder strToSign = new StringBuilder(); diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/signature/TestStringToSignProducer.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/signature/TestStringToSignProducer.java index 44c6ed3ebcc2..f6dc7f38fc69 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/signature/TestStringToSignProducer.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/signature/TestStringToSignProducer.java @@ -17,31 +17,31 @@ */ package org.apache.hadoop.ozone.s3.signature; -import javax.ws.rs.container.ContainerRequestContext; import javax.ws.rs.core.MultivaluedHashMap; import javax.ws.rs.core.MultivaluedMap; -import javax.ws.rs.core.UriInfo; -import java.net.URI; import org.apache.hadoop.ozone.s3.HeaderPreprocessor; import org.apache.hadoop.ozone.s3.exception.OS3Exception; +import org.apache.hadoop.ozone.s3.signature.AWSSignatureProcessor.LowerCaseKeyStringMap; import org.junit.Assert; import org.junit.Test; -import org.mockito.Mockito; +/** + * Test string2sign creation. + */ public class TestStringToSignProducer { @Test public void test() throws Exception { - MultivaluedMap headers = new MultivaluedHashMap<>(); - headers.putSingle("Content-Length", "123"); - headers.putSingle("Host", "0.0.0.0:9878"); - headers.putSingle("X-AMZ-Content-Sha256", "Content-SHA"); - headers.putSingle("X-AMZ-Date", "123"); - headers.putSingle("Content-Type", "ozone/mpu"); - headers.putSingle(HeaderPreprocessor.ORIGINAL_CONTENT_TYPE, "streaming"); + LowerCaseKeyStringMap headers = new LowerCaseKeyStringMap(); + headers.put("Content-Length", "123"); + headers.put("Host", "0.0.0.0:9878"); + headers.put("X-AMZ-Content-Sha256", "Content-SHA"); + headers.put("X-AMZ-Date", "123"); + headers.put("Content-Type", "ozone/mpu"); + headers.put(HeaderPreprocessor.ORIGINAL_CONTENT_TYPE, "streaming"); String authHeader = "AWS4-HMAC-SHA256 Credential=AKIAJWFJK62WUTKNFJJA/20181009/us-east-1" @@ -52,22 +52,11 @@ public void test() throws Exception { + "=db81b057718d7c1b3b8dffa29933099551c51d787b3b13b9e0f9ebed45982bf2"; - headers.putSingle("Authorization", + headers.put("Authorization", authHeader); MultivaluedMap queryParameters = new MultivaluedHashMap<>(); - UriInfo uriInfo = Mockito.mock(UriInfo.class); - Mockito.when(uriInfo.getQueryParameters()).thenReturn(queryParameters); - Mockito.when(uriInfo.getRequestUri()) - .thenReturn(new URI("http://localhost/buckets")); - - ContainerRequestContext context = - Mockito.mock(ContainerRequestContext.class); - Mockito.when(context.getHeaders()).thenReturn(headers); - Mockito.when(context.getMethod()).thenReturn("GET"); - Mockito.when(context.getUriInfo()).thenReturn(uriInfo); - final SignatureInfo signatureInfo = new AuthorizationV4HeaderParser(authHeader) { @Override @@ -77,8 +66,16 @@ public void validateDateRange(Credential credentialObj) } }.parseSignature(); + headers.fixContentType(); + final String signatureBase = - StringToSignProducer.createSignatureBase(signatureInfo, context); + StringToSignProducer.createSignatureBase( + signatureInfo, + "http", + "GET", + "/buckets", + headers, + queryParameters); Assert.assertEquals( "String to sign is invalid", From 8fe1202da1e261ce096f556b76bdcd1787bdacdb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Elek=20M=C3=A1rton?= Date: Wed, 9 Dec 2020 11:12:34 +0100 Subject: [PATCH 05/11] restore debug logging --- .../org/apache/hadoop/ozone/s3/OzoneClientProducer.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/OzoneClientProducer.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/OzoneClientProducer.java index dc9c3e43385e..6e3db73004d1 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/OzoneClientProducer.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/OzoneClientProducer.java @@ -128,7 +128,7 @@ private OzoneClient getClient(OzoneConfiguration config) } ozoneClient = - remoteUser.doAs((PrivilegedExceptionAction)() -> { + remoteUser.doAs((PrivilegedExceptionAction) () -> { return createOzoneClient(); }); } catch (OS3Exception ex) { @@ -139,9 +139,9 @@ private OzoneClient getClient(OzoneConfiguration config) } catch (Throwable t) { // For any other critical errors during object creation throw Internal // error. - // if (LOG.isDebugEnabled()) { - LOG.error("Error during Client Creation: ", t); - // } + if (LOG.isDebugEnabled()) { + LOG.error("Error during Client Creation: ", t); + } throw INTERNAL_ERROR; } return ozoneClient; From 21f7f571e169a97f6811c3ccbff8fd4d95b206fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Elek=20M=C3=A1rton?= Date: Wed, 9 Dec 2020 11:14:02 +0100 Subject: [PATCH 06/11] optimize debug log --- .../apache/hadoop/ozone/s3/signature/StringToSignProducer.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/StringToSignProducer.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/StringToSignProducer.java index f982a0aff054..f92899936737 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/StringToSignProducer.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/StringToSignProducer.java @@ -128,9 +128,10 @@ public static String createSignatureBase( strToSign.append(hash(canonicalRequest)); if (LOG.isDebugEnabled()) { LOG.debug("canonicalRequest:[{}]", canonicalRequest); + LOG.debug("StringToSign:[{}]", strToSign); } - LOG.debug("StringToSign:[{}]", strToSign); + return strToSign.toString(); } From 7dfd70591fb01379584d0801dd2b3a3c66a35a15 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Elek=20M=C3=A1rton?= Date: Wed, 9 Dec 2020 11:34:39 +0100 Subject: [PATCH 07/11] stricter header check for aws v4 header parser --- .../java/org/apache/hadoop/ozone/s3/OzoneClientProducer.java | 2 +- .../hadoop/ozone/s3/signature/AuthorizationV4HeaderParser.java | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/OzoneClientProducer.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/OzoneClientProducer.java index 6e3db73004d1..f5941b0836b2 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/OzoneClientProducer.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/OzoneClientProducer.java @@ -140,7 +140,7 @@ private OzoneClient getClient(OzoneConfiguration config) // For any other critical errors during object creation throw Internal // error. if (LOG.isDebugEnabled()) { - LOG.error("Error during Client Creation: ", t); + LOG.debug("Error during Client Creation: ", t); } throw INTERNAL_ERROR; } diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/AuthorizationV4HeaderParser.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/AuthorizationV4HeaderParser.java index d644e0535d5f..577d17362411 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/AuthorizationV4HeaderParser.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/AuthorizationV4HeaderParser.java @@ -69,8 +69,7 @@ public AuthorizationV4HeaderParser(String authHeader) { @SuppressWarnings("StringSplitter") @Override public SignatureInfo parseSignature() throws OS3Exception { - if (authHeader == null || authHeader - .startsWith(AuthorizationV2HeaderParser.IDENTIFIER + " ")) { + if (authHeader == null || !authHeader.startsWith("AWS4 ")) { return null; } int firstSep = authHeader.indexOf(' '); From 8ac5026576b70fd266cf82111778e6ac4e079161 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Elek=20M=C3=A1rton?= Date: Wed, 9 Dec 2020 14:53:35 +0100 Subject: [PATCH 08/11] fix header conditions --- .../hadoop/ozone/s3/signature/AuthorizationV2HeaderParser.java | 2 +- .../hadoop/ozone/s3/signature/AuthorizationV4HeaderParser.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/AuthorizationV2HeaderParser.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/AuthorizationV2HeaderParser.java index 3b3f7b76c7cc..569cbe52169c 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/AuthorizationV2HeaderParser.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/AuthorizationV2HeaderParser.java @@ -44,7 +44,7 @@ public AuthorizationV2HeaderParser(String authHeader) { */ @Override public SignatureInfo parseSignature() throws OS3Exception { - if (authHeader == null || !authHeader.startsWith(IDENTIFIER)) { + if (authHeader == null || !authHeader.startsWith(IDENTIFIER + " ")) { return null; } String[] split = authHeader.split(" "); diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/AuthorizationV4HeaderParser.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/AuthorizationV4HeaderParser.java index 577d17362411..14f620847102 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/AuthorizationV4HeaderParser.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/AuthorizationV4HeaderParser.java @@ -69,7 +69,7 @@ public AuthorizationV4HeaderParser(String authHeader) { @SuppressWarnings("StringSplitter") @Override public SignatureInfo parseSignature() throws OS3Exception { - if (authHeader == null || !authHeader.startsWith("AWS4 ")) { + if (authHeader == null || !authHeader.startsWith("AWS4")) { return null; } int firstSep = authHeader.indexOf(' '); From b4938f6d977b5408341b2b307bbee85f0a397839 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Elek=20M=C3=A1rton?= Date: Mon, 1 Feb 2021 15:58:30 +0100 Subject: [PATCH 09/11] Support X-Amz-Expires header --- .../s3/signature/AWSSignatureProcessor.java | 2 +- .../AuthorizationV2HeaderParser.java | 3 +- .../AuthorizationV4HeaderParser.java | 3 +- .../signature/AuthorizationV4QueryParser.java | 19 +++- .../ozone/s3/signature/SignatureInfo.java | 10 +- .../s3/signature/StringToSignProducer.java | 17 ++-- .../TestAuthorizationV4QueryParser.java | 96 +++++++++++++++++++ 7 files changed, 136 insertions(+), 14 deletions(-) create mode 100644 hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/signature/TestAuthorizationV4QueryParser.java diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/AWSSignatureProcessor.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/AWSSignatureProcessor.java index a5c0afc2dc1c..94d7350bf602 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/AWSSignatureProcessor.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/AWSSignatureProcessor.java @@ -73,7 +73,7 @@ public SignatureInfo parseSignature() throws OS3Exception { if (signatureInfo == null) { signatureInfo = new SignatureInfo( Version.NONE, - "", "", "", "", "", "" + "", "", "", "", "", "", false ); } return signatureInfo; diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/AuthorizationV2HeaderParser.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/AuthorizationV2HeaderParser.java index 569cbe52169c..7ecd1bfaaf1d 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/AuthorizationV2HeaderParser.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/AuthorizationV2HeaderParser.java @@ -75,7 +75,8 @@ public SignatureInfo parseSignature() throws OS3Exception { signature, "", "", - "" + "", + false ); } } diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/AuthorizationV4HeaderParser.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/AuthorizationV4HeaderParser.java index 14f620847102..b375f84e7b39 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/AuthorizationV4HeaderParser.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/AuthorizationV4HeaderParser.java @@ -95,7 +95,8 @@ public SignatureInfo parseSignature() throws OS3Exception { signature, signedHeaders, credentialObj.createScope(), - algorithm + algorithm, + true ); } diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/AuthorizationV4QueryParser.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/AuthorizationV4QueryParser.java index 7106c36d0bcf..b81d6c05006e 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/AuthorizationV4QueryParser.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/AuthorizationV4QueryParser.java @@ -18,10 +18,12 @@ package org.apache.hadoop.ozone.s3.signature; import javax.ws.rs.core.MultivaluedMap; +import java.time.ZonedDateTime; import org.apache.hadoop.ozone.s3.exception.OS3Exception; import org.apache.hadoop.ozone.s3.signature.SignatureInfo.Version; +import static java.time.temporal.ChronoUnit.SECONDS; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -51,17 +53,30 @@ public SignatureInfo parseSignature() throws OS3Exception { return null; } + final String dateString = queryParameters.getFirst("X-Amz-Date"); + final String expiresString = queryParameters.getFirst("X-Amz-Expires"); + if (expiresString != null && expiresString.length() > 0) { + final Long expires = + Long.valueOf(expiresString); + + if (ZonedDateTime.parse(dateString, StringToSignProducer.TIME_FORMATTER) + .plus(expires, SECONDS).isBefore(ZonedDateTime.now())) { + throw new IllegalArgumentException("Pre-signed S3 url is expired"); + } + } + Credential credential = new Credential(queryParameters.getFirst("X-Amz-Credential")); return new SignatureInfo( Version.V4, - queryParameters.getFirst("X-Amz-Date"), + dateString, credential.getAccessKeyID(), queryParameters.getFirst("X-Amz-Signature"), queryParameters.getFirst("X-Amz-SignedHeaders"), credential.createScope(), - queryParameters.getFirst("X-Amz-Algorithm") + queryParameters.getFirst("X-Amz-Algorithm"), + false ); } } diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/SignatureInfo.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/SignatureInfo.java index 6ba5675d503a..4b45ecf44ab1 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/SignatureInfo.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/SignatureInfo.java @@ -38,6 +38,8 @@ public class SignatureInfo { private String algorithm; + private boolean signPayload = true; + public SignatureInfo( Version version, String date, @@ -45,7 +47,8 @@ public SignatureInfo( String signature, String signedHeaders, String credentialScope, - String algorithm + String algorithm, + boolean signPayload ) { this.version = version; this.date = date; @@ -54,6 +57,7 @@ public SignatureInfo( this.signedHeaders = signedHeaders; this.credentialScope = credentialScope; this.algorithm = algorithm; + this.signPayload = signPayload; } public String getAwsAccessId() { @@ -84,6 +88,10 @@ public Version getVersion() { return version; } + public boolean isSignPayload() { + return signPayload; + } + public enum Version { NONE, V4, V2; } diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/StringToSignProducer.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/StringToSignProducer.java index f92899936737..4d92801b274c 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/StringToSignProducer.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/StringToSignProducer.java @@ -68,7 +68,7 @@ public final class StringToSignProducer { */ private static final long PRESIGN_URL_MAX_EXPIRATION_SECONDS = 60 * 60 * 24 * 7; - private static final DateTimeFormatter TIME_FORMATTER = + public static final DateTimeFormatter TIME_FORMATTER = DateTimeFormatter.ofPattern("yyyyMMdd'T'HHmmss'Z'") .withZone(ZoneOffset.UTC); @@ -108,8 +108,7 @@ public static String createSignatureBase( // RequestDateTime + \n + // CredentialScope + \n + // HashedCanonicalRequest - String credentialScope, canonicalRequest; - credentialScope = signatureInfo.getCredentialScope(); + String credentialScope = signatureInfo.getCredentialScope(); // If the absolute path is empty, use a forward slash (/) uri = (uri.trim().length() > 0) ? uri : "/"; @@ -118,20 +117,21 @@ public static String createSignatureBase( strToSign.append(headers.get(X_AMAZ_DATE) + NEWLINE); strToSign.append(credentialScope + NEWLINE); - canonicalRequest = buildCanonicalRequest( + String canonicalRequest = buildCanonicalRequest( scheme, method, uri, signatureInfo.getSignedHeaders(), headers, - queryParams); + queryParams, + !signatureInfo.isSignPayload()); + strToSign.append(hash(canonicalRequest)); if (LOG.isDebugEnabled()) { LOG.debug("canonicalRequest:[{}]", canonicalRequest); LOG.debug("StringToSign:[{}]", strToSign); } - return strToSign.toString(); } @@ -148,7 +148,8 @@ public static String buildCanonicalRequest( String uri, String signedHeaders, Map headers, - MultivaluedMap queryParams + MultivaluedMap queryParams, + boolean unsignedPayload ) throws OS3Exception { Iterable parts = split("/", uri); @@ -181,7 +182,7 @@ public static String buildCanonicalRequest( String payloadHash; if (UNSIGNED_PAYLOAD.equals( - headers.get(X_AMZ_CONTENT_SHA256))) { + headers.get(X_AMZ_CONTENT_SHA256)) || unsignedPayload) { payloadHash = UNSIGNED_PAYLOAD; } else { payloadHash = headers.get(X_AMZ_CONTENT_SHA256); diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/signature/TestAuthorizationV4QueryParser.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/signature/TestAuthorizationV4QueryParser.java new file mode 100644 index 000000000000..c3ecae5ef596 --- /dev/null +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/signature/TestAuthorizationV4QueryParser.java @@ -0,0 +1,96 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.s3.signature; + +import javax.ws.rs.core.MultivaluedMap; + +import java.time.ZonedDateTime; + +import org.junit.Assert; +import org.junit.Test; +import org.mockito.Mockito; + +public class TestAuthorizationV4QueryParser { + + @Test(expected = IllegalArgumentException.class) + public void testExpiredHeaders() throws Exception { + + //GIVEN + final MultivaluedMap parameters = Mockito.mock(MultivaluedMap.class); + Mockito.when(parameters.getFirst("X-Amz-Date")) + .thenReturn("20160801T083241Z"); + Mockito.when(parameters.getFirst("X-Amz-Expires")).thenReturn("10000"); + Mockito.when(parameters.containsKey("X-Amz-Signature")).thenReturn(true); + + AuthorizationV4QueryParser parser = + new AuthorizationV4QueryParser(parameters); + + //WHEN + parser.parseSignature(); + + //THEN + Assert.fail("Expired header is not detected"); + } + + @Test() + public void testUnExpiredHeaders() throws Exception { + + //GIVEN + final MultivaluedMap parameters = Mockito.mock(MultivaluedMap.class); + Mockito.when(parameters.getFirst("X-Amz-Date")) + .thenReturn( + ZonedDateTime.now().format(StringToSignProducer.TIME_FORMATTER)); + Mockito.when(parameters.getFirst("X-Amz-Expires")).thenReturn("10000"); + Mockito.when(parameters.getFirst("X-Amz-Credential")) + .thenReturn("AKIAIOSFODNN7EXAMPLE/20130524/us-east-1/s3/aws4_request"); + Mockito.when(parameters.containsKey("X-Amz-Signature")).thenReturn(true); + + AuthorizationV4QueryParser parser = + new AuthorizationV4QueryParser(parameters); + + //WHEN + parser.parseSignature(); + + //THEN + //passed + } + + + @Test() + public void testWithoutEpiration() throws Exception { + + //GIVEN + final MultivaluedMap parameters = Mockito.mock(MultivaluedMap.class); + Mockito.when(parameters.getFirst("X-Amz-Date")) + .thenReturn( + ZonedDateTime.now().format(StringToSignProducer.TIME_FORMATTER)); + Mockito.when(parameters.getFirst("X-Amz-Credential")) + .thenReturn("AKIAIOSFODNN7EXAMPLE/20130524/us-east-1/s3/aws4_request"); + Mockito.when(parameters.containsKey("X-Amz-Signature")).thenReturn(true); + + AuthorizationV4QueryParser parser = + new AuthorizationV4QueryParser(parameters); + + //WHEN + parser.parseSignature(); + + //THEN + //passed + } +} \ No newline at end of file From ae957803458a1c8f59cd77979e869e9136e6bbe8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Elek=20M=C3=A1rton?= Date: Tue, 2 Feb 2021 20:44:30 +0100 Subject: [PATCH 10/11] fixing query based authorization --- .../s3/signature/AWSSignatureProcessor.java | 10 +- .../AuthorizationV2HeaderParser.java | 1 + .../AuthorizationV4HeaderParser.java | 6 +- .../signature/AuthorizationV4QueryParser.java | 52 +++++---- .../ozone/s3/signature/SignatureInfo.java | 15 +++ .../s3/signature/StringToSignProducer.java | 42 +++++--- .../TestAuthorizationV4HeaderParser.java | 78 +++++++++----- .../TestAuthorizationV4QueryParser.java | 100 +++++++++++++----- .../signature/TestStringToSignProducer.java | 8 +- 9 files changed, 220 insertions(+), 92 deletions(-) diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/AWSSignatureProcessor.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/AWSSignatureProcessor.java index 94d7350bf602..807b473c71ee 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/AWSSignatureProcessor.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/AWSSignatureProcessor.java @@ -44,7 +44,7 @@ @RequestScoped public class AWSSignatureProcessor implements SignatureProcessor { - private final static Logger LOG = + private static final Logger LOG = LoggerFactory.getLogger(AWSSignatureProcessor.class); @Context @@ -58,9 +58,11 @@ public SignatureInfo parseSignature() throws OS3Exception { String authHeader = headers.get("Authorization"); List signatureParsers = new ArrayList<>(); - signatureParsers.add(new AuthorizationV4HeaderParser(authHeader)); + signatureParsers.add(new AuthorizationV4HeaderParser(authHeader, + headers.get(StringToSignProducer.X_AMAZ_DATE))); signatureParsers.add(new AuthorizationV4QueryParser( - context.getUriInfo().getQueryParameters())); + StringToSignProducer.fromMultiValueToSingleValueMap( + context.getUriInfo().getQueryParameters()))); signatureParsers.add(new AuthorizationV2HeaderParser(authHeader)); SignatureInfo signatureInfo = null; @@ -73,7 +75,7 @@ public SignatureInfo parseSignature() throws OS3Exception { if (signatureInfo == null) { signatureInfo = new SignatureInfo( Version.NONE, - "", "", "", "", "", "", false + "", "", "", "", "", "", "", false ); } return signatureInfo; diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/AuthorizationV2HeaderParser.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/AuthorizationV2HeaderParser.java index fcdd5cb0145d..1681dea8e178 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/AuthorizationV2HeaderParser.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/AuthorizationV2HeaderParser.java @@ -71,6 +71,7 @@ public SignatureInfo parseSignature() throws OS3Exception { return new SignatureInfo( Version.V2, "", + "", accessKeyID, signature, "", diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/AuthorizationV4HeaderParser.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/AuthorizationV4HeaderParser.java index c8ba3316afe6..f62d0f1e3efe 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/AuthorizationV4HeaderParser.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/AuthorizationV4HeaderParser.java @@ -51,8 +51,11 @@ public class AuthorizationV4HeaderParser implements SignatureParser { private String authHeader; - public AuthorizationV4HeaderParser(String authHeader) { + private String dateHeader; + + public AuthorizationV4HeaderParser(String authHeader, String dateHeader) { this.authHeader = authHeader; + this.dateHeader = dateHeader; } /** @@ -91,6 +94,7 @@ public SignatureInfo parseSignature() throws OS3Exception { return new SignatureInfo( Version.V4, credentialObj.getDate(), + dateHeader, credentialObj.getAccessKeyID(), signature, signedHeaders, diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/AuthorizationV4QueryParser.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/AuthorizationV4QueryParser.java index b81d6c05006e..fbf45946fdfd 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/AuthorizationV4QueryParser.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/AuthorizationV4QueryParser.java @@ -17,12 +17,15 @@ */ package org.apache.hadoop.ozone.s3.signature; -import javax.ws.rs.core.MultivaluedMap; +import java.io.UnsupportedEncodingException; +import java.net.URLDecoder; import java.time.ZonedDateTime; +import java.util.Map; import org.apache.hadoop.ozone.s3.exception.OS3Exception; import org.apache.hadoop.ozone.s3.signature.SignatureInfo.Version; +import com.google.common.annotations.VisibleForTesting; import static java.time.temporal.ChronoUnit.SECONDS; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -38,10 +41,10 @@ public class AuthorizationV4QueryParser implements SignatureParser { private static final Logger LOG = LoggerFactory.getLogger(AuthorizationV4QueryParser.class); - private final MultivaluedMap queryParameters; + private final Map queryParameters; public AuthorizationV4QueryParser( - MultivaluedMap queryParameters + Map queryParameters ) { this.queryParameters = queryParameters; } @@ -53,30 +56,43 @@ public SignatureInfo parseSignature() throws OS3Exception { return null; } - final String dateString = queryParameters.getFirst("X-Amz-Date"); - final String expiresString = queryParameters.getFirst("X-Amz-Expires"); - if (expiresString != null && expiresString.length() > 0) { - final Long expires = - Long.valueOf(expiresString); + validateDateAndExpires(); - if (ZonedDateTime.parse(dateString, StringToSignProducer.TIME_FORMATTER) - .plus(expires, SECONDS).isBefore(ZonedDateTime.now())) { - throw new IllegalArgumentException("Pre-signed S3 url is expired"); - } - } + final String rawCredential = queryParameters.get("X-Amz-Credential"); Credential credential = - new Credential(queryParameters.getFirst("X-Amz-Credential")); + null; + try { + credential = new Credential(URLDecoder.decode(rawCredential, "UTF-8")); + } catch (UnsupportedEncodingException e) { + throw new IllegalArgumentException( + "X-Amz-Credential is not proper URL encoded"); + } return new SignatureInfo( Version.V4, - dateString, + credential.getDate(), + queryParameters.get("X-Amz-Date"), credential.getAccessKeyID(), - queryParameters.getFirst("X-Amz-Signature"), - queryParameters.getFirst("X-Amz-SignedHeaders"), + queryParameters.get("X-Amz-Signature"), + queryParameters.get("X-Amz-SignedHeaders"), credential.createScope(), - queryParameters.getFirst("X-Amz-Algorithm"), + queryParameters.get("X-Amz-Algorithm"), false ); } + + @VisibleForTesting + protected void validateDateAndExpires() { + final String dateString = queryParameters.get("X-Amz-Date"); + final String expiresString = queryParameters.get("X-Amz-Expires"); + if (expiresString != null && expiresString.length() > 0) { + final Long expires = Long.valueOf(expiresString); + + if (ZonedDateTime.parse(dateString, StringToSignProducer.TIME_FORMATTER) + .plus(expires, SECONDS).isBefore(ZonedDateTime.now())) { + throw new IllegalArgumentException("Pre-signed S3 url is expired"); + } + } + } } diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/SignatureInfo.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/SignatureInfo.java index 4b45ecf44ab1..cadbd5d0d511 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/SignatureInfo.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/SignatureInfo.java @@ -26,8 +26,16 @@ public class SignatureInfo { private Version version; + /** + * Information comes from the credential (Date only). + */ private String date; + /** + * Information comes from header/query param (full timestamp). + */ + private String dateTime; + private String awsAccessId; private String signature; @@ -40,9 +48,11 @@ public class SignatureInfo { private boolean signPayload = true; + @SuppressWarnings("checkstyle:ParameterNumber") public SignatureInfo( Version version, String date, + String dateTime, String awsAccessId, String signature, String signedHeaders, @@ -52,6 +62,7 @@ public SignatureInfo( ) { this.version = version; this.date = date; + this.dateTime = dateTime; this.awsAccessId = awsAccessId; this.signature = signature; this.signedHeaders = signedHeaders; @@ -92,6 +103,10 @@ public boolean isSignPayload() { return signPayload; } + public String getDateTime() { + return dateTime; + } + public enum Version { NONE, V4, V2; } diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/StringToSignProducer.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/StringToSignProducer.java index 4d92801b274c..7d8b417056f8 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/StringToSignProducer.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/StringToSignProducer.java @@ -34,6 +34,7 @@ import java.time.format.DateTimeFormatter; import java.util.ArrayList; import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.regex.Matcher; @@ -84,7 +85,8 @@ public static String createSignatureBase( context.getMethod(), context.getUriInfo().getRequestUri().getPath(), LowerCaseKeyStringMap.fromHeaderMap(context.getHeaders()), - context.getUriInfo().getQueryParameters()); + fromMultiValueToSingleValueMap( + context.getUriInfo().getQueryParameters())); } @VisibleForTesting @@ -94,7 +96,7 @@ public static String createSignatureBase( String method, String uri, LowerCaseKeyStringMap headers, - MultivaluedMap queryParams + Map queryParams ) throws Exception { StringBuilder strToSign = new StringBuilder(); // According to AWS sigv4 documentation, authorization header should be @@ -114,7 +116,7 @@ public static String createSignatureBase( uri = (uri.trim().length() > 0) ? uri : "/"; // Encode URI and preserve forward slashes strToSign.append(signatureInfo.getAlgorithm() + NEWLINE); - strToSign.append(headers.get(X_AMAZ_DATE) + NEWLINE); + strToSign.append(signatureInfo.getDateTime() + NEWLINE); strToSign.append(credentialScope + NEWLINE); String canonicalRequest = buildCanonicalRequest( @@ -125,7 +127,7 @@ public static String createSignatureBase( headers, queryParams, !signatureInfo.isSignPayload()); - + System.out.println(canonicalRequest); strToSign.append(hash(canonicalRequest)); if (LOG.isDebugEnabled()) { LOG.debug("canonicalRequest:[{}]", canonicalRequest); @@ -135,6 +137,16 @@ public static String createSignatureBase( return strToSign.toString(); } + public static Map fromMultiValueToSingleValueMap( + MultivaluedMap queryParameters + ) { + Map result = new HashMap<>(); + for (String key : queryParameters.keySet()) { + result.put(key, queryParameters.getFirst(key)); + } + return result; + } + public static String hash(String payload) throws NoSuchAlgorithmException { MessageDigest md = MessageDigest.getInstance("SHA-256"); md.update(payload.getBytes(UTF_8)); @@ -148,7 +160,7 @@ public static String buildCanonicalRequest( String uri, String signedHeaders, Map headers, - MultivaluedMap queryParams, + Map queryParams, boolean unsignedPayload ) throws OS3Exception { @@ -193,7 +205,6 @@ public static String buildCanonicalRequest( + canonicalHeaders + NEWLINE + signedHeaders + NEWLINE + payloadHash; - return canonicalRequest; } @@ -247,24 +258,27 @@ private static String urlEncode(String str) { } private static String getQueryParamString( - MultivaluedMap queryMap + Map queryMap ) { List params = new ArrayList<>(queryMap.keySet()); // Sort by name, then by value Collections.sort(params, (o1, o2) -> o1.equals(o2) ? - queryMap.getFirst(o1).compareTo(queryMap.getFirst(o2)) : + queryMap.get(o1).compareTo(queryMap.get(o2)) : o1.compareTo(o2)); StringBuilder result = new StringBuilder(); for (String p : params) { - if (result.length() > 0) { - result.append("&"); - } - result.append(urlEncode(p)); - result.append('='); + if (!p.equals("X-Amz-Signature")) { - result.append(urlEncode(queryMap.getFirst(p))); + if (result.length() > 0) { + result.append("&"); + } + result.append(urlEncode(p)); + result.append('='); + + result.append(urlEncode(queryMap.get(p))); + } } return result.toString(); } diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/signature/TestAuthorizationV4HeaderParser.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/signature/TestAuthorizationV4HeaderParser.java index 953e9981633f..6f7081af371a 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/signature/TestAuthorizationV4HeaderParser.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/signature/TestAuthorizationV4HeaderParser.java @@ -25,6 +25,7 @@ import static java.time.temporal.ChronoUnit.DAYS; import static org.apache.hadoop.ozone.s3.signature.SignatureProcessor.DATE_FORMATTER; +import org.junit.Assert; import static org.junit.Assert.assertEquals; import static org.junit.Assert.fail; import org.junit.Before; @@ -35,6 +36,9 @@ */ public class TestAuthorizationV4HeaderParser { + + private static final String SAMPLE_DATE = "20210202T144559Z"; + private String curDate; @Before @@ -49,7 +53,8 @@ public void testV4HeaderWellFormed() throws Exception { "Credential=ozone/" + curDate + "/us-east-1/s3/aws4_request, " + "SignedHeaders=host;range;x-amz-date, " + "Signature=fe5f80f77d5fa3beca038a248ff027"; - AuthorizationV4HeaderParser v4 = new AuthorizationV4HeaderParser(auth); + AuthorizationV4HeaderParser v4 = + new AuthorizationV4HeaderParser(auth, SAMPLE_DATE); final SignatureInfo signatureInfo = v4.parseSignature(); assertEquals("ozone", signatureInfo.getAwsAccessId()); assertEquals(curDate, signatureInfo.getDate()); @@ -64,7 +69,8 @@ public void testV4HeaderMissingParts() { String auth = "AWS4-HMAC-SHA256 " + "Credential=ozone/" + curDate + "/us-east-1/s3/aws4_request, " + "SignedHeaders=host;range;x-amz-date,"; - AuthorizationV4HeaderParser v4 = new AuthorizationV4HeaderParser(auth); + AuthorizationV4HeaderParser v4 = + new AuthorizationV4HeaderParser(auth, SAMPLE_DATE); v4.parseSignature(); fail("Exception is expected in case of malformed header"); } catch (OS3Exception ex) { @@ -79,7 +85,8 @@ public void testV4HeaderInvalidCredential() { "Credential=" + curDate + "/us-east-1/s3/aws4_request, " + "SignedHeaders=host;range;x-amz-date, " + "Signature=fe5f80f77d5fa3beca038a248ff027"; - AuthorizationV4HeaderParser v4 = new AuthorizationV4HeaderParser(auth); + AuthorizationV4HeaderParser v4 = + new AuthorizationV4HeaderParser(auth, SAMPLE_DATE); v4.parseSignature(); fail("Exception is expected in case of malformed header"); } catch (OS3Exception ex) { @@ -97,7 +104,8 @@ public void testV4HeaderWithoutSpace() throws OS3Exception { + "Signature" + "=fe5f80f77d5fa3beca038a248ff027"; - AuthorizationV4HeaderParser v4 = new AuthorizationV4HeaderParser(auth); + AuthorizationV4HeaderParser v4 = new AuthorizationV4HeaderParser(auth, + SAMPLE_DATE); SignatureInfo signature = v4.parseSignature(); assertEquals("AWS4-HMAC-SHA256", signature.getAlgorithm()); @@ -151,7 +159,8 @@ private void testRequestWithSpecificDate(String dateStr) throws OS3Exception { + "SignedHeaders=host;x-amz-content-sha256;x-amz-date," + "Signature" + "=fe5f80f77d5fa3beca038a248ff027"; - AuthorizationV4HeaderParser v4 = new AuthorizationV4HeaderParser(auth); + AuthorizationV4HeaderParser v4 = + new AuthorizationV4HeaderParser(auth, SAMPLE_DATE); SignatureInfo signature = v4.parseSignature(); assertEquals("AWS4-HMAC-SHA256", signature.getAlgorithm()); @@ -171,14 +180,16 @@ public void testV4HeaderRegionValidationFailure() throws Exception { + "Signature" + "=fe5f80f77d5fa3beca038a248ff027%"; LambdaTestUtils.intercept(OS3Exception.class, "", - () -> new AuthorizationV4HeaderParser(auth).parseSignature()); + () -> new AuthorizationV4HeaderParser(auth, SAMPLE_DATE) + .parseSignature()); String auth2 = "AWS4-HMAC-SHA256 Credential=ozone/" + curDate + "s3/aws4_request," + "SignedHeaders=host;x-amz-content-sha256;x-amz-date," + "Signature" + "=fe5f80f77d5fa3beca038a248ff027%"; LambdaTestUtils.intercept(OS3Exception.class, "", - () -> new AuthorizationV4HeaderParser(auth2).parseSignature()); + () -> new AuthorizationV4HeaderParser(auth2, SAMPLE_DATE) + .parseSignature()); } @Test @@ -190,7 +201,8 @@ public void testV4HeaderServiceValidationFailure() throws Exception { + "Signature" + "=fe5f80f77d5fa3beca038a248ff027"; LambdaTestUtils.intercept(OS3Exception.class, "", - () -> new AuthorizationV4HeaderParser(auth).parseSignature()); + () -> new AuthorizationV4HeaderParser(auth, SAMPLE_DATE) + .parseSignature()); String auth2 = "AWS4-HMAC-SHA256 Credential=ozone/" + curDate + "/us-east-1" + @@ -199,7 +211,8 @@ public void testV4HeaderServiceValidationFailure() throws Exception { + "Signature" + "=fe5f80f77d5fa3beca038a248ff027"; LambdaTestUtils.intercept(OS3Exception.class, "", - () -> new AuthorizationV4HeaderParser(auth2).parseSignature()); + () -> new AuthorizationV4HeaderParser(auth2, SAMPLE_DATE) + .parseSignature()); } @Test @@ -211,7 +224,8 @@ public void testV4HeaderRequestValidationFailure() throws Exception { + "Signature" + "=fe5f80f77d5fa3beca038a248ff027"; LambdaTestUtils.intercept(OS3Exception.class, "", - () -> new AuthorizationV4HeaderParser(auth).parseSignature()); + () -> new AuthorizationV4HeaderParser(auth, SAMPLE_DATE) + .parseSignature()); String auth2 = "AWS4-HMAC-SHA256 Credential=ozone/" + curDate + "/us-east-1/s3" + @@ -220,7 +234,8 @@ public void testV4HeaderRequestValidationFailure() throws Exception { + "Signature" + "=fe5f80f77d5fa3beca038a248ff027"; LambdaTestUtils.intercept(OS3Exception.class, "", - () -> new AuthorizationV4HeaderParser(auth2).parseSignature()); + () -> new AuthorizationV4HeaderParser(auth2, SAMPLE_DATE) + .parseSignature()); String auth3 = "AWS4-HMAC-SHA256 Credential=ozone/" + curDate + "/us-east-1/s3" + @@ -229,7 +244,8 @@ public void testV4HeaderRequestValidationFailure() throws Exception { + "Signature" + "=fe5f80f77d5fa3beca038a248ff027"; LambdaTestUtils.intercept(OS3Exception.class, "", - () -> new AuthorizationV4HeaderParser(auth3).parseSignature()); + () -> new AuthorizationV4HeaderParser(auth3, SAMPLE_DATE) + .parseSignature()); } @Test @@ -241,7 +257,8 @@ public void testV4HeaderSignedHeaderValidationFailure() throws Exception { + "Signature" + "=fe5f80f77d5fa3beca038a248ff027"; LambdaTestUtils.intercept(OS3Exception.class, "", - () -> new AuthorizationV4HeaderParser(auth).parseSignature()); + () -> new AuthorizationV4HeaderParser(auth, SAMPLE_DATE) + .parseSignature()); String auth2 = "AWS4-HMAC-SHA256 Credential=ozone/" + curDate + "/us-east-1/s3" + @@ -250,7 +267,8 @@ public void testV4HeaderSignedHeaderValidationFailure() throws Exception { + "Signature" + "=fe5f80f77d5fa3beca038a248ff027"; LambdaTestUtils.intercept(OS3Exception.class, "", - () -> new AuthorizationV4HeaderParser(auth2).parseSignature()); + () -> new AuthorizationV4HeaderParser(auth2, SAMPLE_DATE) + .parseSignature()); String auth3 = "AWS4-HMAC-SHA256 Credential=ozone/" + curDate + "/us-east-1/s3" + @@ -259,7 +277,8 @@ public void testV4HeaderSignedHeaderValidationFailure() throws Exception { + "Signature" + "=fe5f80f77d5fa3beca038a248ff027"; LambdaTestUtils.intercept(OS3Exception.class, "", - () -> new AuthorizationV4HeaderParser(auth3).parseSignature()); + () -> new AuthorizationV4HeaderParser(auth3, SAMPLE_DATE) + .parseSignature()); String auth4 = "AWS4-HMAC-SHA256 Credential=ozone/" + curDate + "/us-east-1/s3" + @@ -268,7 +287,8 @@ public void testV4HeaderSignedHeaderValidationFailure() throws Exception { + "Signature" + "=fe5f80f77d5fa3beca038a248ff027"; LambdaTestUtils.intercept(OS3Exception.class, "", - () -> new AuthorizationV4HeaderParser(auth4).parseSignature()); + () -> new AuthorizationV4HeaderParser(auth4, SAMPLE_DATE) + .parseSignature()); } @Test @@ -280,7 +300,8 @@ public void testV4HeaderSignatureValidationFailure() throws Exception { + "Signature" + "=fe5f80f77d5fa3beca038a248ff027%"; LambdaTestUtils.intercept(OS3Exception.class, "", - () -> new AuthorizationV4HeaderParser(auth).parseSignature()); + () -> new AuthorizationV4HeaderParser(auth, SAMPLE_DATE) + .parseSignature()); String auth2 = "AWS4-HMAC-SHA256 Credential=ozone/" + curDate + "/us-east-1/s3" + @@ -289,7 +310,8 @@ public void testV4HeaderSignatureValidationFailure() throws Exception { + "Signature" + "="; LambdaTestUtils.intercept(OS3Exception.class, "", - () -> new AuthorizationV4HeaderParser(auth2).parseSignature()); + () -> new AuthorizationV4HeaderParser(auth2, SAMPLE_DATE) + .parseSignature()); String auth3 = "AWS4-HMAC-SHA256 Credential=ozone/" + curDate + "/us-east-1/s3" + @@ -298,7 +320,8 @@ public void testV4HeaderSignatureValidationFailure() throws Exception { + "" + "="; LambdaTestUtils.intercept(OS3Exception.class, "", - () -> new AuthorizationV4HeaderParser(auth3).parseSignature()); + () -> new AuthorizationV4HeaderParser(auth3, SAMPLE_DATE) + .parseSignature()); } @Test @@ -310,7 +333,8 @@ public void testV4HeaderHashAlgoValidationFailure() throws Exception { + "Signature" + "=fe5f80f77d5fa3beca038a248ff027"; LambdaTestUtils.intercept(OS3Exception.class, "", - () -> new AuthorizationV4HeaderParser(auth).parseSignature()); + () -> new AuthorizationV4HeaderParser(auth, SAMPLE_DATE) + .parseSignature()); String auth2 = "SHA-256 Credential=ozone/" + curDate + "/us-east-1/s3" + @@ -318,8 +342,8 @@ public void testV4HeaderHashAlgoValidationFailure() throws Exception { + "SignedHeaders=host;x-amz-content-sha256;x-amz-date," + "Signature" + "=fe5f80f77d5fa3beca038a248ff027"; - LambdaTestUtils.intercept(OS3Exception.class, "", - () -> new AuthorizationV4HeaderParser(auth2).parseSignature()); + Assert.assertNull(new AuthorizationV4HeaderParser(auth2, SAMPLE_DATE) + .parseSignature()); String auth3 = " Credential=ozone/" + curDate + "/us-east-1/s3" + @@ -327,8 +351,8 @@ public void testV4HeaderHashAlgoValidationFailure() throws Exception { + "SignedHeaders=host;x-amz-content-sha256;x-amz-date," + "Signature" + "=fe5f80f77d5fa3beca038a248ff027"; - LambdaTestUtils.intercept(OS3Exception.class, "", - () -> new AuthorizationV4HeaderParser(auth3).parseSignature()); + Assert.assertNull(new AuthorizationV4HeaderParser(auth2, SAMPLE_DATE) + .parseSignature()); } @Test @@ -340,7 +364,8 @@ public void testV4HeaderCredentialValidationFailure() throws Exception { + "Signature" + "=fe5f80f77d5fa3beca038a248ff027"; LambdaTestUtils.intercept(OS3Exception.class, "", - () -> new AuthorizationV4HeaderParser(auth).parseSignature()); + () -> new AuthorizationV4HeaderParser(auth, SAMPLE_DATE) + .parseSignature()); String auth2 = "AWS4-HMAC-SHA =/" + curDate + "//" + @@ -349,7 +374,8 @@ public void testV4HeaderCredentialValidationFailure() throws Exception { + "Signature" + "=fe5f80f77d5fa3beca038a248ff027"; LambdaTestUtils.intercept(OS3Exception.class, "", - () -> new AuthorizationV4HeaderParser(auth2).parseSignature()); + () -> new AuthorizationV4HeaderParser(auth2, SAMPLE_DATE) + .parseSignature()); } } diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/signature/TestAuthorizationV4QueryParser.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/signature/TestAuthorizationV4QueryParser.java index c3ecae5ef596..5f66bf05e597 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/signature/TestAuthorizationV4QueryParser.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/signature/TestAuthorizationV4QueryParser.java @@ -18,13 +18,14 @@ package org.apache.hadoop.ozone.s3.signature; -import javax.ws.rs.core.MultivaluedMap; - import java.time.ZonedDateTime; +import java.util.HashMap; +import java.util.Map; + +import org.apache.hadoop.ozone.s3.signature.AWSSignatureProcessor.LowerCaseKeyStringMap; import org.junit.Assert; import org.junit.Test; -import org.mockito.Mockito; public class TestAuthorizationV4QueryParser { @@ -32,11 +33,14 @@ public class TestAuthorizationV4QueryParser { public void testExpiredHeaders() throws Exception { //GIVEN - final MultivaluedMap parameters = Mockito.mock(MultivaluedMap.class); - Mockito.when(parameters.getFirst("X-Amz-Date")) - .thenReturn("20160801T083241Z"); - Mockito.when(parameters.getFirst("X-Amz-Expires")).thenReturn("10000"); - Mockito.when(parameters.containsKey("X-Amz-Signature")).thenReturn(true); + Map parameters = new HashMap<>(); + parameters.put("X-Amz-Algorithm", "AWS4-HMAC-SHA256"); + parameters.put("X-Amz-Credential", + "AKIAIOSFODNN7EXAMPLE%2F20130524%2Fus-east-1%2Fs3%2Faws4_request"); + parameters.put("X-Amz-Date", "20160801T083241Z"); + parameters.put("X-Amz-Expires", "10000"); + parameters.put("X-Amz-Signature", + "aeeed9bbccd4d02ee5c0109b86d86835f995330da4c265957d157751f604d404"); AuthorizationV4QueryParser parser = new AuthorizationV4QueryParser(parameters); @@ -52,14 +56,16 @@ public void testExpiredHeaders() throws Exception { public void testUnExpiredHeaders() throws Exception { //GIVEN - final MultivaluedMap parameters = Mockito.mock(MultivaluedMap.class); - Mockito.when(parameters.getFirst("X-Amz-Date")) - .thenReturn( - ZonedDateTime.now().format(StringToSignProducer.TIME_FORMATTER)); - Mockito.when(parameters.getFirst("X-Amz-Expires")).thenReturn("10000"); - Mockito.when(parameters.getFirst("X-Amz-Credential")) - .thenReturn("AKIAIOSFODNN7EXAMPLE/20130524/us-east-1/s3/aws4_request"); - Mockito.when(parameters.containsKey("X-Amz-Signature")).thenReturn(true); + Map parameters = new HashMap<>(); + parameters.put("X-Amz-Algorithm", "AWS4-HMAC-SHA256"); + parameters.put("X-Amz-Credential", + "AKIAIOSFODNN7EXAMPLE%2F20130524%2Fus-east-1%2Fs3%2Faws4_request"); + parameters.put("X-Amz-Date", + ZonedDateTime.now().format(StringToSignProducer.TIME_FORMATTER)); + parameters.put("X-Amz-Expires", "10000"); + parameters.put("X-Amz-Signature", + "aeeed9bbccd4d02ee5c0109b86d86835f995330da4c265957d157751f604d404"); + AuthorizationV4QueryParser parser = new AuthorizationV4QueryParser(parameters); @@ -71,18 +77,17 @@ public void testUnExpiredHeaders() throws Exception { //passed } - @Test() - public void testWithoutEpiration() throws Exception { + public void testWithoutExpiration() throws Exception { //GIVEN - final MultivaluedMap parameters = Mockito.mock(MultivaluedMap.class); - Mockito.when(parameters.getFirst("X-Amz-Date")) - .thenReturn( - ZonedDateTime.now().format(StringToSignProducer.TIME_FORMATTER)); - Mockito.when(parameters.getFirst("X-Amz-Credential")) - .thenReturn("AKIAIOSFODNN7EXAMPLE/20130524/us-east-1/s3/aws4_request"); - Mockito.when(parameters.containsKey("X-Amz-Signature")).thenReturn(true); + Map parameters = new HashMap<>(); + parameters.put("X-Amz-Algorithm", "AWS4-HMAC-SHA256"); + parameters.put("X-Amz-Credential", + "AKIAIOSFODNN7EXAMPLE%2F20130524%2Fus-east-1%2Fs3%2Faws4_request"); + parameters.put("X-Amz-Date", "20130524T000000Z"); + parameters.put("X-Amz-Signature", + "aeeed9bbccd4d02ee5c0109b86d86835f995330da4c265957d157751f604d404"); AuthorizationV4QueryParser parser = new AuthorizationV4QueryParser(parameters); @@ -93,4 +98,49 @@ public void testWithoutEpiration() throws Exception { //THEN //passed } + + @Test + /** + * Based on https://docs.aws.amazon + * .com/AmazonS3/latest/API/sigv4-query-string-auth.html. + */ + public void testWithAWSExample() throws Exception { + + Map queryParams = new HashMap<>(); + + queryParams.put("X-Amz-Algorithm", "AWS4-HMAC-SHA256"); + queryParams.put("X-Amz-Credential", + "AKIAIOSFODNN7EXAMPLE/20130524/us-east-1/s3/aws4_request"); + queryParams.put("X-Amz-Date", "20130524T000000Z"); + queryParams.put("X-Amz-Expires", "86400"); + queryParams.put("X-Amz-SignedHeaders", "host"); + queryParams.put("X-Amz-Signature", + "aeeed9bbccd4d02ee5c0109b86d86835f995330da4c265957d157751f604d404"); + + AuthorizationV4QueryParser parser = + new AuthorizationV4QueryParser(queryParams) { + @Override + protected void validateDateAndExpires() { + //noop + } + }; + + final SignatureInfo signatureInfo = parser.parseSignature(); + + LowerCaseKeyStringMap headers = new LowerCaseKeyStringMap(); + headers.put("host", "examplebucket.s3.amazonaws.com"); + + final String stringToSign = + StringToSignProducer.createSignatureBase(signatureInfo, "https", "GET", + "/test.txt", headers, + queryParams); + + Assert.assertEquals("AWS4-HMAC-SHA256\n" + + "20130524T000000Z\n" + + "20130524/us-east-1/s3/aws4_request\n" + + + "3bfa292879f6447bbcda7001decf97f4a54dc650c8942174ae0a9121cf58ad04", + stringToSign); + } + } \ No newline at end of file diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/signature/TestStringToSignProducer.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/signature/TestStringToSignProducer.java index f6dc7f38fc69..37fdbf0751a5 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/signature/TestStringToSignProducer.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/signature/TestStringToSignProducer.java @@ -17,8 +17,8 @@ */ package org.apache.hadoop.ozone.s3.signature; -import javax.ws.rs.core.MultivaluedHashMap; -import javax.ws.rs.core.MultivaluedMap; +import java.util.HashMap; +import java.util.Map; import org.apache.hadoop.ozone.s3.HeaderPreprocessor; import org.apache.hadoop.ozone.s3.exception.OS3Exception; @@ -55,10 +55,10 @@ public void test() throws Exception { headers.put("Authorization", authHeader); - MultivaluedMap queryParameters = new MultivaluedHashMap<>(); + Map queryParameters = new HashMap<>(); final SignatureInfo signatureInfo = - new AuthorizationV4HeaderParser(authHeader) { + new AuthorizationV4HeaderParser(authHeader, "123") { @Override public void validateDateRange(Credential credentialObj) throws OS3Exception { From 261d282e3701d0b840c2d0e34828433bc022b2e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Elek=20M=C3=A1rton?= Date: Tue, 30 Mar 2021 10:44:29 +0200 Subject: [PATCH 11/11] address review comments --- .../java/org/apache/hadoop/ozone/s3/OzoneClientProducer.java | 3 --- .../org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java | 1 - .../apache/hadoop/ozone/s3/signature/StringToSignProducer.java | 1 - 3 files changed, 5 deletions(-) diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/OzoneClientProducer.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/OzoneClientProducer.java index f22594e2b37c..4264f415a598 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/OzoneClientProducer.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/OzoneClientProducer.java @@ -158,9 +158,6 @@ OzoneClient createOzoneClient() throws IOException { } } - private void getSignatureInfo() { - } - // ONLY validate aws access id when needed. private void validateAccessId(String awsAccessId) throws Exception { if (awsAccessId == null || awsAccessId.equals("")) { diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java index e9d24bb18a5e..b8bed64cc830 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java @@ -57,7 +57,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; - /** * Bucket level rest endpoints. */ diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/StringToSignProducer.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/StringToSignProducer.java index 7d8b417056f8..3202a962d8eb 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/StringToSignProducer.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/signature/StringToSignProducer.java @@ -127,7 +127,6 @@ public static String createSignatureBase( headers, queryParams, !signatureInfo.isSignPayload()); - System.out.println(canonicalRequest); strToSign.append(hash(canonicalRequest)); if (LOG.isDebugEnabled()) { LOG.debug("canonicalRequest:[{}]", canonicalRequest);