Use the official aws-sdk instead of jet3t

.travis.yml

@@ -47,6 +47,7 @@ matrix:
     - sudo: false
       env:
         - NAME="other modules test"
+        - AWS_REGION=us-east-1 # set a aws region for unit tests
       install: echo "MAVEN_OPTS='-Xmx3000m'" > ~/.mavenrc && mvn install -q -ff -DskipTests -B
       before_script:
         - unset _JAVA_OPTIONS

api/src/main/java/io/druid/guice/JsonConfigProvider.java

@@ -130,8 +130,8 @@ public static <T> void bind(
       Key<Supplier<T>> supplierKey
   )
   {
-    binder.bind(supplierKey).toProvider((Provider) of(propertyBase, clazz)).in(LazySingleton.class);
-    binder.bind(instanceKey).toProvider(new SupplierProvider<T>(supplierKey));
+    binder.bind(supplierKey).toProvider(of(propertyBase, clazz)).in(LazySingleton.class);
+    binder.bind(instanceKey).toProvider(new SupplierProvider<>(supplierKey));
   }
 
   @SuppressWarnings("unchecked")

api/src/test/java/io/druid/data/input/impl/InputRowParserSerdeTest.java

@@ -101,9 +101,9 @@ public void testMapInputRowParserSerde() throws Exception
             null
         )
     );
-    final MapInputRowParser parser2 = jsonMapper.readValue(
+    final MapInputRowParser parser2 = (MapInputRowParser) jsonMapper.readValue(
         jsonMapper.writeValueAsBytes(parser),
-        MapInputRowParser.class
+        InputRowParser.class
     );
     final InputRow parsed = parser2.parseBatch(
         ImmutableMap.<String, Object>of(
@@ -134,9 +134,9 @@ public void testMapInputRowParserNumbersSerde() throws Exception
             null
         )
     );
-    final MapInputRowParser parser2 = jsonMapper.readValue(
+    final MapInputRowParser parser2 = (MapInputRowParser) jsonMapper.readValue(
         jsonMapper.writeValueAsBytes(parser),
-        MapInputRowParser.class
+        InputRowParser.class
     );
     final InputRow parsed = parser2.parseBatch(
         ImmutableMap.<String, Object>of(

api/src/test/java/io/druid/data/input/impl/JSONParseSpecTest.java

@@ -91,9 +91,9 @@ public void testSerde() throws IOException
         feature
     );
 
-    final JSONParseSpec serde = jsonMapper.readValue(
+    final JSONParseSpec serde = (JSONParseSpec) jsonMapper.readValue(
         jsonMapper.writeValueAsString(spec),
-        JSONParseSpec.class
+        ParseSpec.class
     );
     Assert.assertEquals("timestamp", serde.getTimestampSpec().getTimestampColumn());
     Assert.assertEquals("iso", serde.getTimestampSpec().getTimestampFormat());

api/src/test/java/io/druid/data/input/impl/JavaScriptParseSpecTest.java

@@ -58,9 +58,9 @@ public void testSerde() throws IOException
         "abc",
         JavaScriptConfig.getEnabledInstance()
     );
-    final JavaScriptParseSpec serde = jsonMapper.readValue(
+    final JavaScriptParseSpec serde = (JavaScriptParseSpec) jsonMapper.readValue(
         jsonMapper.writeValueAsString(spec),
-        JavaScriptParseSpec.class
+        ParseSpec.class
     );
     Assert.assertEquals("abc", serde.getTimestampSpec().getTimestampColumn());
     Assert.assertEquals("iso", serde.getTimestampSpec().getTimestampFormat());

api/src/test/java/io/druid/data/input/impl/RegexParseSpecTest.java

@@ -43,9 +43,9 @@ public void testSerde() throws IOException
         Collections.singletonList("abc"),
         "abc"
     );
-    final RegexParseSpec serde = jsonMapper.readValue(
+    final RegexParseSpec serde = (RegexParseSpec) jsonMapper.readValue(
         jsonMapper.writeValueAsString(spec),
-        RegexParseSpec.class
+        ParseSpec.class
     );
     Assert.assertEquals("abc", serde.getTimestampSpec().getTimestampColumn());
     Assert.assertEquals("iso", serde.getTimestampSpec().getTimestampFormat());

aws-common/pom.xml

@@ -37,7 +37,7 @@
         </dependency>
         <dependency>
             <groupId>com.amazonaws</groupId>
-            <artifactId>aws-java-sdk-ec2</artifactId>
+            <artifactId>aws-java-sdk-bundle</artifactId>
         </dependency>
 
         <!-- Tests -->

aws-common/src/main/java/io/druid/common/aws/AWSEndpointConfig.java

@@ -0,0 +1,52 @@
+/*
+ * Licensed to Metamarkets Group Inc. (Metamarkets) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. Metamarkets licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package io.druid.common.aws;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+public class AWSEndpointConfig
+{
+  @JsonProperty
+  private String url;
+
+  @JsonProperty
+  private String serviceName;
+
+  @JsonProperty
+  private String signingRegion;
+
+  @JsonProperty
+  public String getUrl()
+  {
+    return url;
+  }
+
+  @JsonProperty
+  public String getServiceName()
+  {
+    return serviceName;
+  }
+
+  @JsonProperty
+  public String getSigningRegion()
+  {
+    return signingRegion;
+  }
+}

aws-common/src/main/java/io/druid/common/aws/AWSProxyConfig.java

@@ -0,0 +1,61 @@
+/*
+ * Licensed to Metamarkets Group Inc. (Metamarkets) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. Metamarkets licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package io.druid.common.aws;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+public class AWSProxyConfig
+{
+  @JsonProperty
+  private String host;
+
+  @JsonProperty
+  private int port = -1; // AWS's default proxy port is -1
+
+  @JsonProperty
+  private String username;
+
+  @JsonProperty
+  private String password;
+
+  @JsonProperty
+  public String getHost()
+  {
+    return host;
+  }
+
+  @JsonProperty
+  public int getPort()
+  {
+    return port;
+  }
+
+  @JsonProperty
+  public String getUsername()
+  {
+    return username;
+  }
+
+  @JsonProperty
+  public String getPassword()
+  {
+    return password;
+  }
+}

common/pom.xml

@@ -157,21 +157,7 @@
             <groupId>com.lmax</groupId>
             <artifactId>disruptor</artifactId>
         </dependency>
-        <!--
-            Druid doesn't ACTUALLY depend on jets3t in its core, but quite a few of the extensions do. This leads to a nasty ClassLoader problem
-            There's a bug in
-            https://github.com/apache/httpclient/blob/4.5.2/httpclient/src/main/java-deprecated/org/apache/http/impl/client/AbstractHttpClient.java#L332
-            Where httpclient does not care what your context classloader is when looking for the connection manager factories. See https://issues.apache.org/jira/browse/HTTPCLIENT-1727
-            A few extensions depend on jets3t, so we include it explicitly here to make sure it can load up its
-            org.jets3t.service.utils.RestUtils$ConnManagerFactory
-            properly
-            Future releases which include HTTPCLIENT-1727 should probably set the context loader whenever jets3t calls are used
-        -->
-        <dependency>
-            <groupId>net.java.dev.jets3t</groupId>
-            <artifactId>jets3t</artifactId>
-            <version>0.9.4</version>
-        </dependency>
+
         <dependency>
             <groupId>org.antlr</groupId>
             <artifactId>antlr4-runtime</artifactId>

docs/content/development/extensions-core/s3.md

@@ -12,12 +12,18 @@ S3-compatible deep storage is basically either S3 or something like Google Stora
 
 ### Configuration
 
-|Property|Possible Values|Description|Default|
-|--------|---------------|-----------|-------|
-|`druid.s3.accessKey`||S3 access key.|Must be set.|
-|`druid.s3.secretKey`||S3 secret key.|Must be set.|
-|`druid.storage.bucket`||Bucket to store in.|Must be set.|
-|`druid.storage.baseKey`||Base key prefix to use, i.e. what directory.|Must be set.|
+|Property|Description|Default|
+|--------|-----------|-------|
+|`druid.s3.accessKey`|S3 access key.|Must be set.|
+|`druid.s3.secretKey`|S3 secret key.|Must be set.|
+|`druid.storage.bucket`|Bucket to store in.|Must be set.|
+|`druid.storage.baseKey`|Base key prefix to use, i.e. what directory.|Must be set.|
+|`druid.s3.endpoint.url`|Service endpoint either with or without the protocol.|None|
+|`druid.s3.endpoint.signingRegion`|Region to use for SigV4 signing of requests (e.g. us-west-1).|None|
+|`druid.s3.proxy.host`|Proxy host to connect through.|None|
+|`druid.s3.proxy.port`|Port on the proxy host to connect through.|None|
+|`druid.s3.proxy.username`|User name to use when connecting through a proxy.|None|
+|`druid.s3.proxy.password`|Password to use when connecting through a proxy.|None|
 
 ## StaticS3Firehose
 

extensions-core/druid-kerberos/src/main/java/io/druid/security/kerberos/KerberosAuthenticator.java

@@ -334,13 +334,14 @@ public Principal getUserPrincipal()
               };
               if (newToken && !token.isExpired() && token != AuthenticationToken.ANONYMOUS) {
                 String signedToken = mySigner.sign(token.toString());
-                tokenToAuthCookie(httpResponse,
-                                 signedToken,
-                                 getCookieDomain(),
-                                 getCookiePath(),
-                                 token.getExpires(),
-                                 !token.isExpired() && token.getExpires() > 0,
-                                 isHttps
+                tokenToAuthCookie(
+                    httpResponse,
+                    signedToken,
+                    getCookieDomain(),
+                    getCookiePath(),
+                    token.getExpires(),
+                    !token.isExpired() && token.getExpires() > 0,
+                    isHttps
                 );
               }
               doFilter(filterChain, httpRequest, httpResponse);
@@ -361,8 +362,14 @@ public Principal getUserPrincipal()
         }
         if (unauthorizedResponse) {
           if (!httpResponse.isCommitted()) {
-            tokenToAuthCookie(httpResponse, "", getCookieDomain(),
-                             getCookiePath(), 0, false, isHttps
+            tokenToAuthCookie(
+                httpResponse,
+                "",
+                getCookieDomain(),
+                getCookiePath(),
+                0,
+                false,
+                isHttps
             );
             // If response code is 401. Then WWW-Authenticate Header should be
             // present.. reset to 403 if not found..

extensions-core/hdfs-storage/pom.xml

@@ -145,16 +145,7 @@
           <groupId>org.apache.hadoop</groupId>
           <artifactId>hadoop-aws</artifactId>
           <version>${hadoop.compile.version}</version>
-          <exclusions>
-            <exclusion>
-              <groupId>com.amazonaws</groupId>
-              <artifactId>aws-java-sdk</artifactId>
-            </exclusion>
-          </exclusions>
-        </dependency>
-        <dependency>
-          <groupId>com.amazonaws</groupId>
-          <artifactId>aws-java-sdk-s3</artifactId>
+          <scope>provided</scope>
         </dependency>
         <dependency>
             <groupId>commons-io</groupId>

extensions-core/hdfs-storage/src/main/java/io/druid/storage/hdfs/HdfsDataSegmentFinder.java

@@ -22,11 +22,11 @@
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.google.common.collect.Sets;
 import com.google.inject.Inject;
+import io.druid.java.util.common.StringUtils;
 import io.druid.java.util.common.logger.Logger;
 import io.druid.segment.loading.DataSegmentFinder;
 import io.druid.segment.loading.SegmentLoadingException;
 import io.druid.timeline.DataSegment;
-import io.druid.java.util.common.StringUtils;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.LocatedFileStatus;