From d2fdc39c4ac7c2d76139a6e42f47e9bd95e06ae4 Mon Sep 17 00:00:00 2001 From: Jihoon Son Date: Mon, 12 Feb 2018 11:28:15 -0800 Subject: [PATCH 01/21] Use the official aws-sdk instead of jet3t --- aws-common/pom.xml | 2 +- common/pom.xml | 16 +- .../kerberos/KerberosAuthenticator.java | 4 +- .../storage/hdfs/HdfsDataSegmentFinder.java | 6 +- .../loading/HdfsDataSegmentFinderTest.java | 13 +- extensions-core/s3-extensions/pom.xml | 16 +- .../firehose/s3/StaticS3FirehoseFactory.java | 112 ++++------ .../s3/AWSSessionCredentialsAdapter.java | 70 ------ .../storage/s3/S3DataSegmentArchiver.java | 4 +- .../druid/storage/s3/S3DataSegmentFinder.java | 48 ++-- .../druid/storage/s3/S3DataSegmentKiller.java | 14 +- .../druid/storage/s3/S3DataSegmentMover.java | 62 ++--- .../druid/storage/s3/S3DataSegmentPuller.java | 111 ++++++--- .../druid/storage/s3/S3DataSegmentPusher.java | 54 +++-- .../storage/s3/S3StorageDruidModule.java | 15 +- .../java/io/druid/storage/s3/S3TaskLogs.java | 50 ++--- .../s3/S3TimestampVersionedDataFinder.java | 32 ++- .../java/io/druid/storage/s3/S3Utils.java | 211 +++++++++++------- .../s3/StaticS3FirehoseFactoryTest.java | 44 +++- .../storage/s3/S3DataSegmentArchiverTest.java | 4 +- .../storage/s3/S3DataSegmentFinderTest.java | 185 +++++++-------- .../storage/s3/S3DataSegmentMoverTest.java | 142 ++++++++---- .../storage/s3/S3DataSegmentPullerTest.java | 101 +++++---- .../storage/s3/S3DataSegmentPusherTest.java | 40 +++- .../S3TimestampVersionedDataFinderTest.java | 89 ++++---- .../s3/TestAWSCredentialsProvider.java | 4 +- indexing-hadoop/pom.xml | 6 +- .../io/druid/indexer/IndexGeneratorJob.java | 3 +- .../indexing/common/config/TaskConfig.java | 2 +- pom.xml | 69 +++--- .../firehose/HttpFirehoseFactory.java | 2 +- .../server/AsyncQueryForwardingServlet.java | 11 +- .../jetty/JettyServerModule.java | 2 +- 33 files changed, 834 insertions(+), 710 deletions(-) delete mode 100644 extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/AWSSessionCredentialsAdapter.java diff --git a/aws-common/pom.xml b/aws-common/pom.xml index 9ce40e3951ea..c6e69b099f66 100644 --- a/aws-common/pom.xml +++ b/aws-common/pom.xml @@ -37,7 +37,7 @@ com.amazonaws - aws-java-sdk-ec2 + aws-java-sdk-bundle diff --git a/common/pom.xml b/common/pom.xml index 234f21af5bb2..0f07a4105ab8 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -157,21 +157,7 @@ com.lmax disruptor - - - net.java.dev.jets3t - jets3t - 0.9.4 - + org.antlr antlr4-runtime diff --git a/extensions-core/druid-kerberos/src/main/java/io/druid/security/kerberos/KerberosAuthenticator.java b/extensions-core/druid-kerberos/src/main/java/io/druid/security/kerberos/KerberosAuthenticator.java index 785a5a415cb5..d423fd96d988 100644 --- a/extensions-core/druid-kerberos/src/main/java/io/druid/security/kerberos/KerberosAuthenticator.java +++ b/extensions-core/druid-kerberos/src/main/java/io/druid/security/kerberos/KerberosAuthenticator.java @@ -331,7 +331,7 @@ public Principal getUserPrincipal() if (newToken && !token.isExpired() && token != AuthenticationToken.ANONYMOUS) { String signedToken = mySigner.sign(token.toString()); createAuthCookie(httpResponse, signedToken, getCookieDomain(), - getCookiePath(), token.getExpires(), isHttps + getCookiePath(), token.getExpires(), isCookiePersistent(), isHttps ); } doFilter(filterChain, httpRequest, httpResponse); @@ -353,7 +353,7 @@ public Principal getUserPrincipal() if (unauthorizedResponse) { if (!httpResponse.isCommitted()) { createAuthCookie(httpResponse, "", getCookieDomain(), - getCookiePath(), 0, isHttps + getCookiePath(), 0, isCookiePersistent(), isHttps ); // If response code is 401. Then WWW-Authenticate Header should be // present.. reset to 403 if not found.. diff --git a/extensions-core/hdfs-storage/src/main/java/io/druid/storage/hdfs/HdfsDataSegmentFinder.java b/extensions-core/hdfs-storage/src/main/java/io/druid/storage/hdfs/HdfsDataSegmentFinder.java index 6fba009cf865..e5c2d3d52461 100644 --- a/extensions-core/hdfs-storage/src/main/java/io/druid/storage/hdfs/HdfsDataSegmentFinder.java +++ b/extensions-core/hdfs-storage/src/main/java/io/druid/storage/hdfs/HdfsDataSegmentFinder.java @@ -33,6 +33,8 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.RemoteIterator; +import java.io.DataInput; +import java.io.DataOutput; import java.io.IOException; import java.util.Map; import java.util.Set; @@ -90,7 +92,7 @@ public Set findSegments(String workingDirPathStr, boolean updateDes indexZip = new Path(path.getParent(), "index.zip"); } if (fs.exists(indexZip)) { - final DataSegment dataSegment = mapper.readValue(fs.open(path), DataSegment.class); + final DataSegment dataSegment = mapper.readValue((DataInput) fs.open(path), DataSegment.class); log.info("Found segment [%s] located at [%s]", dataSegment.getIdentifier(), indexZip); final Map loadSpec = dataSegment.getLoadSpec(); @@ -102,7 +104,7 @@ public Set findSegments(String workingDirPathStr, boolean updateDes loadSpec.put("path", pathWithoutScheme); if (updateDescriptor) { log.info("Updating loadSpec in descriptor.json at [%s] with new path [%s]", path, pathWithoutScheme); - mapper.writeValue(fs.create(path, true), dataSegment); + mapper.writeValue((DataOutput) fs.create(path, true), dataSegment); } } segments.add(dataSegment); diff --git a/extensions-core/hdfs-storage/src/test/java/io/druid/segment/loading/HdfsDataSegmentFinderTest.java b/extensions-core/hdfs-storage/src/test/java/io/druid/segment/loading/HdfsDataSegmentFinderTest.java index 626124622527..3581f33e5c25 100644 --- a/extensions-core/hdfs-storage/src/test/java/io/druid/segment/loading/HdfsDataSegmentFinderTest.java +++ b/extensions-core/hdfs-storage/src/test/java/io/druid/segment/loading/HdfsDataSegmentFinderTest.java @@ -43,6 +43,7 @@ import org.junit.BeforeClass; import org.junit.Test; +import java.io.DataOutput; import java.io.File; import java.io.IOException; import java.net.URI; @@ -176,12 +177,12 @@ public void setUp() throws IOException indexZip5 = new Path(descriptor5.getParent(), "1_" + INDEX_ZIP); - mapper.writeValue(fs.create(descriptor1), SEGMENT_1); - mapper.writeValue(fs.create(descriptor2), SEGMENT_2); - mapper.writeValue(fs.create(descriptor3), SEGMENT_3); - mapper.writeValue(fs.create(descriptor4_0), SEGMENT_4_0); - mapper.writeValue(fs.create(descriptor4_1), SEGMENT_4_1); - mapper.writeValue(fs.create(descriptor5), SEGMENT_5); + mapper.writeValue((DataOutput) fs.create(descriptor1), SEGMENT_1); + mapper.writeValue((DataOutput) fs.create(descriptor2), SEGMENT_2); + mapper.writeValue((DataOutput) fs.create(descriptor3), SEGMENT_3); + mapper.writeValue((DataOutput) fs.create(descriptor4_0), SEGMENT_4_0); + mapper.writeValue((DataOutput) fs.create(descriptor4_1), SEGMENT_4_1); + mapper.writeValue((DataOutput) fs.create(descriptor5), SEGMENT_5); create(indexZip1); create(indexZip2); diff --git a/extensions-core/s3-extensions/pom.xml b/extensions-core/s3-extensions/pom.xml index 3d389eab4d3f..5513656f4151 100644 --- a/extensions-core/s3-extensions/pom.xml +++ b/extensions-core/s3-extensions/pom.xml @@ -46,16 +46,16 @@ provided - io.druid - java-util - ${project.parent.version} - provided - - - net.java.dev.jets3t - jets3t + com.amazonaws + aws-java-sdk-s3 provided + + io.druid + java-util + ${project.parent.version} + provided + commons-io commons-io diff --git a/extensions-core/s3-extensions/src/main/java/io/druid/firehose/s3/StaticS3FirehoseFactory.java b/extensions-core/s3-extensions/src/main/java/io/druid/firehose/s3/StaticS3FirehoseFactory.java index fa649dc495c7..de93fc942197 100644 --- a/extensions-core/s3-extensions/src/main/java/io/druid/firehose/s3/StaticS3FirehoseFactory.java +++ b/extensions-core/s3-extensions/src/main/java/io/druid/firehose/s3/StaticS3FirehoseFactory.java @@ -19,28 +19,30 @@ package io.druid.firehose.s3; +import com.amazonaws.services.s3.AmazonS3Client; +import com.amazonaws.services.s3.model.AmazonS3Exception; +import com.amazonaws.services.s3.model.GetObjectRequest; +import com.amazonaws.services.s3.model.ObjectMetadata; +import com.amazonaws.services.s3.model.S3ObjectSummary; import com.fasterxml.jackson.annotation.JacksonInject; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Preconditions; import com.google.common.base.Predicate; +import com.google.common.collect.Lists; import io.druid.data.input.impl.prefetch.PrefetchableTextFilesFirehoseFactory; import io.druid.java.util.common.CompressionUtils; import io.druid.java.util.common.IAE; +import io.druid.java.util.common.IOE; import io.druid.java.util.common.logger.Logger; import io.druid.storage.s3.S3Utils; -import org.jets3t.service.S3ServiceException; -import org.jets3t.service.ServiceException; -import org.jets3t.service.StorageObjectsChunk; -import org.jets3t.service.impl.rest.httpclient.RestS3Service; -import org.jets3t.service.model.S3Object; import java.io.IOException; import java.io.InputStream; import java.net.URI; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collection; +import java.util.Iterator; import java.util.List; import java.util.Objects; import java.util.stream.Collectors; @@ -48,18 +50,18 @@ /** * Builds firehoses that read from a predefined list of S3 objects and then dry up. */ -public class StaticS3FirehoseFactory extends PrefetchableTextFilesFirehoseFactory +public class StaticS3FirehoseFactory extends PrefetchableTextFilesFirehoseFactory { private static final Logger log = new Logger(StaticS3FirehoseFactory.class); - private static final long MAX_LISTING_LENGTH = 1024; + private static final int MAX_LISTING_LENGTH = 1024; - private final RestS3Service s3Client; + private final AmazonS3Client s3Client; private final List uris; private final List prefixes; @JsonCreator public StaticS3FirehoseFactory( - @JacksonInject("s3Client") RestS3Service s3Client, + @JacksonInject("s3Client") AmazonS3Client s3Client, @JsonProperty("uris") List uris, @JsonProperty("prefixes") List prefixes, @JsonProperty("maxCacheCapacityBytes") Long maxCacheCapacityBytes, @@ -70,7 +72,7 @@ public StaticS3FirehoseFactory( ) { super(maxCacheCapacityBytes, maxFetchCapacityBytes, prefetchTriggerBytes, fetchTimeout, maxFetchRetry); - this.s3Client = Preconditions.checkNotNull(s3Client, "null s3Client"); + this.s3Client = Preconditions.checkNotNull(s3Client, "s3Client"); this.uris = uris == null ? new ArrayList<>() : uris; this.prefixes = prefixes == null ? new ArrayList<>() : prefixes; @@ -104,7 +106,7 @@ public List getPrefixes() } @Override - protected Collection initObjects() throws IOException + protected Collection initObjects() throws IOException { // Here, the returned s3 objects contain minimal information without data. // Getting data is deferred until openObjectStream() is called for each object. @@ -113,53 +115,49 @@ protected Collection initObjects() throws IOException .map( uri -> { final String s3Bucket = uri.getAuthority(); - final S3Object s3Object = new S3Object(extractS3Key(uri)); - s3Object.setBucketName(s3Bucket); - return s3Object; + final String key = S3Utils.extractS3Key(uri); + return S3Utils.getSingleObjectSummary(s3Client, s3Bucket, key); } ) .collect(Collectors.toList()); } else { - final List objects = new ArrayList<>(); + final List objects = new ArrayList<>(); for (URI uri : prefixes) { final String bucket = uri.getAuthority(); - final String prefix = extractS3Key(uri); + final String prefix = S3Utils.extractS3Key(uri); + try { - String lastKey = null; - StorageObjectsChunk objectsChunk; - do { - objectsChunk = s3Client.listObjectsChunked( - bucket, - prefix, - null, - MAX_LISTING_LENGTH, - lastKey - ); - Arrays.stream(objectsChunk.getObjects()) - .filter(storageObject -> !storageObject.isDirectoryPlaceholder()) - .forEach(storageObject -> objects.add((S3Object) storageObject)); - lastKey = objectsChunk.getPriorLastKey(); - } while (!objectsChunk.isListingComplete()); + final Iterator objectSummaryIterator = S3Utils.objectSummaryIterator( + s3Client, + bucket, + prefix, + MAX_LISTING_LENGTH + ); + objects.addAll(Lists.newArrayList(objectSummaryIterator)); } - catch (ServiceException outerException) { + catch (AmazonS3Exception outerException) { log.error(outerException, "Exception while listing on %s", uri); - if (outerException.getResponseCode() == 403) { + if (outerException.getStatusCode() == 403) { // The "Access Denied" means users might not have a proper permission for listing on the given uri. // Usually this is not a problem, but the uris might be the full paths to input objects instead of prefixes. // In this case, users should be able to get objects if they have a proper permission for GetObject. log.warn("Access denied for %s. Try to get the object from the uri without listing", uri); try { - final S3Object s3Object = s3Client.getObject(bucket, prefix); - if (!s3Object.isDirectoryPlaceholder()) { - objects.add(s3Object); + final ObjectMetadata objectMetadata = s3Client.getObjectMetadata(bucket, prefix); + + if (!S3Utils.isDirectoryPlaceholder(prefix, objectMetadata)) { + objects.add(S3Utils.getSingleObjectSummary(s3Client, bucket, prefix)); } else { - throw new IOException(uri + " is a directory placeholder, " - + "but failed to get the object list under the directory due to permission"); + throw new IOE( + "[%s] is a directory placeholder, " + + "but failed to get the object list under the directory due to permission", + uri + ); } } - catch (S3ServiceException innerException) { + catch (AmazonS3Exception innerException) { throw new IOException(innerException); } } else { @@ -171,49 +169,33 @@ protected Collection initObjects() throws IOException } } - private static String extractS3Key(URI uri) - { - return uri.getPath().startsWith("/") - ? uri.getPath().substring(1) - : uri.getPath(); - } - @Override - protected InputStream openObjectStream(S3Object object) throws IOException + protected InputStream openObjectStream(S3ObjectSummary object) throws IOException { try { // Get data of the given object and open an input stream - return s3Client.getObject(object.getBucketName(), object.getKey()).getDataInputStream(); + return s3Client.getObject(object.getBucketName(), object.getKey()).getObjectContent(); } - catch (ServiceException e) { + catch (AmazonS3Exception e) { throw new IOException(e); } } @Override - protected InputStream openObjectStream(S3Object object, long start) throws IOException + protected InputStream openObjectStream(S3ObjectSummary object, long start) throws IOException { + final GetObjectRequest request = new GetObjectRequest(object.getBucketName(), object.getKey()); + request.setRange(start); try { - final S3Object result = s3Client.getObject( - object.getBucketName(), - object.getKey(), - null, - null, - null, - null, - start, - null - ); - - return result.getDataInputStream(); + return s3Client.getObject(request).getObjectContent(); } - catch (ServiceException e) { + catch (AmazonS3Exception e) { throw new IOException(e); } } @Override - protected InputStream wrapObjectStream(S3Object object, InputStream stream) throws IOException + protected InputStream wrapObjectStream(S3ObjectSummary object, InputStream stream) throws IOException { return object.getKey().endsWith(".gz") ? CompressionUtils.gzipInputStream(stream) : stream; } diff --git a/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/AWSSessionCredentialsAdapter.java b/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/AWSSessionCredentialsAdapter.java deleted file mode 100644 index 7a64a81e7c96..000000000000 --- a/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/AWSSessionCredentialsAdapter.java +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Licensed to Metamarkets Group Inc. (Metamarkets) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. Metamarkets licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package io.druid.storage.s3; - -import com.amazonaws.auth.AWSCredentialsProvider; -import org.jets3t.service.security.AWSSessionCredentials; - -public class AWSSessionCredentialsAdapter extends AWSSessionCredentials -{ - private final AWSCredentialsProvider provider; - - public AWSSessionCredentialsAdapter(AWSCredentialsProvider provider) - { - super(null, null, null); - if (provider.getCredentials() instanceof com.amazonaws.auth.AWSSessionCredentials) { - this.provider = provider; - } else { - throw new IllegalArgumentException("provider does not contain session credentials"); - } - } - - @Override - protected String getTypeName() - { - return "AWSSessionCredentialsAdapter"; - } - - @Override - public String getVersionPrefix() - { - return "AWSSessionCredentialsAdapter, version: "; - } - - @Override - public String getAccessKey() - { - return provider.getCredentials().getAWSAccessKeyId(); - } - - @Override - public String getSecretKey() - { - return provider.getCredentials().getAWSSecretKey(); - } - - @Override - public String getSessionToken() - { - com.amazonaws.auth.AWSSessionCredentials sessionCredentials = - (com.amazonaws.auth.AWSSessionCredentials) provider.getCredentials(); - return sessionCredentials.getSessionToken(); - } -} diff --git a/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3DataSegmentArchiver.java b/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3DataSegmentArchiver.java index d7bc1b2d4912..af7c6e4afff5 100644 --- a/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3DataSegmentArchiver.java +++ b/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3DataSegmentArchiver.java @@ -19,6 +19,7 @@ package io.druid.storage.s3; +import com.amazonaws.services.s3.AmazonS3Client; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Objects; import com.google.common.collect.ImmutableMap; @@ -28,7 +29,6 @@ import io.druid.segment.loading.LoadSpec; import io.druid.segment.loading.SegmentLoadingException; import io.druid.timeline.DataSegment; -import org.jets3t.service.impl.rest.httpclient.RestS3Service; public class S3DataSegmentArchiver extends S3DataSegmentMover implements DataSegmentArchiver @@ -40,7 +40,7 @@ public class S3DataSegmentArchiver extends S3DataSegmentMover implements DataSeg @Inject public S3DataSegmentArchiver( @Json ObjectMapper mapper, - RestS3Service s3Client, + AmazonS3Client s3Client, S3DataSegmentArchiverConfig archiveConfig, S3DataSegmentPusherConfig restoreConfig ) diff --git a/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3DataSegmentFinder.java b/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3DataSegmentFinder.java index d6d773640e8e..42d87c73d14b 100644 --- a/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3DataSegmentFinder.java +++ b/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3DataSegmentFinder.java @@ -19,22 +19,24 @@ package io.druid.storage.s3; +import com.amazonaws.AmazonServiceException; +import com.amazonaws.services.s3.AmazonS3Client; +import com.amazonaws.services.s3.model.ObjectMetadata; +import com.amazonaws.services.s3.model.S3Object; +import com.amazonaws.services.s3.model.S3ObjectInputStream; +import com.amazonaws.services.s3.model.S3ObjectSummary; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Throwables; import com.google.common.collect.Sets; import com.google.inject.Inject; - +import io.druid.java.util.common.StringUtils; import io.druid.java.util.common.logger.Logger; import io.druid.segment.loading.DataSegmentFinder; import io.druid.segment.loading.SegmentLoadingException; import io.druid.timeline.DataSegment; -import org.jets3t.service.ServiceException; -import org.jets3t.service.impl.rest.httpclient.RestS3Service; -import org.jets3t.service.model.S3Object; -import org.jets3t.service.model.StorageObject; +import java.io.ByteArrayInputStream; import java.io.IOException; -import java.io.InputStream; import java.util.Iterator; import java.util.Map; import java.util.Set; @@ -43,13 +45,13 @@ public class S3DataSegmentFinder implements DataSegmentFinder { private static final Logger log = new Logger(S3DataSegmentFinder.class); - private final RestS3Service s3Client; + private final AmazonS3Client s3Client; private final ObjectMapper jsonMapper; private final S3DataSegmentPusherConfig config; @Inject public S3DataSegmentFinder( - RestS3Service s3Client, + AmazonS3Client s3Client, S3DataSegmentPusherConfig config, ObjectMapper jsonMapper ) @@ -65,24 +67,24 @@ public Set findSegments(String workingDirPath, boolean updateDescri final Set segments = Sets.newHashSet(); try { - Iterator objectsIterator = S3Utils.storageObjectsIterator( + final Iterator objectSummaryIterator = S3Utils.objectSummaryIterator( s3Client, config.getBucket(), workingDirPath.length() == 0 ? config.getBaseKey() : workingDirPath, - config.getMaxListingLength()); + config.getMaxListingLength() + ); - while (objectsIterator.hasNext()) { - StorageObject storageObject = objectsIterator.next(); - storageObject.closeDataInputStream(); + while (objectSummaryIterator.hasNext()) { + final S3ObjectSummary objectSummary = objectSummaryIterator.next(); - if (S3Utils.toFilename(storageObject.getKey()).equals("descriptor.json")) { - final String descriptorJson = storageObject.getKey(); + if (S3Utils.toFilename(objectSummary.getKey()).equals("descriptor.json")) { + final String descriptorJson = objectSummary.getKey(); String indexZip = S3Utils.indexZipForSegmentPath(descriptorJson); - if (S3Utils.isObjectInBucket(s3Client, config.getBucket(), indexZip)) { - S3Object indexObject = s3Client.getObject(config.getBucket(), descriptorJson); - - try (InputStream is = indexObject.getDataInputStream()) { + if (S3Utils.isObjectInBucketIgnoringPermission(s3Client, config.getBucket(), indexZip)) { + try (S3Object indexObject = s3Client.getObject(config.getBucket(), descriptorJson); + S3ObjectInputStream is = indexObject.getObjectContent()) { + final ObjectMetadata objectMetadata = indexObject.getObjectMetadata(); final DataSegment dataSegment = jsonMapper.readValue(is, DataSegment.class); log.info("Found segment [%s] located at [%s]", dataSegment.getIdentifier(), indexZip); @@ -99,8 +101,10 @@ public Set findSegments(String workingDirPath, boolean updateDescri descriptorJson, indexObject ); - S3Object newDescJsonObject = new S3Object(descriptorJson, jsonMapper.writeValueAsString(dataSegment)); - s3Client.putObject(config.getBucket(), newDescJsonObject); + final ByteArrayInputStream bais = new ByteArrayInputStream( + StringUtils.toUtf8(jsonMapper.writeValueAsString(dataSegment)) + ); + s3Client.putObject(config.getBucket(), descriptorJson, bais, objectMetadata); } } segments.add(dataSegment); @@ -114,7 +118,7 @@ public Set findSegments(String workingDirPath, boolean updateDescri } } } - catch (ServiceException e) { + catch (AmazonServiceException e) { throw new SegmentLoadingException(e, "Problem interacting with S3"); } catch (IOException e) { diff --git a/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3DataSegmentKiller.java b/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3DataSegmentKiller.java index ab7ec4938d52..4947d60fdc57 100644 --- a/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3DataSegmentKiller.java +++ b/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3DataSegmentKiller.java @@ -19,14 +19,14 @@ package io.druid.storage.s3; +import com.amazonaws.AmazonServiceException; +import com.amazonaws.services.s3.AmazonS3Client; import com.google.inject.Inject; import io.druid.java.util.common.MapUtils; import io.druid.java.util.common.logger.Logger; import io.druid.segment.loading.DataSegmentKiller; import io.druid.segment.loading.SegmentLoadingException; import io.druid.timeline.DataSegment; -import org.jets3t.service.ServiceException; -import org.jets3t.service.impl.rest.httpclient.RestS3Service; import java.io.IOException; import java.util.Map; @@ -37,11 +37,11 @@ public class S3DataSegmentKiller implements DataSegmentKiller { private static final Logger log = new Logger(S3DataSegmentKiller.class); - private final RestS3Service s3Client; + private final AmazonS3Client s3Client; @Inject public S3DataSegmentKiller( - RestS3Service s3Client + AmazonS3Client s3Client ) { this.s3Client = s3Client; @@ -56,16 +56,16 @@ public void kill(DataSegment segment) throws SegmentLoadingException String s3Path = MapUtils.getString(loadSpec, "key"); String s3DescriptorPath = S3Utils.descriptorPathForSegmentPath(s3Path); - if (s3Client.isObjectInBucket(s3Bucket, s3Path)) { + if (s3Client.doesObjectExist(s3Bucket, s3Path)) { log.info("Removing index file[s3://%s/%s] from s3!", s3Bucket, s3Path); s3Client.deleteObject(s3Bucket, s3Path); } - if (s3Client.isObjectInBucket(s3Bucket, s3DescriptorPath)) { + if (s3Client.doesObjectExist(s3Bucket, s3DescriptorPath)) { log.info("Removing descriptor file[s3://%s/%s] from s3!", s3Bucket, s3DescriptorPath); s3Client.deleteObject(s3Bucket, s3DescriptorPath); } } - catch (ServiceException e) { + catch (AmazonServiceException e) { throw new SegmentLoadingException(e, "Couldn't kill segment[%s]: [%s]", segment.getIdentifier(), e); } } diff --git a/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3DataSegmentMover.java b/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3DataSegmentMover.java index 1c22418e2b4c..62df758c55b0 100644 --- a/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3DataSegmentMover.java +++ b/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3DataSegmentMover.java @@ -19,6 +19,13 @@ package io.druid.storage.s3; +import com.amazonaws.AmazonServiceException; +import com.amazonaws.services.s3.AmazonS3Client; +import com.amazonaws.services.s3.model.CopyObjectRequest; +import com.amazonaws.services.s3.model.ListObjectsV2Request; +import com.amazonaws.services.s3.model.ListObjectsV2Result; +import com.amazonaws.services.s3.model.S3ObjectSummary; +import com.amazonaws.services.s3.model.StorageClass; import com.google.common.base.Predicate; import com.google.common.base.Throwables; import com.google.common.collect.ImmutableMap; @@ -34,10 +41,6 @@ import io.druid.segment.loading.DataSegmentPusher; import io.druid.segment.loading.SegmentLoadingException; import io.druid.timeline.DataSegment; -import org.jets3t.service.ServiceException; -import org.jets3t.service.acl.gs.GSAccessControlList; -import org.jets3t.service.impl.rest.httpclient.RestS3Service; -import org.jets3t.service.model.S3Object; import java.io.IOException; import java.util.Map; @@ -46,12 +49,12 @@ public class S3DataSegmentMover implements DataSegmentMover { private static final Logger log = new Logger(S3DataSegmentMover.class); - private final RestS3Service s3Client; + private final AmazonS3Client s3Client; private final S3DataSegmentPusherConfig config; @Inject public S3DataSegmentMover( - RestS3Service s3Client, + AmazonS3Client s3Client, S3DataSegmentPusherConfig config ) { @@ -103,7 +106,7 @@ public boolean apply(String input) .build() ); } - catch (ServiceException e) { + catch (AmazonServiceException e) { throw new SegmentLoadingException(e, "Unable to move segment[%s]: [%s]", segment.getIdentifier(), e); } } @@ -113,7 +116,7 @@ private void safeMove( final String s3Path, final String targetS3Bucket, final String targetS3Path - ) throws ServiceException, SegmentLoadingException + ) throws SegmentLoadingException { try { S3Utils.retryS3Operation( @@ -129,7 +132,7 @@ private void safeMove( selfCheckingMove(s3Bucket, targetS3Bucket, s3Path, targetS3Path, copyMsg); return null; } - catch (ServiceException | IOException | SegmentLoadingException e) { + catch (AmazonServiceException | IOException | SegmentLoadingException e) { log.info(e, "Error while trying to move " + copyMsg); throw e; } @@ -137,7 +140,7 @@ private void safeMove( ); } catch (Exception e) { - Throwables.propagateIfInstanceOf(e, ServiceException.class); + Throwables.propagateIfInstanceOf(e, AmazonServiceException.class); Throwables.propagateIfInstanceOf(e, SegmentLoadingException.class); throw Throwables.propagate(e); } @@ -155,40 +158,39 @@ private void selfCheckingMove( String s3Path, String targetS3Path, String copyMsg - ) throws ServiceException, IOException, SegmentLoadingException + ) throws IOException, SegmentLoadingException { if (s3Bucket.equals(targetS3Bucket) && s3Path.equals(targetS3Path)) { log.info("No need to move file[s3://%s/%s] onto itself", s3Bucket, s3Path); return; } - if (s3Client.isObjectInBucket(s3Bucket, s3Path)) { - final S3Object[] list = s3Client.listObjects(s3Bucket, s3Path, ""); - if (list.length == 0) { + if (s3Client.doesObjectExist(s3Bucket, s3Path)) { + final ListObjectsV2Result listResult = s3Client.listObjectsV2( + new ListObjectsV2Request() + .withBucketName(s3Bucket) + .withPrefix(s3Path) + .withMaxKeys(1) + ); + if (listResult.getKeyCount() == 0) { // should never happen throw new ISE("Unable to list object [s3://%s/%s]", s3Bucket, s3Path); } - final S3Object s3Object = list[0]; - if (s3Object.getStorageClass() != null && - s3Object.getStorageClass().equals(S3Object.STORAGE_CLASS_GLACIER)) { - throw new ServiceException(StringUtils.format( + final S3ObjectSummary objectSummary = listResult.getObjectSummaries().get(0); + if (objectSummary.getStorageClass() != null && + StorageClass.fromValue(StringUtils.toUpperCase(objectSummary.getStorageClass())).equals(StorageClass.Glacier)) { + throw new ISE( "Cannot move file[s3://%s/%s] of storage class glacier, skipping.", s3Bucket, s3Path - )); + ); } else { log.info("Moving file %s", copyMsg); - final S3Object target = new S3Object(targetS3Path); + final CopyObjectRequest copyRequest = new CopyObjectRequest(s3Bucket, s3Path, targetS3Bucket, targetS3Path); if (!config.getDisableAcl()) { - target.setAcl(GSAccessControlList.REST_CANNED_BUCKET_OWNER_FULL_CONTROL); + copyRequest.setAccessControlList(S3Utils.grantFullControlToBucketOwver(s3Client, targetS3Bucket)); } - s3Client.copyObject( - s3Bucket, - s3Path, - targetS3Bucket, - target, - false - ); - if (!s3Client.isObjectInBucket(targetS3Bucket, targetS3Path)) { + s3Client.copyObject(copyRequest); + if (!s3Client.doesObjectExist(targetS3Bucket, targetS3Path)) { throw new IOE( "After copy was reported as successful the file doesn't exist in the target location [%s]", copyMsg @@ -199,7 +201,7 @@ private void selfCheckingMove( } } else { // ensure object exists in target location - if (s3Client.isObjectInBucket(targetS3Bucket, targetS3Path)) { + if (s3Client.doesObjectExist(targetS3Bucket, targetS3Path)) { log.info( "Not moving file [s3://%s/%s], already present in target location [s3://%s/%s]", s3Bucket, s3Path, diff --git a/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3DataSegmentPuller.java b/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3DataSegmentPuller.java index 4e2ae23e16b5..dee3c5c2978c 100644 --- a/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3DataSegmentPuller.java +++ b/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3DataSegmentPuller.java @@ -19,6 +19,11 @@ package io.druid.storage.s3; +import com.amazonaws.AmazonServiceException; +import com.amazonaws.services.s3.AmazonS3Client; +import com.amazonaws.services.s3.model.AmazonS3Exception; +import com.amazonaws.services.s3.model.S3Object; +import com.amazonaws.services.s3.model.S3ObjectSummary; import com.google.common.base.Predicate; import com.google.common.base.Strings; import com.google.common.base.Throwables; @@ -38,10 +43,6 @@ import io.druid.segment.loading.SegmentLoadingException; import io.druid.segment.loading.URIDataPuller; import io.druid.timeline.DataSegment; -import org.jets3t.service.S3ServiceException; -import org.jets3t.service.ServiceException; -import org.jets3t.service.impl.rest.httpclient.RestS3Service; -import org.jets3t.service.model.StorageObject; import javax.tools.FileObject; import java.io.File; @@ -60,17 +61,16 @@ public class S3DataSegmentPuller implements DataSegmentPuller, URIDataPuller { public static final int DEFAULT_RETRY_COUNT = 3; - public static FileObject buildFileObject(final URI uri, final RestS3Service s3Client) throws ServiceException + private static FileObject buildFileObject(final URI uri, final AmazonS3Client s3Client) throws AmazonServiceException { final S3Coords coords = new S3Coords(checkURI(uri)); - final StorageObject s3Obj = s3Client.getObjectDetails(coords.bucket, coords.path); + final S3ObjectSummary objectSummary = S3Utils.getSingleObjectSummary(s3Client, coords.bucket, coords.path); final String path = uri.getPath(); return new FileObject() { final Object inputStreamOpener = new Object(); - volatile boolean streamAcquired = false; - volatile StorageObject storageObject = s3Obj; + S3Object s3Object = null; @Override public URI toUri() @@ -89,18 +89,70 @@ public String getName() public InputStream openInputStream() throws IOException { try { - synchronized (inputStreamOpener) { - if (streamAcquired) { - return storageObject.getDataInputStream(); + if (s3Object == null) { + synchronized (inputStreamOpener) { + if (s3Object == null) { + // lazily promote to full GET + s3Object = s3Client.getObject(objectSummary.getBucketName(), objectSummary.getKey()); + } } - // lazily promote to full GET - storageObject = s3Client.getObject(s3Obj.getBucketName(), s3Obj.getKey()); - final InputStream stream = storageObject.getDataInputStream(); - streamAcquired = true; - return stream; } + + return new InputStream() + { + final InputStream delegate = s3Object.getObjectContent(); + + @Override + public int read() throws IOException + { + return delegate.read(); + } + + @Override + public int read(byte[] b, int off, int len) throws IOException + { + return delegate.read(b, off, len); + } + + @Override + public long skip(long n) throws IOException + { + return delegate.skip(n); + } + + @Override + public int available() throws IOException + { + return delegate.available(); + } + + @Override + public void reset() throws IOException + { + delegate.reset(); + } + + @Override + public void close() throws IOException + { + delegate.close(); + s3Object.close(); + } + + @Override + public boolean markSupported() + { + return delegate.markSupported(); + } + + @Override + public void mark(int readlimit) + { + delegate.mark(readlimit); + } + }; } - catch (ServiceException e) { + catch (AmazonServiceException e) { throw new IOE(e, "Could not load S3 URI [%s]", uri); } } @@ -132,7 +184,7 @@ public Writer openWriter() throws IOException @Override public long getLastModified() { - return s3Obj.getLastModifiedDate().getTime(); + return objectSummary.getLastModified().getTime(); } @Override @@ -150,11 +202,11 @@ public boolean delete() protected static final String BUCKET = "bucket"; protected static final String KEY = "key"; - protected final RestS3Service s3Client; + protected final AmazonS3Client s3Client; @Inject public S3DataSegmentPuller( - RestS3Service s3Client + AmazonS3Client s3Client ) { this.s3Client = s3Client; @@ -188,7 +240,7 @@ public InputStream openStream() throws IOException try { return buildFileObject(uri, s3Client).openInputStream(); } - catch (ServiceException e) { + catch (AmazonServiceException e) { if (e.getCause() != null) { if (S3Utils.S3RETRY.apply(e)) { throw new IOException("Recoverable exception", e); @@ -250,7 +302,7 @@ public InputStream getInputStream(URI uri) throws IOException try { return buildFileObject(uri, s3Client).openInputStream(); } - catch (ServiceException e) { + catch (AmazonServiceException e) { throw new IOE(e, "Could not load URI [%s]", uri); } } @@ -267,8 +319,8 @@ public boolean apply(Throwable e) if (e == null) { return false; } - if (e instanceof ServiceException) { - return S3Utils.isServiceExceptionRecoverable((ServiceException) e); + if (e instanceof AmazonServiceException) { + return S3Utils.isServiceExceptionRecoverable((AmazonServiceException) e); } if (S3Utils.S3RETRY.apply(e)) { return true; @@ -292,10 +344,11 @@ public boolean apply(Throwable e) public String getVersion(URI uri) throws IOException { try { - final FileObject object = buildFileObject(uri, s3Client); - return StringUtils.format("%d", object.getLastModified()); + final S3Coords coords = new S3Coords(checkURI(uri)); + final S3ObjectSummary objectSummary = S3Utils.getSingleObjectSummary(s3Client, coords.bucket, coords.path); + return StringUtils.format("%d", objectSummary.getLastModified().getTime()); } - catch (ServiceException e) { + catch (AmazonServiceException e) { if (S3Utils.isServiceExceptionRecoverable(e)) { // The recoverable logic is always true for IOException, so we want to only pass IOException if it is recoverable throw new IOE(e, "Could not fetch last modified timestamp from URI [%s]", uri); @@ -309,10 +362,10 @@ private boolean isObjectInBucket(final S3Coords coords) throws SegmentLoadingExc { try { return S3Utils.retryS3Operation( - () -> S3Utils.isObjectInBucket(s3Client, coords.bucket, coords.path) + () -> S3Utils.isObjectInBucketIgnoringPermission(s3Client, coords.bucket, coords.path) ); } - catch (S3ServiceException | IOException e) { + catch (AmazonS3Exception | IOException e) { throw new SegmentLoadingException(e, "S3 fail! Key[%s]", coords); } catch (Exception e) { diff --git a/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3DataSegmentPusher.java b/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3DataSegmentPusher.java index 985121bef622..1d8063610789 100644 --- a/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3DataSegmentPusher.java +++ b/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3DataSegmentPusher.java @@ -19,6 +19,9 @@ package io.druid.storage.s3; +import com.amazonaws.AmazonServiceException; +import com.amazonaws.services.s3.AmazonS3Client; +import com.amazonaws.services.s3.model.PutObjectRequest; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Throwables; import com.google.common.collect.ImmutableList; @@ -30,10 +33,6 @@ import io.druid.segment.SegmentUtils; import io.druid.segment.loading.DataSegmentPusher; import io.druid.timeline.DataSegment; -import org.jets3t.service.ServiceException; -import org.jets3t.service.acl.gs.GSAccessControlList; -import org.jets3t.service.impl.rest.httpclient.RestS3Service; -import org.jets3t.service.model.S3Object; import java.io.File; import java.io.IOException; @@ -46,13 +45,13 @@ public class S3DataSegmentPusher implements DataSegmentPusher { private static final EmittingLogger log = new EmittingLogger(S3DataSegmentPusher.class); - private final RestS3Service s3Client; + private final AmazonS3Client s3Client; private final S3DataSegmentPusherConfig config; private final ObjectMapper jsonMapper; @Inject public S3DataSegmentPusher( - RestS3Service s3Client, + AmazonS3Client s3Client, S3DataSegmentPusherConfig config, ObjectMapper jsonMapper ) @@ -100,37 +99,30 @@ public DataSegment push(final File indexFilesDir, final DataSegment inSegment, f try { return S3Utils.retryS3Operation( () -> { - S3Object toPush = new S3Object(zipOutFile); - putObject(config.getBucket(), s3Path, toPush, replaceExisting); + uploadFileAndClear(s3Client, config.getBucket(), s3Path, zipOutFile, replaceExisting); final DataSegment outSegment = inSegment.withSize(indexSize) - .withLoadSpec(makeLoadSpec(config.getBucket(), toPush.getKey())) + .withLoadSpec(makeLoadSpec(config.getBucket(), s3Path)) .withBinaryVersion(SegmentUtils.getVersionFromDir(indexFilesDir)); File descriptorFile = File.createTempFile("druid", "descriptor.json"); // Avoid using Guava in DataSegmentPushers because they might be used with very diverse Guava versions in // runtime, and because Guava deletes methods over time, that causes incompatibilities. Files.write(descriptorFile.toPath(), jsonMapper.writeValueAsBytes(outSegment)); - S3Object descriptorObject = new S3Object(descriptorFile); - putObject( + uploadFileAndClear( + s3Client, config.getBucket(), S3Utils.descriptorPathForSegmentPath(s3Path), - descriptorObject, + descriptorFile, replaceExisting ); - log.info("Deleting zipped index File[%s]", zipOutFile); - zipOutFile.delete(); - - log.info("Deleting descriptor file[%s]", descriptorFile); - descriptorFile.delete(); - return outSegment; } ); } - catch (ServiceException e) { + catch (AmazonServiceException e) { throw new IOException(e); } catch (Exception e) { @@ -163,21 +155,25 @@ private Map makeLoadSpec(String bucket, String key) ); } - private void putObject(String bucketName, String path, S3Object object, boolean replaceExisting) - throws ServiceException + private void uploadFileAndClear(AmazonS3Client s3Client, String bucket, String key, File file, boolean replaceExisting) { - object.setBucketName(bucketName); - object.setKey(path); + final PutObjectRequest indexFilePutRequest = new PutObjectRequest(bucket, key, file); + if (!config.getDisableAcl()) { - object.setAcl(GSAccessControlList.REST_CANNED_BUCKET_OWNER_FULL_CONTROL); + indexFilePutRequest.setAccessControlList( + S3Utils.grantFullControlToBucketOwver(s3Client, bucket) + ); } - log.info("Pushing %s.", object); - - if (!replaceExisting && S3Utils.isObjectInBucket(s3Client, bucketName, object.getKey())) { - log.info("Skipping push because key [%s] exists && replaceExisting == false", object.getKey()); + // TODO: maybe the below check can be moved out to the caller + if (!replaceExisting && S3Utils.isObjectInBucketIgnoringPermission(s3Client, bucket, key)) { + log.info("Skipping push because key [%s] exists && replaceExisting == false", key); } else { - s3Client.putObject(bucketName, object); + log.info("Pushing [%s] to bucket[%s] and key[%s].", file, bucket, key); + s3Client.putObject(indexFilePutRequest); } + + log.info("Deleting file [%s]", file.getAbsolutePath()); + file.delete(); } } diff --git a/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3StorageDruidModule.java b/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3StorageDruidModule.java index c9c0568f088d..f091f13cd880 100644 --- a/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3StorageDruidModule.java +++ b/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3StorageDruidModule.java @@ -20,7 +20,7 @@ package io.druid.storage.s3; import com.amazonaws.auth.AWSCredentialsProvider; -import com.amazonaws.auth.AWSSessionCredentials; +import com.amazonaws.services.s3.AmazonS3Client; import com.fasterxml.jackson.core.Version; import com.fasterxml.jackson.databind.Module; import com.google.common.collect.ImmutableList; @@ -33,8 +33,6 @@ import io.druid.guice.JsonConfigProvider; import io.druid.guice.LazySingleton; import io.druid.initialization.DruidModule; -import org.jets3t.service.impl.rest.httpclient.RestS3Service; -import org.jets3t.service.security.AWSCredentials; import java.util.List; @@ -102,15 +100,8 @@ public void configure(Binder binder) @Provides @LazySingleton - public RestS3Service getRestS3Service(AWSCredentialsProvider provider) + public AmazonS3Client getAmazonS3Client(AWSCredentialsProvider provider) { - if (provider.getCredentials() instanceof AWSSessionCredentials) { - return new RestS3Service(new AWSSessionCredentialsAdapter(provider)); - } else { - return new RestS3Service(new AWSCredentials( - provider.getCredentials().getAWSAccessKeyId(), - provider.getCredentials().getAWSSecretKey() - )); - } + return new AmazonS3Client(provider); } } diff --git a/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3TaskLogs.java b/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3TaskLogs.java index 3676d0942f47..89a41b093ae3 100644 --- a/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3TaskLogs.java +++ b/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3TaskLogs.java @@ -19,6 +19,11 @@ package io.druid.storage.s3; +import com.amazonaws.AmazonServiceException; +import com.amazonaws.services.s3.AmazonS3Client; +import com.amazonaws.services.s3.model.AmazonS3Exception; +import com.amazonaws.services.s3.model.GetObjectRequest; +import com.amazonaws.services.s3.model.ObjectMetadata; import com.google.common.base.Optional; import com.google.common.base.Throwables; import com.google.common.io.ByteSource; @@ -27,10 +32,6 @@ import io.druid.java.util.common.StringUtils; import io.druid.java.util.common.logger.Logger; import io.druid.tasklogs.TaskLogs; -import org.jets3t.service.ServiceException; -import org.jets3t.service.StorageService; -import org.jets3t.service.impl.rest.httpclient.RestS3Service; -import org.jets3t.service.model.StorageObject; import java.io.File; import java.io.IOException; @@ -43,11 +44,11 @@ public class S3TaskLogs implements TaskLogs { private static final Logger log = new Logger(S3TaskLogs.class); - private final StorageService service; + private final AmazonS3Client service; private final S3TaskLogsConfig config; @Inject - public S3TaskLogs(S3TaskLogsConfig config, RestS3Service service) + public S3TaskLogs(S3TaskLogsConfig config, AmazonS3Client service) { this.config = config; this.service = service; @@ -59,9 +60,9 @@ public Optional streamTaskLog(final String taskid, final long offset final String taskKey = getTaskLogKey(taskid); try { - final StorageObject objectDetails = service.getObjectDetails(config.getS3Bucket(), taskKey, null, null, null, null); + final ObjectMetadata objectMetadata = service.getObjectMetadata(config.getS3Bucket(), taskKey); - return Optional.of( + return Optional.of( new ByteSource() { @Override @@ -69,36 +70,31 @@ public InputStream openStream() throws IOException { try { final long start; - final long end = objectDetails.getContentLength() - 1; + final long end = objectMetadata.getContentLength() - 1; - if (offset > 0 && offset < objectDetails.getContentLength()) { + if (offset > 0 && offset < objectMetadata.getContentLength()) { start = offset; - } else if (offset < 0 && (-1 * offset) < objectDetails.getContentLength()) { - start = objectDetails.getContentLength() + offset; + } else if (offset < 0 && (-1 * offset) < objectMetadata.getContentLength()) { + start = objectMetadata.getContentLength() + offset; } else { start = 0; } - return service.getObject( - config.getS3Bucket(), - taskKey, - null, - null, - new String[]{objectDetails.getETag()}, - null, - start, - end - ).getDataInputStream(); + final GetObjectRequest request = new GetObjectRequest(config.getS3Bucket(), taskKey) + .withMatchingETagConstraint(objectMetadata.getETag()) + .withRange(start, end); + + return service.getObject(request).getObjectContent(); } - catch (ServiceException e) { + catch (AmazonServiceException e) { throw new IOException(e); } } } ); } - catch (ServiceException e) { - if (404 == e.getResponseCode() + catch (AmazonS3Exception e) { + if (404 == e.getStatusCode() || "NoSuchKey".equals(e.getErrorCode()) || "NoSuchBucket".equals(e.getErrorCode())) { return Optional.absent(); @@ -117,9 +113,7 @@ public void pushTaskLog(final String taskid, final File logFile) throws IOExcept try { S3Utils.retryS3Operation( () -> { - final StorageObject object = new StorageObject(logFile); - object.setKey(taskKey); - service.putObject(config.getS3Bucket(), object); + service.putObject(config.getS3Bucket(), taskKey, logFile); return null; } ); diff --git a/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3TimestampVersionedDataFinder.java b/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3TimestampVersionedDataFinder.java index 8014ec8ac88d..2401e713b39e 100644 --- a/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3TimestampVersionedDataFinder.java +++ b/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3TimestampVersionedDataFinder.java @@ -19,16 +19,17 @@ package io.druid.storage.s3; +import com.amazonaws.services.s3.AmazonS3Client; +import com.amazonaws.services.s3.model.S3ObjectSummary; import com.google.common.base.Throwables; import com.google.inject.Inject; import io.druid.data.SearchableVersionedDataFinder; import io.druid.java.util.common.RetryUtils; import io.druid.java.util.common.StringUtils; -import org.jets3t.service.impl.rest.httpclient.RestS3Service; -import org.jets3t.service.model.S3Object; import javax.annotation.Nullable; import java.net.URI; +import java.util.Iterator; import java.util.regex.Pattern; /** @@ -37,8 +38,10 @@ */ public class S3TimestampVersionedDataFinder extends S3DataSegmentPuller implements SearchableVersionedDataFinder { + private static final int MAX_LISTING_KEYS = 1000; + @Inject - public S3TimestampVersionedDataFinder(RestS3Service s3Client) + public S3TimestampVersionedDataFinder(AmazonS3Client s3Client) { super(s3Client); } @@ -65,23 +68,28 @@ public URI getLatestVersion(final URI uri, final @Nullable Pattern pattern) final S3Coords coords = new S3Coords(checkURI(uri)); long mostRecent = Long.MIN_VALUE; URI latest = null; - S3Object[] objects = s3Client.listObjects(coords.bucket, coords.path, null); - if (objects == null) { - return null; - } - for (S3Object storageObject : objects) { - storageObject.closeDataInputStream(); - String keyString = storageObject.getKey().substring(coords.path.length()); + final Iterator objectSummaryIterator = S3Utils.objectSummaryIterator( + s3Client, + coords.bucket, + coords.path, + MAX_LISTING_KEYS + ); + while (objectSummaryIterator.hasNext()) { + final S3ObjectSummary objectSummary = objectSummaryIterator.next(); + // TODO: what is going on here? + String keyString = objectSummary.getKey().substring(coords.path.length()); if (keyString.startsWith("/")) { keyString = keyString.substring(1); } if (pattern != null && !pattern.matcher(keyString).matches()) { continue; } - final long latestModified = storageObject.getLastModifiedDate().getTime(); + final long latestModified = objectSummary.getLastModified().getTime(); if (latestModified >= mostRecent) { mostRecent = latestModified; - latest = new URI(StringUtils.format("s3://%s/%s", storageObject.getBucketName(), storageObject.getKey())); + latest = new URI( + StringUtils.format("s3://%s/%s", objectSummary.getBucketName(), objectSummary.getKey()) + ); } } return latest; diff --git a/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3Utils.java b/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3Utils.java index 2a9372e96d08..8b1249810e58 100644 --- a/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3Utils.java +++ b/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3Utils.java @@ -19,19 +19,27 @@ package io.druid.storage.s3; +import com.amazonaws.AmazonServiceException; +import com.amazonaws.services.s3.AmazonS3Client; +import com.amazonaws.services.s3.model.AccessControlList; +import com.amazonaws.services.s3.model.AmazonS3Exception; +import com.amazonaws.services.s3.model.CanonicalGrantee; +import com.amazonaws.services.s3.model.Grant; +import com.amazonaws.services.s3.model.ListObjectsV2Request; +import com.amazonaws.services.s3.model.ListObjectsV2Result; +import com.amazonaws.services.s3.model.ObjectMetadata; +import com.amazonaws.services.s3.model.Permission; +import com.amazonaws.services.s3.model.S3ObjectSummary; import com.google.common.base.Joiner; import com.google.common.base.Predicate; -import com.google.common.base.Throwables; +import io.druid.java.util.common.ISE; import io.druid.java.util.common.RetryUtils; import io.druid.java.util.common.RetryUtils.Task; -import org.jets3t.service.ServiceException; -import org.jets3t.service.StorageObjectsChunk; -import org.jets3t.service.impl.rest.httpclient.RestS3Service; -import org.jets3t.service.model.S3Object; -import org.jets3t.service.model.StorageObject; import java.io.IOException; +import java.net.URI; import java.util.Iterator; +import java.util.NoSuchElementException; /** * @@ -39,25 +47,13 @@ public class S3Utils { private static final Joiner JOINER = Joiner.on("/").skipNulls(); + private static final String MIMETYPE_JETS3T_DIRECTORY = "application/x-directory"; - public static void closeStreamsQuietly(S3Object s3Obj) - { - if (s3Obj == null) { - return; - } - - try { - s3Obj.closeDataInputStream(); - } - catch (IOException e) { - - } - } - - public static boolean isServiceExceptionRecoverable(ServiceException ex) + static boolean isServiceExceptionRecoverable(AmazonServiceException ex) { final boolean isIOException = ex.getCause() instanceof IOException; - final boolean isTimeout = "RequestTimeout".equals(((ServiceException) ex).getErrorCode()); + // TODO: check the below error codes + final boolean isTimeout = "RequestTimeout".equals(ex.getErrorCode()); return isIOException || isTimeout; } @@ -70,8 +66,8 @@ public boolean apply(Throwable e) return false; } else if (e instanceof IOException) { return true; - } else if (e instanceof ServiceException) { - return isServiceExceptionRecoverable((ServiceException) e); + } else if (e instanceof AmazonServiceException) { + return isServiceExceptionRecoverable((AmazonServiceException) e); } else { return apply(e.getCause()); } @@ -88,95 +84,86 @@ public static T retryS3Operation(Task f) throws Exception return RetryUtils.retry(f, S3RETRY, maxTries); } - public static boolean isObjectInBucket(RestS3Service s3Client, String bucketName, String objectKey) - throws ServiceException + static boolean isObjectInBucketIgnoringPermission(AmazonS3Client s3Client, String bucketName, String objectKey) { try { - s3Client.getObjectDetails(bucketName, objectKey); + // TODO: check the below error codes + return s3Client.doesObjectExist(bucketName, objectKey); } - catch (ServiceException e) { - if (404 == e.getResponseCode() - || "NoSuchKey".equals(e.getErrorCode()) - || "NoSuchBucket".equals(e.getErrorCode())) { - return false; - } - if ("AccessDenied".equals(e.getErrorCode())) { + catch (AmazonS3Exception e) { + if (e.getStatusCode() == 403) { // Object is inaccessible to current user, but does exist. return true; } // Something else has gone wrong throw e; } - return true; } - public static Iterator storageObjectsIterator( - final RestS3Service s3Client, + public static Iterator objectSummaryIterator( + final AmazonS3Client s3Client, final String bucket, final String prefix, - final long maxListingLength + final int numMaxKeys ) { - return new Iterator() + final ListObjectsV2Request request = new ListObjectsV2Request() + .withBucketName(bucket) + .withPrefix(prefix) + .withMaxKeys(numMaxKeys); + + return new Iterator() { - private StorageObjectsChunk objectsChunk; - private int objectsChunkOffset; + private ListObjectsV2Result result; + private Iterator objectSummaryIterator; - @Override - public boolean hasNext() { - if (objectsChunk == null) { - objectsChunk = listObjectsChunkedAfter(""); - objectsChunkOffset = 0; - } - - if (objectsChunk.getObjects().length <= objectsChunkOffset) { - if (objectsChunk.isListingComplete()) { - return false; - } else { - objectsChunk = listObjectsChunkedAfter(objectsChunk.getPriorLastKey()); - objectsChunkOffset = 0; - } - } + fetchNextBatch(); + } - return true; + private void fetchNextBatch() + { + result = s3Client.listObjectsV2(request); + objectSummaryIterator = result.getObjectSummaries().iterator(); + request.setContinuationToken(result.getContinuationToken()); } - private StorageObjectsChunk listObjectsChunkedAfter(final String priorLastKey) + @Override + public boolean hasNext() { - try { - return retryS3Operation( - () -> s3Client.listObjectsChunked(bucket, prefix, null, maxListingLength, priorLastKey) - ); - } - catch (Exception e) { - throw Throwables.propagate(e); - } + return objectSummaryIterator.hasNext() || result.isTruncated(); } @Override - public StorageObject next() + public S3ObjectSummary next() { if (!hasNext()) { - throw new IllegalStateException(); + throw new NoSuchElementException(); } - StorageObject storageObject = objectsChunk.getObjects()[objectsChunkOffset]; - objectsChunkOffset++; - return storageObject; - } + if (objectSummaryIterator.hasNext()) { + return objectSummaryIterator.next(); + } - @Override - public void remove() - { - throw new UnsupportedOperationException(); - } + if (result.isTruncated()) { + fetchNextBatch(); + } + if (!objectSummaryIterator.hasNext()) { + throw new ISE( + "Failed to further iterate on bucket[%s] and prefix[%s]. The last continuationToken was [%s]", + bucket, + prefix, + result.getContinuationToken() + ); + } + return objectSummaryIterator.next(); + } }; } - public static String constructSegmentPath(String baseKey, String storageDir) + static String constructSegmentPath(String baseKey, String storageDir) { return JOINER.join( baseKey.isEmpty() ? null : baseKey, @@ -184,25 +171,85 @@ public static String constructSegmentPath(String baseKey, String storageDir) ) + "/index.zip"; } - public static String descriptorPathForSegmentPath(String s3Path) + static String descriptorPathForSegmentPath(String s3Path) { return s3Path.substring(0, s3Path.lastIndexOf("/")) + "/descriptor.json"; } - public static String indexZipForSegmentPath(String s3Path) + static String indexZipForSegmentPath(String s3Path) { return s3Path.substring(0, s3Path.lastIndexOf("/")) + "/index.zip"; } - public static String toFilename(String key) + static String toFilename(String key) { return toFilename(key, ""); } - public static String toFilename(String key, final String suffix) + static String toFilename(String key, final String suffix) { String filename = key.substring(key.lastIndexOf("/") + 1); // characters after last '/' filename = filename.substring(0, filename.length() - suffix.length()); // remove the suffix from the end return filename; } + + static AccessControlList grantFullControlToBucketOwver(AmazonS3Client s3Client, String bucket) + { + final AccessControlList acl = s3Client.getBucketAcl(bucket); + acl.grantAllPermissions(new Grant(new CanonicalGrantee(acl.getOwner().getId()), Permission.FullControl)); + return acl; + } + + public static String extractS3Key(URI uri) + { + return uri.getPath().startsWith("/") ? uri.getPath().substring(1) : uri.getPath(); + } + + // TODO: check this is possible with the official sdk + // Copied from org.jets3t.service.model.StorageObject.isDirectoryPlaceholder() + public static boolean isDirectoryPlaceholder(String key, ObjectMetadata objectMetadata) + { + // Recognize "standard" directory place-holder indications used by + // Amazon's AWS Console and Panic's Transmit. + if (key.endsWith("/") && objectMetadata.getContentLength() == 0) { + return true; + } + // Recognize s3sync.rb directory placeholders by MD5/ETag value. + if ("d66759af42f282e1ba19144df2d405d0".equals(objectMetadata.getETag())) { + return true; + } + // Recognize place-holder objects created by the Google Storage console + // or S3 Organizer Firefox extension. + if (key.endsWith("_$folder$") && objectMetadata.getContentLength() == 0) { + return true; + } + + // We don't use JetS3t APIs anymore, but the below check is still needed for backward compatibility. + + // Recognize legacy JetS3t directory place-holder objects, only gives + // accurate results if an object's metadata is populated. + if (objectMetadata.getContentLength() == 0 && MIMETYPE_JETS3T_DIRECTORY.equals(objectMetadata.getContentType())) { + return true; + } + return false; + } + + public static S3ObjectSummary getSingleObjectSummary(AmazonS3Client s3Client, String bucket, String key) + { + final ListObjectsV2Request request = new ListObjectsV2Request() + .withBucketName(bucket) + .withPrefix(key) + .withMaxKeys(1); + final ListObjectsV2Result result = s3Client.listObjectsV2(request); + + if (result.getKeyCount() == 0) { + throw new ISE("Cannot find object for bucket[%s] and key[%s]", bucket, key); + } + final S3ObjectSummary objectSummary = result.getObjectSummaries().get(0); + if (!objectSummary.getBucketName().equals(bucket) || !objectSummary.getKey().equals(key)) { + throw new ISE("Wrong object[%s] for bucket[%s] and key[%s]", objectSummary, bucket, key); + } + + return objectSummary; + } } diff --git a/extensions-core/s3-extensions/src/test/java/io/druid/firehose/s3/StaticS3FirehoseFactoryTest.java b/extensions-core/s3-extensions/src/test/java/io/druid/firehose/s3/StaticS3FirehoseFactoryTest.java index 3a5c94471cda..7b70d874e4e6 100644 --- a/extensions-core/s3-extensions/src/test/java/io/druid/firehose/s3/StaticS3FirehoseFactoryTest.java +++ b/extensions-core/s3-extensions/src/test/java/io/druid/firehose/s3/StaticS3FirehoseFactoryTest.java @@ -19,8 +19,12 @@ package io.druid.firehose.s3; +import com.amazonaws.services.s3.AmazonS3Client; +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.databind.DeserializationContext; import com.fasterxml.jackson.databind.Module; import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.deser.std.StdDeserializer; import com.fasterxml.jackson.databind.module.SimpleModule; import com.fasterxml.jackson.module.guice.ObjectMapperModule; import com.google.common.collect.ImmutableList; @@ -29,11 +33,10 @@ import com.google.inject.Injector; import com.google.inject.Provides; import io.druid.initialization.DruidModule; -import io.druid.jackson.DefaultObjectMapper; -import org.jets3t.service.impl.rest.httpclient.RestS3Service; import org.junit.Assert; import org.junit.Test; +import java.io.IOException; import java.net.URI; import java.util.Arrays; import java.util.List; @@ -42,7 +45,7 @@ */ public class StaticS3FirehoseFactoryTest { - private static final RestS3Service SERVICE = new RestS3Service(null); + private static final AmazonS3Client SERVICE = new AmazonS3Client(); @Test public void testSerde() throws Exception @@ -75,14 +78,14 @@ public void testSerde() throws Exception private static ObjectMapper createObjectMapper(DruidModule baseModule) { - final ObjectMapper baseMapper = new DefaultObjectMapper(); - baseModule.getJacksonModules().forEach(baseMapper::registerModule); - final Injector injector = Guice.createInjector( new ObjectMapperModule(), baseModule ); - return injector.getInstance(ObjectMapper.class); + final ObjectMapper baseMapper = injector.getInstance(ObjectMapper.class); + + baseModule.getJacksonModules().forEach(baseMapper::registerModule); + return baseMapper; } private static class TestS3Module implements DruidModule @@ -90,7 +93,9 @@ private static class TestS3Module implements DruidModule @Override public List getJacksonModules() { - return ImmutableList.of(new SimpleModule()); + // Deserializer is need for AmazonS3Client even though it is injected. + // See https://github.com/FasterXML/jackson-databind/issues/962. + return ImmutableList.of(new SimpleModule().addDeserializer(AmazonS3Client.class, new ItemDeserializer())); } @Override @@ -100,9 +105,30 @@ public void configure(Binder binder) } @Provides - public RestS3Service getRestS3Service() + public AmazonS3Client getAmazonS3Client() { return SERVICE; } } + + public static class ItemDeserializer extends StdDeserializer + { + public ItemDeserializer() + { + this(null); + } + + public ItemDeserializer(Class vc) + { + super(vc); + } + + @Override + public AmazonS3Client deserialize( + JsonParser jp, DeserializationContext ctxt + ) throws IOException + { + throw new UnsupportedOperationException(); + } + } } diff --git a/extensions-core/s3-extensions/src/test/java/io/druid/storage/s3/S3DataSegmentArchiverTest.java b/extensions-core/s3-extensions/src/test/java/io/druid/storage/s3/S3DataSegmentArchiverTest.java index 75a4d2f0f13e..152ac2416ff9 100644 --- a/extensions-core/s3-extensions/src/test/java/io/druid/storage/s3/S3DataSegmentArchiverTest.java +++ b/extensions-core/s3-extensions/src/test/java/io/druid/storage/s3/S3DataSegmentArchiverTest.java @@ -19,6 +19,7 @@ package io.druid.storage.s3; +import com.amazonaws.services.s3.AmazonS3Client; import com.fasterxml.jackson.databind.BeanProperty; import com.fasterxml.jackson.databind.DeserializationContext; import com.fasterxml.jackson.databind.InjectableValues; @@ -31,7 +32,6 @@ import io.druid.segment.loading.SegmentLoadingException; import io.druid.timeline.DataSegment; import org.easymock.EasyMock; -import org.jets3t.service.impl.rest.httpclient.RestS3Service; import org.junit.Assert; import org.junit.BeforeClass; import org.junit.Test; @@ -66,7 +66,7 @@ public String getArchiveBaseKey() } }; private static final S3DataSegmentPusherConfig PUSHER_CONFIG = new S3DataSegmentPusherConfig(); - private static final RestS3Service S3_SERVICE = EasyMock.createStrictMock(RestS3Service.class); + private static final AmazonS3Client S3_SERVICE = EasyMock.createStrictMock(AmazonS3Client.class); private static final S3DataSegmentPuller PULLER = new S3DataSegmentPuller(S3_SERVICE); private static final DataSegment SOURCE_SEGMENT = DataSegment .builder() diff --git a/extensions-core/s3-extensions/src/test/java/io/druid/storage/s3/S3DataSegmentFinderTest.java b/extensions-core/s3-extensions/src/test/java/io/druid/storage/s3/S3DataSegmentFinderTest.java index f0df427ccb29..08b2f40c35d3 100644 --- a/extensions-core/s3-extensions/src/test/java/io/druid/storage/s3/S3DataSegmentFinderTest.java +++ b/extensions-core/s3-extensions/src/test/java/io/druid/storage/s3/S3DataSegmentFinderTest.java @@ -19,6 +19,15 @@ package io.druid.storage.s3; +import com.amazonaws.AmazonServiceException; +import com.amazonaws.services.s3.AmazonS3Client; +import com.amazonaws.services.s3.model.AmazonS3Exception; +import com.amazonaws.services.s3.model.ListObjectsV2Request; +import com.amazonaws.services.s3.model.ListObjectsV2Result; +import com.amazonaws.services.s3.model.ObjectMetadata; +import com.amazonaws.services.s3.model.PutObjectResult; +import com.amazonaws.services.s3.model.S3Object; +import com.amazonaws.services.s3.model.S3ObjectSummary; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.jsontype.NamedType; import com.google.common.base.Predicate; @@ -31,18 +40,13 @@ import com.google.common.collect.Ordering; import com.google.common.collect.Sets; import io.druid.java.util.common.Intervals; +import io.druid.java.util.common.StringUtils; import io.druid.segment.TestHelper; import io.druid.segment.loading.SegmentLoadingException; import io.druid.timeline.DataSegment; import io.druid.timeline.partition.NumberedShardSpec; import org.apache.commons.io.FileUtils; import org.apache.commons.io.IOUtils; -import org.jets3t.service.S3ServiceException; -import org.jets3t.service.ServiceException; -import org.jets3t.service.StorageObjectsChunk; -import org.jets3t.service.impl.rest.httpclient.RestS3Service; -import org.jets3t.service.model.S3Object; -import org.jets3t.service.model.StorageObject; import org.junit.Assert; import org.junit.Before; import org.junit.BeforeClass; @@ -51,8 +55,12 @@ import org.junit.rules.TemporaryFolder; import javax.annotation.Nullable; +import java.io.ByteArrayInputStream; import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; import java.io.InputStream; +import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Set; @@ -106,7 +114,7 @@ public class S3DataSegmentFinderTest @Rule public final TemporaryFolder temporaryFolder = new TemporaryFolder(); - RestS3Service mockS3Client; + MockAmazonS3Client mockS3Client; S3DataSegmentPusherConfig config; private String bucket; @@ -123,8 +131,6 @@ public class S3DataSegmentFinderTest private String indexZip4_0; private String indexZip4_1; - - @BeforeClass public static void setUpStatic() { @@ -141,7 +147,7 @@ public void setUp() throws Exception config.setBucket(bucket); config.setBaseKey(baseKey); - mockS3Client = new MockStorageService(temporaryFolder.newFolder()); + mockS3Client = new MockAmazonS3Client(temporaryFolder.newFolder()); descriptor1 = S3Utils.descriptorPathForSegmentPath(baseKey + "/interval1/v1/0/"); @@ -155,17 +161,17 @@ public void setUp() throws Exception indexZip4_0 = S3Utils.indexZipForSegmentPath(descriptor4_0); indexZip4_1 = S3Utils.indexZipForSegmentPath(descriptor4_1); - mockS3Client.putObject(bucket, new S3Object(descriptor1, mapper.writeValueAsString(SEGMENT_1))); - mockS3Client.putObject(bucket, new S3Object(descriptor2, mapper.writeValueAsString(SEGMENT_2))); - mockS3Client.putObject(bucket, new S3Object(descriptor3, mapper.writeValueAsString(SEGMENT_3))); - mockS3Client.putObject(bucket, new S3Object(descriptor4_0, mapper.writeValueAsString(SEGMENT_4_0))); - mockS3Client.putObject(bucket, new S3Object(descriptor4_1, mapper.writeValueAsString(SEGMENT_4_1))); - - mockS3Client.putObject(bucket, new S3Object(indexZip1, "dummy")); - mockS3Client.putObject(bucket, new S3Object(indexZip2, "dummy")); - mockS3Client.putObject(bucket, new S3Object(indexZip3, "dummy")); - mockS3Client.putObject(bucket, new S3Object(indexZip4_0, "dummy")); - mockS3Client.putObject(bucket, new S3Object(indexZip4_1, "dummy")); + mockS3Client.putObject(bucket, descriptor1, mapper.writeValueAsString(SEGMENT_1)); + mockS3Client.putObject(bucket, descriptor2, mapper.writeValueAsString(SEGMENT_2)); + mockS3Client.putObject(bucket, descriptor3, mapper.writeValueAsString(SEGMENT_3)); + mockS3Client.putObject(bucket, descriptor4_0, mapper.writeValueAsString(SEGMENT_4_0)); + mockS3Client.putObject(bucket, descriptor4_1, mapper.writeValueAsString(SEGMENT_4_1)); + + mockS3Client.putObject(bucket, indexZip1, "dummy"); + mockS3Client.putObject(bucket, indexZip2, "dummy"); + mockS3Client.putObject(bucket, indexZip3, "dummy"); + mockS3Client.putObject(bucket, indexZip4_0, "dummy"); + mockS3Client.putObject(bucket, indexZip4_1, "dummy"); } @Test @@ -211,34 +217,34 @@ public void testFindSegments() throws Exception final String serializedSegment4_1 = mapper.writeValueAsString(updatedSegment4_1); Assert.assertNotEquals(serializedSegment1, - IOUtils.toString(mockS3Client.getObject(bucket, descriptor1).getDataInputStream())); + IOUtils.toString(mockS3Client.getObject(bucket, descriptor1).getObjectContent())); Assert.assertNotEquals(serializedSegment2, - IOUtils.toString(mockS3Client.getObject(bucket, descriptor2).getDataInputStream())); + IOUtils.toString(mockS3Client.getObject(bucket, descriptor2).getObjectContent())); Assert.assertNotEquals(serializedSegment3, - IOUtils.toString(mockS3Client.getObject(bucket, descriptor3).getDataInputStream())); + IOUtils.toString(mockS3Client.getObject(bucket, descriptor3).getObjectContent())); Assert.assertNotEquals(serializedSegment4_0, - IOUtils.toString(mockS3Client.getObject(bucket, descriptor4_0).getDataInputStream())); + IOUtils.toString(mockS3Client.getObject(bucket, descriptor4_0).getObjectContent())); Assert.assertNotEquals(serializedSegment4_1, - IOUtils.toString(mockS3Client.getObject(bucket, descriptor4_1).getDataInputStream())); + IOUtils.toString(mockS3Client.getObject(bucket, descriptor4_1).getObjectContent())); final Set segments2 = s3DataSegmentFinder.findSegments("", true); Assert.assertEquals(segments, segments2); Assert.assertEquals(serializedSegment1, - IOUtils.toString(mockS3Client.getObject(bucket, descriptor1).getDataInputStream())); + IOUtils.toString(mockS3Client.getObject(bucket, descriptor1).getObjectContent())); Assert.assertEquals(serializedSegment2, - IOUtils.toString(mockS3Client.getObject(bucket, descriptor2).getDataInputStream())); + IOUtils.toString(mockS3Client.getObject(bucket, descriptor2).getObjectContent())); Assert.assertEquals(serializedSegment3, - IOUtils.toString(mockS3Client.getObject(bucket, descriptor3).getDataInputStream())); + IOUtils.toString(mockS3Client.getObject(bucket, descriptor3).getObjectContent())); Assert.assertEquals(serializedSegment4_0, - IOUtils.toString(mockS3Client.getObject(bucket, descriptor4_0).getDataInputStream())); + IOUtils.toString(mockS3Client.getObject(bucket, descriptor4_0).getObjectContent())); Assert.assertEquals(serializedSegment4_1, - IOUtils.toString(mockS3Client.getObject(bucket, descriptor4_1).getDataInputStream())); + IOUtils.toString(mockS3Client.getObject(bucket, descriptor4_1).getObjectContent())); } @Test(expected = SegmentLoadingException.class) - public void testFindSegmentsFail() throws SegmentLoadingException, ServiceException + public void testFindSegmentsFail() throws SegmentLoadingException { mockS3Client.deleteObject(bucket, indexZip4_1); @@ -276,21 +282,8 @@ public void testFindSegmentsUpdateLoadSpec() throws Exception final String descriptorPath = S3Utils.descriptorPathForSegmentPath(segmentPath); final String indexPath = S3Utils.indexZipForSegmentPath(segmentPath); - mockS3Client.putObject( - config.getBucket(), - new S3Object( - descriptorPath, - mapper.writeValueAsString(segmentMissingLoadSpec) - ) - ); - - mockS3Client.putObject( - config.getBucket(), - new S3Object( - indexPath, - "dummy" - ) - ); + mockS3Client.putObject(config.getBucket(), descriptorPath, mapper.writeValueAsString(segmentMissingLoadSpec)); + mockS3Client.putObject(config.getBucket(), indexPath, "dummy"); Set segments = s3DataSegmentFinder.findSegments(segmentPath, false); Assert.assertEquals(1, segments.size()); @@ -309,24 +302,34 @@ private String getDescriptorPath(DataSegment segment) return S3Utils.descriptorPathForSegmentPath(String.valueOf(segment.getLoadSpec().get("key"))); } - private static class MockStorageService extends RestS3Service + private static class MockAmazonS3Client extends AmazonS3Client { private final File baseDir; private final Map> storage = Maps.newHashMap(); - public MockStorageService(File baseDir) + public MockAmazonS3Client(File baseDir) { - super(null); + super(); this.baseDir = baseDir; } @Override - public StorageObjectsChunk listObjectsChunked( - final String bucketName, final String prefix, final String delimiter, - final long maxListingLength, final String priorLastKey - ) throws ServiceException + public boolean doesObjectExist(String bucketName, String objectName) { - List keysOrigin = Lists.newArrayList(storage.get(bucketName)); + final Set keys = storage.get(bucketName); + if (keys != null) { + return keys.contains(objectName); + } + return false; + } + + @Override + public ListObjectsV2Result listObjectsV2(ListObjectsV2Request listObjectsV2Request) + { + final String bucketName = listObjectsV2Request.getBucketName(); + final String prefix = listObjectsV2Request.getPrefix(); + + final List keysOrigin = Lists.newArrayList(storage.get(bucketName)); Predicate prefixFilter = new Predicate() { @@ -342,11 +345,11 @@ public boolean apply(@Nullable String input) ); int startOffset = 0; - if (priorLastKey != null) { - startOffset = keys.indexOf(priorLastKey) + 1; + if (listObjectsV2Request.getContinuationToken() != null) { + startOffset = keys.indexOf(listObjectsV2Request.getContinuationToken()) + 1; } - int endOffset = startOffset + (int) maxListingLength; // exclusive + int endOffset = startOffset + listObjectsV2Request.getMaxKeys(); // exclusive if (endOffset > keys.size()) { endOffset = keys.size(); } @@ -356,64 +359,72 @@ public boolean apply(@Nullable String input) newPriorLastkey = null; } - List objects = Lists.newArrayList(); + List objects = new ArrayList<>(); for (String objectKey : keys.subList(startOffset, endOffset)) { - objects.add(getObjectDetails(bucketName, objectKey)); + final S3ObjectSummary objectSummary = new S3ObjectSummary(); + objectSummary.setBucketName(bucketName); + objectSummary.setKey(objectKey); + objects.add(objectSummary); } - return new StorageObjectsChunk( - prefix, delimiter, objects.toArray(new StorageObject[]{}), null, newPriorLastkey); + final ListObjectsV2Result result = new ListObjectsV2Result(); + result.setBucketName(bucketName); + result.setKeyCount(objects.size()); + result.getObjectSummaries().addAll(objects); + result.setContinuationToken(newPriorLastkey); + result.setTruncated(newPriorLastkey != null); + + return result; } @Override - public StorageObject getObjectDetails(String bucketName, String objectKey) throws ServiceException + public S3Object getObject(String bucketName, String objectKey) { - if (!storage.containsKey(bucketName)) { - ServiceException ex = new ServiceException(); - ex.setResponseCode(404); + AmazonServiceException ex = new AmazonS3Exception("S3DataSegmentFinderTest"); + ex.setStatusCode(404); ex.setErrorCode("NoSuchBucket"); throw ex; } if (!storage.get(bucketName).contains(objectKey)) { - ServiceException ex = new ServiceException(); - ex.setResponseCode(404); + AmazonServiceException ex = new AmazonS3Exception("S3DataSegmentFinderTest"); + ex.setStatusCode(404); ex.setErrorCode("NoSuchKey"); throw ex; } final File objectPath = new File(baseDir, objectKey); - StorageObject storageObject = new StorageObject(); + S3Object storageObject = new S3Object(); storageObject.setBucketName(bucketName); storageObject.setKey(objectKey); - storageObject.setDataInputFile(objectPath); + try { + storageObject.setObjectContent(new FileInputStream(objectPath)); + } + catch (FileNotFoundException e) { + AmazonServiceException ex = new AmazonS3Exception("S3DataSegmentFinderTest", e); + ex.setStatusCode(500); + ex.setErrorCode("InternalError"); + throw ex; + } return storageObject; } - @Override - public S3Object getObject(String bucketName, String objectKey) throws S3ServiceException + public PutObjectResult putObject(String bucketName, String key, String data) { - final File objectPath = new File(baseDir, objectKey); - S3Object s3Object = new S3Object(); - s3Object.setBucketName(bucketName); - s3Object.setKey(objectKey); - s3Object.setDataInputFile(objectPath); - - return s3Object; - + return putObject(bucketName, key, new ByteArrayInputStream(StringUtils.toUtf8(data)), null); } @Override - public S3Object putObject(final String bucketName, final S3Object object) throws S3ServiceException + public PutObjectResult putObject(String bucketName, String key, InputStream input, ObjectMetadata metadata) { if (!storage.containsKey(bucketName)) { - storage.put(bucketName, Sets.newHashSet()); + storage.put(bucketName, Sets.newHashSet()); } - storage.get(bucketName).add(object.getKey()); + storage.get(bucketName).add(key); - final File objectPath = new File(baseDir, object.getKey()); + final File objectPath = new File(baseDir, key); if (!objectPath.getParentFile().exists()) { objectPath.getParentFile().mkdirs(); @@ -421,7 +432,7 @@ public S3Object putObject(final String bucketName, final S3Object object) throws try { try ( - InputStream in = object.getDataInputStream() + InputStream in = input ) { FileUtils.copyInputStreamToFile(in, objectPath); } @@ -430,11 +441,11 @@ public S3Object putObject(final String bucketName, final S3Object object) throws throw Throwables.propagate(e); } - return object; + return new PutObjectResult(); } @Override - public void deleteObject(String bucketName, String objectKey) throws ServiceException + public void deleteObject(String bucketName, String objectKey) { storage.get(bucketName).remove(objectKey); final File objectPath = new File(baseDir, objectKey); diff --git a/extensions-core/s3-extensions/src/test/java/io/druid/storage/s3/S3DataSegmentMoverTest.java b/extensions-core/s3-extensions/src/test/java/io/druid/storage/s3/S3DataSegmentMoverTest.java index 220cf93da77c..a848eb180822 100644 --- a/extensions-core/s3-extensions/src/test/java/io/druid/storage/s3/S3DataSegmentMoverTest.java +++ b/extensions-core/s3-extensions/src/test/java/io/druid/storage/s3/S3DataSegmentMoverTest.java @@ -19,6 +19,22 @@ package io.druid.storage.s3; +import com.amazonaws.services.s3.AmazonS3Client; +import com.amazonaws.services.s3.model.AccessControlList; +import com.amazonaws.services.s3.model.AmazonS3Exception; +import com.amazonaws.services.s3.model.CanonicalGrantee; +import com.amazonaws.services.s3.model.CopyObjectRequest; +import com.amazonaws.services.s3.model.CopyObjectResult; +import com.amazonaws.services.s3.model.GetObjectMetadataRequest; +import com.amazonaws.services.s3.model.Grant; +import com.amazonaws.services.s3.model.ListObjectsV2Request; +import com.amazonaws.services.s3.model.ListObjectsV2Result; +import com.amazonaws.services.s3.model.ObjectMetadata; +import com.amazonaws.services.s3.model.Owner; +import com.amazonaws.services.s3.model.Permission; +import com.amazonaws.services.s3.model.PutObjectResult; +import com.amazonaws.services.s3.model.S3ObjectSummary; +import com.amazonaws.services.s3.model.StorageClass; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Maps; @@ -28,14 +44,11 @@ import io.druid.segment.loading.SegmentLoadingException; import io.druid.timeline.DataSegment; import io.druid.timeline.partition.NoneShardSpec; -import org.jets3t.service.S3ServiceException; -import org.jets3t.service.ServiceException; -import org.jets3t.service.impl.rest.httpclient.RestS3Service; -import org.jets3t.service.model.S3Object; -import org.jets3t.service.model.StorageObject; import org.junit.Assert; import org.junit.Test; +import java.io.File; +import java.util.HashSet; import java.util.Map; import java.util.Set; @@ -61,11 +74,17 @@ public class S3DataSegmentMoverTest @Test public void testMove() throws Exception { - MockStorageService mockS3Client = new MockStorageService(); + MockAmazonS3Client mockS3Client = new MockAmazonS3Client(); S3DataSegmentMover mover = new S3DataSegmentMover(mockS3Client, new S3DataSegmentPusherConfig()); - mockS3Client.putObject("main", new S3Object("baseKey/test/2013-01-01T00:00:00.000Z_2013-01-02T00:00:00.000Z/1/0/index.zip")); - mockS3Client.putObject("main", new S3Object("baseKey/test/2013-01-01T00:00:00.000Z_2013-01-02T00:00:00.000Z/1/0/descriptor.json")); + mockS3Client.putObject( + "main", + "baseKey/test/2013-01-01T00:00:00.000Z_2013-01-02T00:00:00.000Z/1/0/index.zip" + ); + mockS3Client.putObject( + "main", + "baseKey/test/2013-01-01T00:00:00.000Z_2013-01-02T00:00:00.000Z/1/0/descriptor.json" + ); DataSegment movedSegment = mover.move( sourceSegment, @@ -81,11 +100,17 @@ public void testMove() throws Exception @Test public void testMoveNoop() throws Exception { - MockStorageService mockS3Client = new MockStorageService(); + MockAmazonS3Client mockS3Client = new MockAmazonS3Client(); S3DataSegmentMover mover = new S3DataSegmentMover(mockS3Client, new S3DataSegmentPusherConfig()); - mockS3Client.putObject("archive", new S3Object("targetBaseKey/test/2013-01-01T00:00:00.000Z_2013-01-02T00:00:00.000Z/1/0/index.zip")); - mockS3Client.putObject("archive", new S3Object("targetBaseKey/test/2013-01-01T00:00:00.000Z_2013-01-02T00:00:00.000Z/1/0/descriptor.json")); + mockS3Client.putObject( + "archive", + "targetBaseKey/test/2013-01-01T00:00:00.000Z_2013-01-02T00:00:00.000Z/1/0/index.zip" + ); + mockS3Client.putObject( + "archive", + "targetBaseKey/test/2013-01-01T00:00:00.000Z_2013-01-02T00:00:00.000Z/1/0/descriptor.json" + ); DataSegment movedSegment = mover.move( sourceSegment, @@ -102,7 +127,7 @@ public void testMoveNoop() throws Exception @Test(expected = SegmentLoadingException.class) public void testMoveException() throws Exception { - MockStorageService mockS3Client = new MockStorageService(); + MockAmazonS3Client mockS3Client = new MockAmazonS3Client(); S3DataSegmentMover mover = new S3DataSegmentMover(mockS3Client, new S3DataSegmentPusherConfig()); mover.move( @@ -114,7 +139,7 @@ public void testMoveException() throws Exception @Test public void testIgnoresGoneButAlreadyMoved() throws Exception { - MockStorageService mockS3Client = new MockStorageService(); + MockAmazonS3Client mockS3Client = new MockAmazonS3Client(); S3DataSegmentMover mover = new S3DataSegmentMover(mockS3Client, new S3DataSegmentPusherConfig()); mover.move(new DataSegment( "test", @@ -137,7 +162,7 @@ public void testIgnoresGoneButAlreadyMoved() throws Exception @Test(expected = SegmentLoadingException.class) public void testFailsToMoveMissing() throws Exception { - MockStorageService mockS3Client = new MockStorageService(); + MockAmazonS3Client mockS3Client = new MockAmazonS3Client(); S3DataSegmentMover mover = new S3DataSegmentMover(mockS3Client, new S3DataSegmentPusherConfig()); mover.move(new DataSegment( "test", @@ -157,15 +182,15 @@ public void testFailsToMoveMissing() throws Exception ), ImmutableMap.of("bucket", "DOES NOT EXIST", "baseKey", "baseKey2")); } - private static class MockStorageService extends RestS3Service + private static class MockAmazonS3Client extends AmazonS3Client { Map> storage = Maps.newHashMap(); boolean copied = false; boolean deletedOld = false; - private MockStorageService() throws S3ServiceException + private MockAmazonS3Client() { - super(null); + super(); } public boolean didMove() @@ -174,59 +199,90 @@ public boolean didMove() } @Override - public boolean isObjectInBucket(String bucketName, String objectKey) throws ServiceException + public AccessControlList getBucketAcl(String bucketName) + { + final AccessControlList acl = new AccessControlList(); + acl.setOwner(new Owner("ownerId", "owner")); + acl.grantAllPermissions(new Grant(new CanonicalGrantee(acl.getOwner().getId()), Permission.FullControl)); + return acl; + } + + @Override + public ObjectMetadata getObjectMetadata(GetObjectMetadataRequest getObjectMetadataRequest) + { + return new ObjectMetadata(); + } + + @Override + public boolean doesObjectExist(String bucketName, String objectKey) { Set objects = storage.get(bucketName); return (objects != null && objects.contains(objectKey)); } @Override - public S3Object[] listObjects(String bucketName, String objectKey, String separator) + public ListObjectsV2Result listObjectsV2(ListObjectsV2Request listObjectsV2Request) { - try { - if (isObjectInBucket(bucketName, objectKey)) { - final S3Object object = new S3Object(objectKey); - object.setStorageClass(S3Object.STORAGE_CLASS_STANDARD); - return new S3Object[]{object}; - } - } - catch (ServiceException e) { - // return empty list + final String bucketName = listObjectsV2Request.getBucketName(); + final String objectKey = listObjectsV2Request.getPrefix(); + if (doesObjectExist(bucketName, objectKey)) { + final S3ObjectSummary objectSummary = new S3ObjectSummary(); + objectSummary.setBucketName(bucketName); + objectSummary.setKey(objectKey); + objectSummary.setStorageClass(StorageClass.Standard.name()); + + final ListObjectsV2Result result = new ListObjectsV2Result(); + result.setBucketName(bucketName); + result.setPrefix(objectKey); + result.setKeyCount(1); + result.getObjectSummaries().add(objectSummary); + result.setTruncated(true); + return result; + } else { + return new ListObjectsV2Result(); } - return new S3Object[]{}; } @Override - public Map copyObject( - String sourceBucketName, - String sourceObjectKey, - String destinationBucketName, - StorageObject destinationObject, - boolean replaceMetadata - ) throws ServiceException + public CopyObjectResult copyObject(CopyObjectRequest copyObjectRequest) { + final String sourceBucketName = copyObjectRequest.getSourceBucketName(); + final String sourceObjectKey = copyObjectRequest.getSourceKey(); + final String destinationBucketName = copyObjectRequest.getDestinationBucketName(); + final String destinationObjectKey = copyObjectRequest.getDestinationKey(); copied = true; - if (isObjectInBucket(sourceBucketName, sourceObjectKey)) { - this.putObject(destinationBucketName, new S3Object(destinationObject.getKey())); + if (doesObjectExist(sourceBucketName, sourceObjectKey)) { + storage.computeIfAbsent(destinationBucketName, k -> new HashSet<>()) + .add(destinationObjectKey); + return new CopyObjectResult(); + } else { + final AmazonS3Exception exception = new AmazonS3Exception("S3DataSegmentMoverTest"); + exception.setErrorCode("NoSuchKey"); + exception.setStatusCode(404); + throw exception; } - return null; } @Override - public void deleteObject(String bucket, String objectKey) throws S3ServiceException + public void deleteObject(String bucket, String objectKey) { deletedOld = true; storage.get(bucket).remove(objectKey); } + public PutObjectResult putObject(String bucketName, String key) + { + return putObject(bucketName, key, (File) null); + } + @Override - public S3Object putObject(String bucketName, S3Object object) throws S3ServiceException + public PutObjectResult putObject(String bucketName, String key, File file) { if (!storage.containsKey(bucketName)) { storage.put(bucketName, Sets.newHashSet()); } - storage.get(bucketName).add(object.getKey()); - return object; + storage.get(bucketName).add(key); + return new PutObjectResult(); } } } diff --git a/extensions-core/s3-extensions/src/test/java/io/druid/storage/s3/S3DataSegmentPullerTest.java b/extensions-core/s3-extensions/src/test/java/io/druid/storage/s3/S3DataSegmentPullerTest.java index 303bf657d6e5..8bc028a64f0a 100644 --- a/extensions-core/s3-extensions/src/test/java/io/druid/storage/s3/S3DataSegmentPullerTest.java +++ b/extensions-core/s3-extensions/src/test/java/io/druid/storage/s3/S3DataSegmentPullerTest.java @@ -19,9 +19,21 @@ package io.druid.storage.s3; +import com.amazonaws.services.s3.AmazonS3Client; +import com.amazonaws.services.s3.model.AmazonS3Exception; +import com.amazonaws.services.s3.model.ListObjectsV2Request; +import com.amazonaws.services.s3.model.ListObjectsV2Result; +import com.amazonaws.services.s3.model.S3Object; +import com.amazonaws.services.s3.model.S3ObjectSummary; import io.druid.java.util.common.FileUtils; import io.druid.java.util.common.StringUtils; import io.druid.segment.loading.SegmentLoadingException; +import org.easymock.EasyMock; +import org.junit.Assert; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + import java.io.File; import java.io.FileInputStream; import java.io.FileOutputStream; @@ -30,15 +42,6 @@ import java.net.URI; import java.util.Date; import java.util.zip.GZIPOutputStream; -import org.easymock.EasyMock; -import org.jets3t.service.S3ServiceException; -import org.jets3t.service.ServiceException; -import org.jets3t.service.impl.rest.httpclient.RestS3Service; -import org.jets3t.service.model.S3Object; -import org.junit.Assert; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; /** * @@ -50,26 +53,29 @@ public class S3DataSegmentPullerTest public TemporaryFolder temporaryFolder = new TemporaryFolder(); @Test - public void testSimpleGetVersion() throws ServiceException, IOException + public void testSimpleGetVersion() throws IOException { String bucket = "bucket"; String keyPrefix = "prefix/dir/0"; - RestS3Service s3Client = EasyMock.createStrictMock(RestS3Service.class); + AmazonS3Client s3Client = EasyMock.createStrictMock(AmazonS3Client.class); - S3Object object0 = new S3Object(); + final S3ObjectSummary objectSummary = new S3ObjectSummary(); + objectSummary.setBucketName(bucket); + objectSummary.setKey(keyPrefix + "/renames-0.gz"); + objectSummary.setLastModified(new Date(0)); - object0.setBucketName(bucket); - object0.setKey(keyPrefix + "/renames-0.gz"); - object0.setLastModifiedDate(new Date(0)); + final ListObjectsV2Result result = new ListObjectsV2Result(); + result.setKeyCount(1); + result.getObjectSummaries().add(objectSummary); - EasyMock.expect(s3Client.getObjectDetails(EasyMock.eq(bucket), EasyMock.eq(object0.getKey()))) - .andReturn(object0) + EasyMock.expect(s3Client.listObjectsV2(EasyMock.anyObject(ListObjectsV2Request.class))) + .andReturn(result) .once(); S3DataSegmentPuller puller = new S3DataSegmentPuller(s3Client); EasyMock.replay(s3Client); - String version = puller.getVersion(URI.create(StringUtils.format("s3://%s/%s", bucket, object0.getKey()))); + String version = puller.getVersion(URI.create(StringUtils.format("s3://%s/%s", bucket, objectSummary.getKey()))); EasyMock.verify(s3Client); @@ -77,11 +83,11 @@ public void testSimpleGetVersion() throws ServiceException, IOException } @Test - public void testGZUncompress() throws ServiceException, IOException, SegmentLoadingException + public void testGZUncompress() throws IOException, SegmentLoadingException { final String bucket = "bucket"; final String keyPrefix = "prefix/dir/0"; - final RestS3Service s3Client = EasyMock.createStrictMock(RestS3Service.class); + final AmazonS3Client s3Client = EasyMock.createStrictMock(AmazonS3Client.class); final byte[] value = bucket.getBytes("utf8"); final File tmpFile = temporaryFolder.newFile("gzTest.gz"); @@ -91,19 +97,27 @@ public void testGZUncompress() throws ServiceException, IOException, SegmentLoad } final S3Object object0 = new S3Object(); - object0.setBucketName(bucket); object0.setKey(keyPrefix + "/renames-0.gz"); - object0.setLastModifiedDate(new Date(0)); - object0.setDataInputStream(new FileInputStream(tmpFile)); + object0.getObjectMetadata().setLastModified(new Date(0)); + object0.setObjectContent(new FileInputStream(tmpFile)); + + final S3ObjectSummary objectSummary = new S3ObjectSummary(); + objectSummary.setBucketName(bucket); + objectSummary.setKey(keyPrefix + "/renames-0.gz"); + objectSummary.setLastModified(new Date(0)); + + final ListObjectsV2Result listObjectsResult = new ListObjectsV2Result(); + listObjectsResult.setKeyCount(1); + listObjectsResult.getObjectSummaries().add(objectSummary); final File tmpDir = temporaryFolder.newFolder("gzTestDir"); - EasyMock.expect(s3Client.getObjectDetails(EasyMock.eq(object0.getBucketName()), EasyMock.eq(object0.getKey()))) - .andReturn(null) + EasyMock.expect(s3Client.doesObjectExist(EasyMock.eq(object0.getBucketName()), EasyMock.eq(object0.getKey()))) + .andReturn(true) .once(); - EasyMock.expect(s3Client.getObjectDetails(EasyMock.eq(object0.getBucketName()), EasyMock.eq(object0.getKey()))) - .andReturn(object0) + EasyMock.expect(s3Client.listObjectsV2(EasyMock.anyObject(ListObjectsV2Request.class))) + .andReturn(listObjectsResult) .once(); EasyMock.expect(s3Client.getObject(EasyMock.eq(object0.getBucketName()), EasyMock.eq(object0.getKey()))) .andReturn(object0) @@ -126,11 +140,11 @@ public void testGZUncompress() throws ServiceException, IOException, SegmentLoad } @Test - public void testGZUncompressRetries() throws ServiceException, IOException, SegmentLoadingException + public void testGZUncompressRetries() throws IOException, SegmentLoadingException { final String bucket = "bucket"; final String keyPrefix = "prefix/dir/0"; - final RestS3Service s3Client = EasyMock.createStrictMock(RestS3Service.class); + final AmazonS3Client s3Client = EasyMock.createStrictMock(AmazonS3Client.class); final byte[] value = bucket.getBytes("utf8"); final File tmpFile = temporaryFolder.newFile("gzTest.gz"); @@ -143,25 +157,34 @@ public void testGZUncompressRetries() throws ServiceException, IOException, Segm object0.setBucketName(bucket); object0.setKey(keyPrefix + "/renames-0.gz"); - object0.setLastModifiedDate(new Date(0)); - object0.setDataInputStream(new FileInputStream(tmpFile)); + object0.getObjectMetadata().setLastModified(new Date(0)); + object0.setObjectContent(new FileInputStream(tmpFile)); + + final S3ObjectSummary objectSummary = new S3ObjectSummary(); + objectSummary.setBucketName(bucket); + objectSummary.setKey(keyPrefix + "/renames-0.gz"); + objectSummary.setLastModified(new Date(0)); + + final ListObjectsV2Result listObjectsResult = new ListObjectsV2Result(); + listObjectsResult.setKeyCount(1); + listObjectsResult.getObjectSummaries().add(objectSummary); File tmpDir = temporaryFolder.newFolder("gzTestDir"); - S3ServiceException exception = new S3ServiceException(); + AmazonS3Exception exception = new AmazonS3Exception("S3DataSegmentPullerTest"); exception.setErrorCode("NoSuchKey"); - exception.setResponseCode(404); - EasyMock.expect(s3Client.getObjectDetails(EasyMock.eq(object0.getBucketName()), EasyMock.eq(object0.getKey()))) - .andReturn(null) + exception.setStatusCode(404); + EasyMock.expect(s3Client.doesObjectExist(EasyMock.eq(object0.getBucketName()), EasyMock.eq(object0.getKey()))) + .andReturn(true) .once(); - EasyMock.expect(s3Client.getObjectDetails(EasyMock.eq(object0.getBucketName()), EasyMock.eq(object0.getKey()))) - .andReturn(object0) + EasyMock.expect(s3Client.listObjectsV2(EasyMock.anyObject(ListObjectsV2Request.class))) + .andReturn(listObjectsResult) .once(); EasyMock.expect(s3Client.getObject(EasyMock.eq(bucket), EasyMock.eq(object0.getKey()))) .andThrow(exception) .once(); - EasyMock.expect(s3Client.getObjectDetails(EasyMock.eq(object0.getBucketName()), EasyMock.eq(object0.getKey()))) - .andReturn(object0) + EasyMock.expect(s3Client.listObjectsV2(EasyMock.anyObject(ListObjectsV2Request.class))) + .andReturn(listObjectsResult) .once(); EasyMock.expect(s3Client.getObject(EasyMock.eq(bucket), EasyMock.eq(object0.getKey()))) .andReturn(object0) diff --git a/extensions-core/s3-extensions/src/test/java/io/druid/storage/s3/S3DataSegmentPusherTest.java b/extensions-core/s3-extensions/src/test/java/io/druid/storage/s3/S3DataSegmentPusherTest.java index f26bd1610398..c787ed7d900c 100644 --- a/extensions-core/s3-extensions/src/test/java/io/druid/storage/s3/S3DataSegmentPusherTest.java +++ b/extensions-core/s3-extensions/src/test/java/io/druid/storage/s3/S3DataSegmentPusherTest.java @@ -19,6 +19,14 @@ package io.druid.storage.s3; +import com.amazonaws.services.s3.AmazonS3Client; +import com.amazonaws.services.s3.model.AccessControlList; +import com.amazonaws.services.s3.model.CanonicalGrantee; +import com.amazonaws.services.s3.model.Grant; +import com.amazonaws.services.s3.model.Owner; +import com.amazonaws.services.s3.model.Permission; +import com.amazonaws.services.s3.model.PutObjectRequest; +import com.amazonaws.services.s3.model.PutObjectResult; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.Lists; import com.google.common.collect.Maps; @@ -31,14 +39,13 @@ import org.easymock.Capture; import org.easymock.EasyMock; import org.easymock.IAnswer; -import org.jets3t.service.impl.rest.httpclient.RestS3Service; -import org.jets3t.service.model.S3Object; import org.junit.Assert; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; import java.io.File; +import java.io.FileInputStream; /** */ @@ -65,27 +72,38 @@ public void setValue(T value) @Test public void testPush() throws Exception { - RestS3Service s3Client = EasyMock.createStrictMock(RestS3Service.class); + AmazonS3Client s3Client = EasyMock.createStrictMock(AmazonS3Client.class); - Capture capturedS3Object = Capture.newInstance(); + final AccessControlList acl = new AccessControlList(); + acl.setOwner(new Owner("ownerId", "owner")); + acl.grantAllPermissions(new Grant(new CanonicalGrantee(acl.getOwner().getId()), Permission.FullControl)); + EasyMock.expect(s3Client.getBucketAcl(EasyMock.eq("bucket"))).andReturn(acl).once(); + + EasyMock.expect(s3Client.putObject(EasyMock.anyObject())) + .andReturn(new PutObjectResult()) + .once(); + + EasyMock.expect(s3Client.getBucketAcl(EasyMock.eq("bucket"))).andReturn(acl).once(); + + Capture capturedPutRequest = Capture.newInstance(); ValueContainer capturedS3SegmentJson = new ValueContainer<>(); - EasyMock.expect(s3Client.putObject(EasyMock.anyString(), EasyMock.capture(capturedS3Object))) + EasyMock.expect(s3Client.putObject(EasyMock.capture(capturedPutRequest))) .andAnswer( - new IAnswer() + new IAnswer() { @Override - public S3Object answer() throws Throwable + public PutObjectResult answer() throws Throwable { capturedS3SegmentJson.setValue( - IOUtils.toString(capturedS3Object.getValue().getDataInputStream(), "utf-8") + IOUtils.toString(new FileInputStream(capturedPutRequest.getValue().getFile()), "utf-8") ); - return null; + return new PutObjectResult(); } } ) - .atLeastOnce(); - EasyMock.replay(s3Client); + .once(); + EasyMock.replay(s3Client); S3DataSegmentPusherConfig config = new S3DataSegmentPusherConfig(); config.setBucket("bucket"); diff --git a/extensions-core/s3-extensions/src/test/java/io/druid/storage/s3/S3TimestampVersionedDataFinderTest.java b/extensions-core/s3-extensions/src/test/java/io/druid/storage/s3/S3TimestampVersionedDataFinderTest.java index c2bdc347334d..ce19c9b10fc7 100644 --- a/extensions-core/s3-extensions/src/test/java/io/druid/storage/s3/S3TimestampVersionedDataFinderTest.java +++ b/extensions-core/s3-extensions/src/test/java/io/druid/storage/s3/S3TimestampVersionedDataFinderTest.java @@ -19,11 +19,12 @@ package io.druid.storage.s3; +import com.amazonaws.services.s3.AmazonS3Client; +import com.amazonaws.services.s3.model.ListObjectsV2Request; +import com.amazonaws.services.s3.model.ListObjectsV2Result; +import com.amazonaws.services.s3.model.S3ObjectSummary; import io.druid.java.util.common.StringUtils; import org.easymock.EasyMock; -import org.jets3t.service.S3ServiceException; -import org.jets3t.service.impl.rest.httpclient.RestS3Service; -import org.jets3t.service.model.S3Object; import org.junit.Assert; import org.junit.Test; @@ -35,25 +36,31 @@ public class S3TimestampVersionedDataFinderTest { @Test - public void testSimpleLatestVersion() throws S3ServiceException + public void testSimpleLatestVersion() { String bucket = "bucket"; String keyPrefix = "prefix/dir/0"; - RestS3Service s3Client = EasyMock.createStrictMock(RestS3Service.class); + AmazonS3Client s3Client = EasyMock.createStrictMock(AmazonS3Client.class); - S3Object object0 = new S3Object(), object1 = new S3Object(); + S3ObjectSummary object0 = new S3ObjectSummary(), object1 = new S3ObjectSummary(); object0.setBucketName(bucket); object0.setKey(keyPrefix + "/renames-0.gz"); - object0.setLastModifiedDate(new Date(0)); + object0.setLastModified(new Date(0)); object1.setBucketName(bucket); object1.setKey(keyPrefix + "/renames-1.gz"); - object1.setLastModifiedDate(new Date(1)); + object1.setLastModified(new Date(1)); - EasyMock.expect(s3Client.listObjects(EasyMock.eq(bucket), EasyMock.anyString(), EasyMock.isNull())).andReturn( - new S3Object[]{object0, object1} - ).once(); + final ListObjectsV2Result result = new ListObjectsV2Result(); + result.getObjectSummaries().add(object0); + result.getObjectSummaries().add(object1); + result.setKeyCount(2); + result.setTruncated(false); + + EasyMock.expect(s3Client.listObjectsV2(EasyMock.anyObject(ListObjectsV2Request.class))) + .andReturn(result) + .once(); S3TimestampVersionedDataFinder finder = new S3TimestampVersionedDataFinder(s3Client); Pattern pattern = Pattern.compile("renames-[0-9]*\\.gz"); @@ -71,25 +78,19 @@ public void testSimpleLatestVersion() throws S3ServiceException } @Test - public void testMissing() throws S3ServiceException + public void testMissing() { String bucket = "bucket"; String keyPrefix = "prefix/dir/0"; - RestS3Service s3Client = EasyMock.createStrictMock(RestS3Service.class); + AmazonS3Client s3Client = EasyMock.createStrictMock(AmazonS3Client.class); - S3Object object0 = new S3Object(), object1 = new S3Object(); + final ListObjectsV2Result result = new ListObjectsV2Result(); + result.setKeyCount(0); + result.setTruncated(false); - object0.setBucketName(bucket); - object0.setKey(keyPrefix + "/renames-0.gz"); - object0.setLastModifiedDate(new Date(0)); - - object1.setBucketName(bucket); - object1.setKey(keyPrefix + "/renames-1.gz"); - object1.setLastModifiedDate(new Date(1)); - - EasyMock.expect(s3Client.listObjects(EasyMock.eq(bucket), EasyMock.anyString(), EasyMock.isNull())).andReturn( - null - ).once(); + EasyMock.expect(s3Client.listObjectsV2(EasyMock.anyObject(ListObjectsV2Request.class))) + .andReturn(result) + .once(); S3TimestampVersionedDataFinder finder = new S3TimestampVersionedDataFinder(s3Client); Pattern pattern = Pattern.compile("renames-[0-9]*\\.gz"); @@ -105,21 +106,26 @@ public void testMissing() throws S3ServiceException } @Test - public void testFindSelf() throws S3ServiceException + public void testFindSelf() { String bucket = "bucket"; String keyPrefix = "prefix/dir/0"; - RestS3Service s3Client = EasyMock.createStrictMock(RestS3Service.class); + AmazonS3Client s3Client = EasyMock.createStrictMock(AmazonS3Client.class); - S3Object object0 = new S3Object(); + S3ObjectSummary object0 = new S3ObjectSummary(); object0.setBucketName(bucket); object0.setKey(keyPrefix + "/renames-0.gz"); - object0.setLastModifiedDate(new Date(0)); + object0.setLastModified(new Date(0)); - EasyMock.expect(s3Client.listObjects(EasyMock.eq(bucket), EasyMock.anyString(), EasyMock.isNull())).andReturn( - new S3Object[]{object0} - ).once(); + final ListObjectsV2Result result = new ListObjectsV2Result(); + result.getObjectSummaries().add(object0); + result.setKeyCount(1); + result.setTruncated(false); + + EasyMock.expect(s3Client.listObjectsV2(EasyMock.anyObject(ListObjectsV2Request.class))) + .andReturn(result) + .once(); S3TimestampVersionedDataFinder finder = new S3TimestampVersionedDataFinder(s3Client); Pattern pattern = Pattern.compile("renames-[0-9]*\\.gz"); @@ -137,21 +143,26 @@ public void testFindSelf() throws S3ServiceException } @Test - public void testFindExact() throws S3ServiceException + public void testFindExact() { String bucket = "bucket"; String keyPrefix = "prefix/dir/0"; - RestS3Service s3Client = EasyMock.createStrictMock(RestS3Service.class); + AmazonS3Client s3Client = EasyMock.createStrictMock(AmazonS3Client.class); - S3Object object0 = new S3Object(); + S3ObjectSummary object0 = new S3ObjectSummary(); object0.setBucketName(bucket); object0.setKey(keyPrefix + "/renames-0.gz"); - object0.setLastModifiedDate(new Date(0)); + object0.setLastModified(new Date(0)); + + final ListObjectsV2Result result = new ListObjectsV2Result(); + result.getObjectSummaries().add(object0); + result.setKeyCount(1); + result.setTruncated(false); - EasyMock.expect(s3Client.listObjects(EasyMock.eq(bucket), EasyMock.anyString(), EasyMock.isNull())).andReturn( - new S3Object[]{object0} - ).once(); + EasyMock.expect(s3Client.listObjectsV2(EasyMock.anyObject(ListObjectsV2Request.class))) + .andReturn(result) + .once(); S3TimestampVersionedDataFinder finder = new S3TimestampVersionedDataFinder(s3Client); diff --git a/extensions-core/s3-extensions/src/test/java/io/druid/storage/s3/TestAWSCredentialsProvider.java b/extensions-core/s3-extensions/src/test/java/io/druid/storage/s3/TestAWSCredentialsProvider.java index a3bf27a40b47..db97c424bc18 100644 --- a/extensions-core/s3-extensions/src/test/java/io/druid/storage/s3/TestAWSCredentialsProvider.java +++ b/extensions-core/s3-extensions/src/test/java/io/druid/storage/s3/TestAWSCredentialsProvider.java @@ -58,7 +58,7 @@ public void testWithFixedAWSKeys() assertEquals(credentials.getAWSSecretKey(), "secretKeySample"); // try to create - s3Module.getRestS3Service(provider); + s3Module.getAmazonS3Client(provider); } @Rule @@ -86,6 +86,6 @@ public void testWithFileSessionCredentials() throws IOException assertEquals(sessionCredentials.getSessionToken(), "sessionTokenSample"); // try to create - s3Module.getRestS3Service(provider); + s3Module.getAmazonS3Client(provider); } } diff --git a/indexing-hadoop/pom.xml b/indexing-hadoop/pom.xml index e2bf002c3d55..75499854cf1e 100644 --- a/indexing-hadoop/pom.xml +++ b/indexing-hadoop/pom.xml @@ -56,13 +56,11 @@ com.google.guava guava - - net.java.dev.jets3t - jets3t + com.amazonaws + aws-java-sdk-s3 test - org.apache.httpcomponents httpclient diff --git a/indexing-hadoop/src/main/java/io/druid/indexer/IndexGeneratorJob.java b/indexing-hadoop/src/main/java/io/druid/indexer/IndexGeneratorJob.java index e094809cf3be..8040eef57666 100644 --- a/indexing-hadoop/src/main/java/io/druid/indexer/IndexGeneratorJob.java +++ b/indexing-hadoop/src/main/java/io/druid/indexer/IndexGeneratorJob.java @@ -73,6 +73,7 @@ import org.joda.time.DateTime; import org.joda.time.Interval; +import java.io.DataInput; import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; @@ -113,7 +114,7 @@ public static List getPublishedSegments(HadoopDruidIndexerConfig co FileSystem fs = descriptorInfoDir.getFileSystem(conf); for (FileStatus status : fs.listStatus(descriptorInfoDir)) { - final DataSegment segment = jsonMapper.readValue(fs.open(status.getPath()), DataSegment.class); + final DataSegment segment = jsonMapper.readValue((DataInput) fs.open(status.getPath()), DataSegment.class); publishedSegmentsBuilder.add(segment); log.info("Adding segment %s to the list of published segments", segment.getIdentifier()); } diff --git a/indexing-service/src/main/java/io/druid/indexing/common/config/TaskConfig.java b/indexing-service/src/main/java/io/druid/indexing/common/config/TaskConfig.java index 6a9370324d0a..fcf8aa3f4f63 100644 --- a/indexing-service/src/main/java/io/druid/indexing/common/config/TaskConfig.java +++ b/indexing-service/src/main/java/io/druid/indexing/common/config/TaskConfig.java @@ -31,7 +31,7 @@ public class TaskConfig { public static final List DEFAULT_DEFAULT_HADOOP_COORDINATES = ImmutableList.of( - "org.apache.hadoop:hadoop-client:2.7.3" + "org.apache.hadoop:hadoop-client:2.9.0" ); private static final Period DEFAULT_DIRECTORY_LOCK_TIMEOUT = new Period("PT10M"); diff --git a/pom.xml b/pom.xml index 3aa4cd472c63..a18262b6b418 100644 --- a/pom.xml +++ b/pom.xml @@ -68,8 +68,7 @@ 4.1.0 9.3.19.v20170502 1.19.3 - - 2.4.6 + 2.8.10 2.5 3.10.6.Final @@ -78,12 +77,11 @@ 4.0.52.Final 1.7.12 - 2.7.3 + 2.9.0 2.0.0 1.6.6 - - 1.10.77 + 1.11.29 + 1.11.199 2.5.5 3.4.11 @@ -192,12 +190,25 @@ com.fasterxml.jackson.core jackson-annotations + + com.fasterxml.jackson.dataformat + jackson-dataformat-cbor + commons-codec commons-codec + + aws-java-sdk-core + com.amazonaws + + + com.amazonaws + aws-java-sdk-bundle + ${aws.sdk.bundle.version} + com.amazonaws aws-java-sdk-s3 @@ -215,10 +226,18 @@ com.fasterxml.jackson.core jackson-annotations + + com.fasterxml.jackson.dataformat + jackson-dataformat-cbor + commons-codec commons-codec + + aws-java-sdk-core + com.amazonaws + @@ -595,49 +614,15 @@ aether-api 0.9.0.M2 - - net.java.dev.jets3t - jets3t - 0.9.4 - - - commons-codec - commons-codec - - - commons-logging - commons-logging - - - org.apache.httpcomponents - httpclient - - - org.apache.httpcomponents - httpcore - - - org.codehaus.jackson - jackson-core-asl - - - org.codehaus.jackson - jackson-mapper-asl - - - - - org.apache.httpcomponents httpclient - 4.5.1 + 4.5.3 org.apache.httpcomponents httpcore - 4.4.3 + 4.4.4 org.apache.hadoop diff --git a/server/src/main/java/io/druid/segment/realtime/firehose/HttpFirehoseFactory.java b/server/src/main/java/io/druid/segment/realtime/firehose/HttpFirehoseFactory.java index cf3ecea6fba2..aaab6f9dae55 100644 --- a/server/src/main/java/io/druid/segment/realtime/firehose/HttpFirehoseFactory.java +++ b/server/src/main/java/io/druid/segment/realtime/firehose/HttpFirehoseFactory.java @@ -23,11 +23,11 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Preconditions; import com.google.common.base.Predicate; +import com.google.common.net.HttpHeaders; import io.druid.data.input.impl.prefetch.PrefetchableTextFilesFirehoseFactory; import io.druid.java.util.common.CompressionUtils; import io.druid.java.util.common.StringUtils; import io.druid.java.util.common.logger.Logger; -import org.apache.http.HttpHeaders; import java.io.IOException; import java.io.InputStream; diff --git a/server/src/main/java/io/druid/server/AsyncQueryForwardingServlet.java b/server/src/main/java/io/druid/server/AsyncQueryForwardingServlet.java index 689f3f7456d1..f93706784ac7 100644 --- a/server/src/main/java/io/druid/server/AsyncQueryForwardingServlet.java +++ b/server/src/main/java/io/druid/server/AsyncQueryForwardingServlet.java @@ -19,6 +19,7 @@ package io.druid.server; +import com.amazonaws.thirdparty.apache.http.client.utils.URIBuilder; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.jaxrs.smile.SmileMediaTypes; @@ -26,14 +27,14 @@ import com.google.common.collect.ImmutableMap; import com.google.inject.Inject; import com.google.inject.Provider; -import io.druid.java.util.emitter.EmittingLogger; -import io.druid.java.util.emitter.service.ServiceEmitter; import io.druid.client.selector.Server; import io.druid.guice.annotations.Json; import io.druid.guice.annotations.Smile; import io.druid.guice.http.DruidHttpClientConfig; import io.druid.java.util.common.DateTimes; import io.druid.java.util.common.IAE; +import io.druid.java.util.emitter.EmittingLogger; +import io.druid.java.util.emitter.service.ServiceEmitter; import io.druid.query.DruidMetrics; import io.druid.query.GenericQueryMetricsFactory; import io.druid.query.Query; @@ -44,7 +45,6 @@ import io.druid.server.router.QueryHostFinder; import io.druid.server.router.Router; import io.druid.server.security.AuthConfig; -import org.apache.http.client.utils.URIBuilder; import org.eclipse.jetty.client.HttpClient; import org.eclipse.jetty.client.api.Request; import org.eclipse.jetty.client.api.Response; @@ -57,6 +57,7 @@ import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response.Status; import java.io.IOException; import java.net.URI; import java.net.URISyntaxException; @@ -444,13 +445,11 @@ public void onComplete(Result result) TimeUnit.NANOSECONDS.toMillis(requestTimeNs), "success", success - && result.getResponse().getStatus() == javax.ws.rs.core.Response.Status.OK.getStatusCode() + && result.getResponse().getStatus() == Status.OK.getStatusCode() ) ) ) ); - - } catch (Exception e) { log.error(e, "Unable to log query [%s]!", query); diff --git a/server/src/main/java/io/druid/server/initialization/jetty/JettyServerModule.java b/server/src/main/java/io/druid/server/initialization/jetty/JettyServerModule.java index 4eb92b253edd..a0302f1f3eaa 100644 --- a/server/src/main/java/io/druid/server/initialization/jetty/JettyServerModule.java +++ b/server/src/main/java/io/druid/server/initialization/jetty/JettyServerModule.java @@ -60,7 +60,7 @@ import io.druid.server.metrics.DataSourceTaskIdHolder; import io.druid.server.metrics.MetricsModule; import io.druid.server.metrics.MonitorsConfig; -import org.apache.http.HttpVersion; +import io.netty.handler.codec.http.HttpVersion; import org.eclipse.jetty.server.ConnectionFactory; import org.eclipse.jetty.server.Handler; import org.eclipse.jetty.server.HttpConfiguration; From 19f502120d52b690bb43eb2ffa62fd8e04a8d830 Mon Sep 17 00:00:00 2001 From: Jihoon Son Date: Mon, 12 Feb 2018 13:42:55 -0800 Subject: [PATCH 02/21] fix compile and serde tests --- .../input/impl/InputRowParserSerdeTest.java | 8 ++--- .../data/input/impl/JSONParseSpecTest.java | 4 +-- .../input/impl/JavaScriptParseSpecTest.java | 4 +-- .../data/input/impl/RegexParseSpecTest.java | 4 +-- .../storage/s3/S3DataSegmentFinderTest.java | 1 + .../groupby/orderby/DefaultLimitSpecTest.java | 30 +++++++++---------- .../topn/AlphaNumericTopNMetricSpecTest.java | 4 +-- .../topn/DimensionTopNMetricSpecTest.java | 16 +++++----- .../dimension/LookupDimensionSpecTest.java | 4 ++- 9 files changed, 39 insertions(+), 36 deletions(-) diff --git a/api/src/test/java/io/druid/data/input/impl/InputRowParserSerdeTest.java b/api/src/test/java/io/druid/data/input/impl/InputRowParserSerdeTest.java index 1a5d59f2f612..c464d65a5926 100644 --- a/api/src/test/java/io/druid/data/input/impl/InputRowParserSerdeTest.java +++ b/api/src/test/java/io/druid/data/input/impl/InputRowParserSerdeTest.java @@ -97,9 +97,9 @@ public void testMapInputRowParserSerde() throws Exception null ) ); - final MapInputRowParser parser2 = jsonMapper.readValue( + final MapInputRowParser parser2 = (MapInputRowParser) jsonMapper.readValue( jsonMapper.writeValueAsBytes(parser), - MapInputRowParser.class + InputRowParser.class ); final InputRow parsed = parser2.parseBatch( ImmutableMap.of( @@ -126,9 +126,9 @@ public void testMapInputRowParserNumbersSerde() throws Exception null ) ); - final MapInputRowParser parser2 = jsonMapper.readValue( + final MapInputRowParser parser2 = (MapInputRowParser) jsonMapper.readValue( jsonMapper.writeValueAsBytes(parser), - MapInputRowParser.class + InputRowParser.class ); final InputRow parsed = parser2.parseBatch( ImmutableMap.of( diff --git a/api/src/test/java/io/druid/data/input/impl/JSONParseSpecTest.java b/api/src/test/java/io/druid/data/input/impl/JSONParseSpecTest.java index de2814eda9a7..c7c73802f73d 100644 --- a/api/src/test/java/io/druid/data/input/impl/JSONParseSpecTest.java +++ b/api/src/test/java/io/druid/data/input/impl/JSONParseSpecTest.java @@ -91,9 +91,9 @@ public void testSerde() throws IOException feature ); - final JSONParseSpec serde = jsonMapper.readValue( + final JSONParseSpec serde = (JSONParseSpec) jsonMapper.readValue( jsonMapper.writeValueAsString(spec), - JSONParseSpec.class + ParseSpec.class ); Assert.assertEquals("timestamp", serde.getTimestampSpec().getTimestampColumn()); Assert.assertEquals("iso", serde.getTimestampSpec().getTimestampFormat()); diff --git a/api/src/test/java/io/druid/data/input/impl/JavaScriptParseSpecTest.java b/api/src/test/java/io/druid/data/input/impl/JavaScriptParseSpecTest.java index 805e019b1b08..b63caf43ff33 100644 --- a/api/src/test/java/io/druid/data/input/impl/JavaScriptParseSpecTest.java +++ b/api/src/test/java/io/druid/data/input/impl/JavaScriptParseSpecTest.java @@ -58,9 +58,9 @@ public void testSerde() throws IOException "abc", JavaScriptConfig.getEnabledInstance() ); - final JavaScriptParseSpec serde = jsonMapper.readValue( + final JavaScriptParseSpec serde = (JavaScriptParseSpec) jsonMapper.readValue( jsonMapper.writeValueAsString(spec), - JavaScriptParseSpec.class + ParseSpec.class ); Assert.assertEquals("abc", serde.getTimestampSpec().getTimestampColumn()); Assert.assertEquals("iso", serde.getTimestampSpec().getTimestampFormat()); diff --git a/api/src/test/java/io/druid/data/input/impl/RegexParseSpecTest.java b/api/src/test/java/io/druid/data/input/impl/RegexParseSpecTest.java index 68930ea6269d..5468ae0302f9 100644 --- a/api/src/test/java/io/druid/data/input/impl/RegexParseSpecTest.java +++ b/api/src/test/java/io/druid/data/input/impl/RegexParseSpecTest.java @@ -43,9 +43,9 @@ public void testSerde() throws IOException Collections.singletonList("abc"), "abc" ); - final RegexParseSpec serde = jsonMapper.readValue( + final RegexParseSpec serde = (RegexParseSpec) jsonMapper.readValue( jsonMapper.writeValueAsString(spec), - RegexParseSpec.class + ParseSpec.class ); Assert.assertEquals("abc", serde.getTimestampSpec().getTimestampColumn()); Assert.assertEquals("iso", serde.getTimestampSpec().getTimestampFormat()); diff --git a/extensions-core/s3-extensions/src/test/java/io/druid/storage/s3/S3DataSegmentFinderTest.java b/extensions-core/s3-extensions/src/test/java/io/druid/storage/s3/S3DataSegmentFinderTest.java index 08b2f40c35d3..5c449faf2e10 100644 --- a/extensions-core/s3-extensions/src/test/java/io/druid/storage/s3/S3DataSegmentFinderTest.java +++ b/extensions-core/s3-extensions/src/test/java/io/druid/storage/s3/S3DataSegmentFinderTest.java @@ -411,6 +411,7 @@ public S3Object getObject(String bucketName, String objectKey) return storageObject; } + @Override public PutObjectResult putObject(String bucketName, String key, String data) { return putObject(bucketName, key, new ByteArrayInputStream(StringUtils.toUtf8(data)), null); diff --git a/processing/src/test/java/io/druid/query/groupby/orderby/DefaultLimitSpecTest.java b/processing/src/test/java/io/druid/query/groupby/orderby/DefaultLimitSpecTest.java index bb00b1fc0384..43f2e5805c0d 100644 --- a/processing/src/test/java/io/druid/query/groupby/orderby/DefaultLimitSpecTest.java +++ b/processing/src/test/java/io/druid/query/groupby/orderby/DefaultLimitSpecTest.java @@ -72,9 +72,9 @@ public void testSerde() throws Exception //defaults String json = "{\"type\": \"default\"}"; - DefaultLimitSpec spec = mapper.readValue( - mapper.writeValueAsString(mapper.readValue(json, DefaultLimitSpec.class)), - DefaultLimitSpec.class + DefaultLimitSpec spec = (DefaultLimitSpec) mapper.readValue( + mapper.writeValueAsString(mapper.readValue(json, LimitSpec.class)), + LimitSpec.class ); Assert.assertEquals( @@ -88,9 +88,9 @@ public void testSerde() throws Exception + " \"columns\":[{\"dimension\":\"d\",\"direction\":\"DESCENDING\", \"dimensionOrder\":\"numeric\"}],\n" + " \"limit\":10\n" + "}"; - spec = mapper.readValue( - mapper.writeValueAsString(mapper.readValue(json, DefaultLimitSpec.class)), - DefaultLimitSpec.class + spec = (DefaultLimitSpec) mapper.readValue( + mapper.writeValueAsString(mapper.readValue(json, LimitSpec.class)), + LimitSpec.class ); Assert.assertEquals( new DefaultLimitSpec(ImmutableList.of(new OrderByColumnSpec("d", OrderByColumnSpec.Direction.DESCENDING, @@ -104,9 +104,9 @@ public void testSerde() throws Exception + " \"limit\":10\n" + "}"; - spec = mapper.readValue( - mapper.writeValueAsString(mapper.readValue(json, DefaultLimitSpec.class)), - DefaultLimitSpec.class + spec = (DefaultLimitSpec) mapper.readValue( + mapper.writeValueAsString(mapper.readValue(json, LimitSpec.class)), + LimitSpec.class ); Assert.assertEquals( @@ -120,9 +120,9 @@ public void testSerde() throws Exception + " \"columns\":[{\"dimension\":\"d\"}],\n" + " \"limit\":10\n" + "}"; - spec = mapper.readValue( - mapper.writeValueAsString(mapper.readValue(json, DefaultLimitSpec.class)), - DefaultLimitSpec.class + spec = (DefaultLimitSpec) mapper.readValue( + mapper.writeValueAsString(mapper.readValue(json, LimitSpec.class)), + LimitSpec.class ); Assert.assertEquals( new DefaultLimitSpec(ImmutableList.of(new OrderByColumnSpec("d", OrderByColumnSpec.Direction.ASCENDING, @@ -135,9 +135,9 @@ public void testSerde() throws Exception + " \"columns\":[\"d\"],\n" + " \"limit\":10\n" + "}"; - spec = mapper.readValue( - mapper.writeValueAsString(mapper.readValue(json, DefaultLimitSpec.class)), - DefaultLimitSpec.class + spec = (DefaultLimitSpec) mapper.readValue( + mapper.writeValueAsString(mapper.readValue(json, LimitSpec.class)), + LimitSpec.class ); Assert.assertEquals( new DefaultLimitSpec(ImmutableList.of(new OrderByColumnSpec("d", OrderByColumnSpec.Direction.ASCENDING, diff --git a/processing/src/test/java/io/druid/query/topn/AlphaNumericTopNMetricSpecTest.java b/processing/src/test/java/io/druid/query/topn/AlphaNumericTopNMetricSpecTest.java index c88338d4c1d5..163de69c2172 100644 --- a/processing/src/test/java/io/druid/query/topn/AlphaNumericTopNMetricSpecTest.java +++ b/processing/src/test/java/io/druid/query/topn/AlphaNumericTopNMetricSpecTest.java @@ -104,8 +104,8 @@ public void testSerdeAlphaNumericTopNMetricSpec() throws IOException + " \"previousStop\": \"test\"\n" + "}"; ObjectMapper jsonMapper = TestHelper.makeJsonMapper(); - TopNMetricSpec actualMetricSpec = jsonMapper.readValue(jsonMapper.writeValueAsString(jsonMapper.readValue(jsonSpec, TopNMetricSpec.class)), AlphaNumericTopNMetricSpec.class); - TopNMetricSpec actualMetricSpec1 = jsonMapper.readValue(jsonMapper.writeValueAsString(jsonMapper.readValue(jsonSpec1, TopNMetricSpec.class)), AlphaNumericTopNMetricSpec.class); + TopNMetricSpec actualMetricSpec = jsonMapper.readValue(jsonMapper.writeValueAsString(jsonMapper.readValue(jsonSpec, TopNMetricSpec.class)), TopNMetricSpec.class); + TopNMetricSpec actualMetricSpec1 = jsonMapper.readValue(jsonMapper.writeValueAsString(jsonMapper.readValue(jsonSpec1, TopNMetricSpec.class)), TopNMetricSpec.class); Assert.assertEquals(expectedMetricSpec, actualMetricSpec); Assert.assertEquals(expectedMetricSpec1, actualMetricSpec1); } diff --git a/processing/src/test/java/io/druid/query/topn/DimensionTopNMetricSpecTest.java b/processing/src/test/java/io/druid/query/topn/DimensionTopNMetricSpecTest.java index 292618ac821c..7091ef5ee93a 100644 --- a/processing/src/test/java/io/druid/query/topn/DimensionTopNMetricSpecTest.java +++ b/processing/src/test/java/io/druid/query/topn/DimensionTopNMetricSpecTest.java @@ -44,8 +44,8 @@ public void testSerdeAlphaNumericDimensionTopNMetricSpec() throws IOException + " \"previousStop\": \"test\"\n" + "}"; ObjectMapper jsonMapper = TestHelper.makeJsonMapper(); - TopNMetricSpec actualMetricSpec = jsonMapper.readValue(jsonMapper.writeValueAsString(jsonMapper.readValue(jsonSpec, TopNMetricSpec.class)), DimensionTopNMetricSpec.class); - TopNMetricSpec actualMetricSpec1 = jsonMapper.readValue(jsonMapper.writeValueAsString(jsonMapper.readValue(jsonSpec1, TopNMetricSpec.class)), DimensionTopNMetricSpec.class); + TopNMetricSpec actualMetricSpec = jsonMapper.readValue(jsonMapper.writeValueAsString(jsonMapper.readValue(jsonSpec, TopNMetricSpec.class)), TopNMetricSpec.class); + TopNMetricSpec actualMetricSpec1 = jsonMapper.readValue(jsonMapper.writeValueAsString(jsonMapper.readValue(jsonSpec1, TopNMetricSpec.class)), TopNMetricSpec.class); Assert.assertEquals(expectedMetricSpec, actualMetricSpec); Assert.assertEquals(expectedMetricSpec1, actualMetricSpec1); } @@ -65,8 +65,8 @@ public void testSerdeLexicographicDimensionTopNMetricSpec() throws IOException + " \"previousStop\": \"test\"\n" + "}"; ObjectMapper jsonMapper = TestHelper.makeJsonMapper(); - TopNMetricSpec actualMetricSpec = jsonMapper.readValue(jsonMapper.writeValueAsString(jsonMapper.readValue(jsonSpec, TopNMetricSpec.class)), DimensionTopNMetricSpec.class); - TopNMetricSpec actualMetricSpec1 = jsonMapper.readValue(jsonMapper.writeValueAsString(jsonMapper.readValue(jsonSpec1, TopNMetricSpec.class)), DimensionTopNMetricSpec.class); + TopNMetricSpec actualMetricSpec = jsonMapper.readValue(jsonMapper.writeValueAsString(jsonMapper.readValue(jsonSpec, TopNMetricSpec.class)), TopNMetricSpec.class); + TopNMetricSpec actualMetricSpec1 = jsonMapper.readValue(jsonMapper.writeValueAsString(jsonMapper.readValue(jsonSpec1, TopNMetricSpec.class)), TopNMetricSpec.class); Assert.assertEquals(expectedMetricSpec, actualMetricSpec); Assert.assertEquals(expectedMetricSpec1, actualMetricSpec1); } @@ -86,8 +86,8 @@ public void testSerdeStrlenDimensionTopNMetricSpec() throws IOException + " \"previousStop\": \"test\"\n" + "}"; ObjectMapper jsonMapper = TestHelper.makeJsonMapper(); - TopNMetricSpec actualMetricSpec = jsonMapper.readValue(jsonMapper.writeValueAsString(jsonMapper.readValue(jsonSpec, TopNMetricSpec.class)), DimensionTopNMetricSpec.class); - TopNMetricSpec actualMetricSpec1 = jsonMapper.readValue(jsonMapper.writeValueAsString(jsonMapper.readValue(jsonSpec1, TopNMetricSpec.class)), DimensionTopNMetricSpec.class); + TopNMetricSpec actualMetricSpec = jsonMapper.readValue(jsonMapper.writeValueAsString(jsonMapper.readValue(jsonSpec, TopNMetricSpec.class)), TopNMetricSpec.class); + TopNMetricSpec actualMetricSpec1 = jsonMapper.readValue(jsonMapper.writeValueAsString(jsonMapper.readValue(jsonSpec1, TopNMetricSpec.class)), TopNMetricSpec.class); Assert.assertEquals(expectedMetricSpec, actualMetricSpec); Assert.assertEquals(expectedMetricSpec1, actualMetricSpec1); } @@ -107,8 +107,8 @@ public void testSerdeNumericDimensionTopNMetricSpec() throws IOException + " \"previousStop\": \"test\"\n" + "}"; ObjectMapper jsonMapper = TestHelper.makeJsonMapper(); - TopNMetricSpec actualMetricSpec = jsonMapper.readValue(jsonMapper.writeValueAsString(jsonMapper.readValue(jsonSpec, TopNMetricSpec.class)), DimensionTopNMetricSpec.class); - TopNMetricSpec actualMetricSpec1 = jsonMapper.readValue(jsonMapper.writeValueAsString(jsonMapper.readValue(jsonSpec1, TopNMetricSpec.class)), DimensionTopNMetricSpec.class); + TopNMetricSpec actualMetricSpec = jsonMapper.readValue(jsonMapper.writeValueAsString(jsonMapper.readValue(jsonSpec, TopNMetricSpec.class)), TopNMetricSpec.class); + TopNMetricSpec actualMetricSpec1 = jsonMapper.readValue(jsonMapper.writeValueAsString(jsonMapper.readValue(jsonSpec1, TopNMetricSpec.class)), TopNMetricSpec.class); Assert.assertEquals(expectedMetricSpec, actualMetricSpec); Assert.assertEquals(expectedMetricSpec1, actualMetricSpec1); } diff --git a/server/src/test/java/io/druid/query/dimension/LookupDimensionSpecTest.java b/server/src/test/java/io/druid/query/dimension/LookupDimensionSpecTest.java index 7e1ef423f959..02c44c297253 100644 --- a/server/src/test/java/io/druid/query/dimension/LookupDimensionSpecTest.java +++ b/server/src/test/java/io/druid/query/dimension/LookupDimensionSpecTest.java @@ -21,6 +21,7 @@ import com.fasterxml.jackson.databind.InjectableValues; import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.jsontype.NamedType; import com.google.common.base.Strings; import com.google.common.collect.ImmutableMap; import io.druid.jackson.DefaultObjectMapper; @@ -70,12 +71,13 @@ public class LookupDimensionSpecTest public void testSerDesr(DimensionSpec lookupDimSpec) throws IOException { ObjectMapper mapper = new DefaultObjectMapper(); + mapper.registerSubtypes(new NamedType(LookupDimensionSpec.class, "lookup")); InjectableValues injectableValues = new InjectableValues.Std().addValue( LookupReferencesManager.class, LOOKUP_REF_MANAGER ); String serLookup = mapper.writeValueAsString(lookupDimSpec); - Assert.assertEquals(lookupDimSpec, mapper.reader(LookupDimensionSpec.class).with(injectableValues).readValue(serLookup)); + Assert.assertEquals(lookupDimSpec, mapper.reader(DimensionSpec.class).with(injectableValues).readValue(serLookup)); } private Object[] parametersForTestSerDesr() From dbc945c10fe56042ba3d70840b92456ab1ba2796 Mon Sep 17 00:00:00 2001 From: Jihoon Son Date: Tue, 13 Feb 2018 18:48:11 -0800 Subject: [PATCH 03/21] address comments and fix test --- .../druid/storage/s3/S3DataSegmentMover.java | 10 ++-- .../druid/storage/s3/S3DataSegmentPuller.java | 57 ++++--------------- .../druid/storage/s3/S3DataSegmentPusher.java | 2 +- .../s3/S3TimestampVersionedDataFinder.java | 1 - .../java/io/druid/storage/s3/S3Utils.java | 11 +++- .../autoscaling/EC2AutoScalerSerdeTest.java | 6 +- .../JavaScriptWorkerSelectStrategyTest.java | 2 +- .../segment/indexing/DataSchemaTest.java | 2 +- 8 files changed, 32 insertions(+), 59 deletions(-) diff --git a/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3DataSegmentMover.java b/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3DataSegmentMover.java index 62df758c55b0..ea11d95a2da3 100644 --- a/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3DataSegmentMover.java +++ b/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3DataSegmentMover.java @@ -178,10 +178,12 @@ private void selfCheckingMove( final S3ObjectSummary objectSummary = listResult.getObjectSummaries().get(0); if (objectSummary.getStorageClass() != null && StorageClass.fromValue(StringUtils.toUpperCase(objectSummary.getStorageClass())).equals(StorageClass.Glacier)) { - throw new ISE( - "Cannot move file[s3://%s/%s] of storage class glacier, skipping.", - s3Bucket, - s3Path + throw new AmazonServiceException( + StringUtils.format( + "Cannot move file[s3://%s/%s] of storage class glacier, skipping.", + s3Bucket, + s3Path + ) ); } else { log.info("Moving file %s", copyMsg); diff --git a/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3DataSegmentPuller.java b/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3DataSegmentPuller.java index dee3c5c2978c..3aba36a5be75 100644 --- a/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3DataSegmentPuller.java +++ b/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3DataSegmentPuller.java @@ -38,6 +38,7 @@ import io.druid.java.util.common.RE; import io.druid.java.util.common.StringUtils; import io.druid.java.util.common.UOE; +import io.druid.java.util.common.io.Closer; import io.druid.java.util.common.logger.Logger; import io.druid.segment.loading.DataSegmentPuller; import io.druid.segment.loading.SegmentLoadingException; @@ -46,6 +47,7 @@ import javax.tools.FileObject; import java.io.File; +import java.io.FilterInputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; @@ -89,6 +91,7 @@ public String getName() public InputStream openInputStream() throws IOException { try { + // TODO: check this method can be called by differernt threads if (s3Object == null) { synchronized (inputStreamOpener) { if (s3Object == null) { @@ -98,57 +101,17 @@ public InputStream openInputStream() throws IOException } } - return new InputStream() - { - final InputStream delegate = s3Object.getObjectContent(); - - @Override - public int read() throws IOException - { - return delegate.read(); - } - - @Override - public int read(byte[] b, int off, int len) throws IOException - { - return delegate.read(b, off, len); - } - - @Override - public long skip(long n) throws IOException - { - return delegate.skip(n); - } - - @Override - public int available() throws IOException - { - return delegate.available(); - } - - @Override - public void reset() throws IOException - { - delegate.reset(); - } + final InputStream in = s3Object.getObjectContent(); + final Closer closer = Closer.create(); + closer.register(in); + closer.register(s3Object); + return new FilterInputStream(in) + { @Override public void close() throws IOException { - delegate.close(); - s3Object.close(); - } - - @Override - public boolean markSupported() - { - return delegate.markSupported(); - } - - @Override - public void mark(int readlimit) - { - delegate.mark(readlimit); + closer.close(); } }; } diff --git a/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3DataSegmentPusher.java b/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3DataSegmentPusher.java index 1d8063610789..e01fe0046fe9 100644 --- a/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3DataSegmentPusher.java +++ b/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3DataSegmentPusher.java @@ -173,7 +173,7 @@ private void uploadFileAndClear(AmazonS3Client s3Client, String bucket, String k s3Client.putObject(indexFilePutRequest); } - log.info("Deleting file [%s]", file.getAbsolutePath()); + log.info("Deleting temporary cached file [%s]", file.getAbsolutePath()); file.delete(); } } diff --git a/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3TimestampVersionedDataFinder.java b/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3TimestampVersionedDataFinder.java index 2401e713b39e..914546cba8fc 100644 --- a/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3TimestampVersionedDataFinder.java +++ b/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3TimestampVersionedDataFinder.java @@ -76,7 +76,6 @@ public URI getLatestVersion(final URI uri, final @Nullable Pattern pattern) ); while (objectSummaryIterator.hasNext()) { final S3ObjectSummary objectSummary = objectSummaryIterator.next(); - // TODO: what is going on here? String keyString = objectSummary.getKey().substring(coords.path.length()); if (keyString.startsWith("/")) { keyString = keyString.substring(1); diff --git a/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3Utils.java b/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3Utils.java index 8b1249810e58..c7b3feb21968 100644 --- a/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3Utils.java +++ b/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3Utils.java @@ -163,7 +163,7 @@ public S3ObjectSummary next() }; } - static String constructSegmentPath(String baseKey, String storageDir) + public static String constructSegmentPath(String baseKey, String storageDir) { return JOINER.join( baseKey.isEmpty() ? null : baseKey, @@ -234,6 +234,15 @@ public static boolean isDirectoryPlaceholder(String key, ObjectMetadata objectMe return false; } + /** + * Gets a single {@link S3ObjectSummary} from s3. Since this method might throw an exception if there are multiple + * objets that match the given key, this method should be used only when it's guaranteed that the given key is unique + * in the given bucket. + * + * @param s3Client s3 client + * @param bucket s3 bucket + * @param key unique key for the object to be retrieved + */ public static S3ObjectSummary getSingleObjectSummary(AmazonS3Client s3Client, String bucket, String key) { final ListObjectsV2Request request = new ListObjectsV2Request() diff --git a/indexing-service/src/test/java/io/druid/indexing/overlord/autoscaling/EC2AutoScalerSerdeTest.java b/indexing-service/src/test/java/io/druid/indexing/overlord/autoscaling/EC2AutoScalerSerdeTest.java index 5d891d62e641..18a46365edb6 100644 --- a/indexing-service/src/test/java/io/druid/indexing/overlord/autoscaling/EC2AutoScalerSerdeTest.java +++ b/indexing-service/src/test/java/io/druid/indexing/overlord/autoscaling/EC2AutoScalerSerdeTest.java @@ -74,12 +74,12 @@ public Object findInjectableValue( } ); - final EC2AutoScaler autoScaler = objectMapper.readValue(json, EC2AutoScaler.class); + final EC2AutoScaler autoScaler = (EC2AutoScaler) objectMapper.readValue(json, AutoScaler.class); verifyAutoScaler(autoScaler); - final EC2AutoScaler roundTripAutoScaler = objectMapper.readValue( + final EC2AutoScaler roundTripAutoScaler = (EC2AutoScaler) objectMapper.readValue( objectMapper.writeValueAsBytes(autoScaler), - EC2AutoScaler.class + AutoScaler.class ); verifyAutoScaler(roundTripAutoScaler); diff --git a/indexing-service/src/test/java/io/druid/indexing/overlord/setup/JavaScriptWorkerSelectStrategyTest.java b/indexing-service/src/test/java/io/druid/indexing/overlord/setup/JavaScriptWorkerSelectStrategyTest.java index 1eb7028a5da4..174f7a85f1fd 100644 --- a/indexing-service/src/test/java/io/druid/indexing/overlord/setup/JavaScriptWorkerSelectStrategyTest.java +++ b/indexing-service/src/test/java/io/druid/indexing/overlord/setup/JavaScriptWorkerSelectStrategyTest.java @@ -86,7 +86,7 @@ public void testSerde() throws Exception STRATEGY, mapper.readValue( mapper.writeValueAsString(STRATEGY), - JavaScriptWorkerSelectStrategy.class + WorkerSelectStrategy.class ) ); } diff --git a/server/src/test/java/io/druid/segment/indexing/DataSchemaTest.java b/server/src/test/java/io/druid/segment/indexing/DataSchemaTest.java index 65df40f60938..e7902256437a 100644 --- a/server/src/test/java/io/druid/segment/indexing/DataSchemaTest.java +++ b/server/src/test/java/io/druid/segment/indexing/DataSchemaTest.java @@ -279,7 +279,7 @@ public void testSerdeWithInvalidParserMap() throws Exception expectedException.expect(CoreMatchers.instanceOf(IllegalArgumentException.class)); expectedException.expectCause(CoreMatchers.instanceOf(JsonMappingException.class)); expectedException.expectMessage( - "Instantiation of [simple type, class io.druid.data.input.impl.StringInputRowParser] value failed: parseSpec" + "Can not construct instance of io.druid.data.input.impl.StringInputRowParser, problem: parseSpec" ); // Jackson creates a default type parser (StringInputRowParser) for an invalid type. From 1e28b1abbda04ed8d05c1cefeb59822e1eba7f8d Mon Sep 17 00:00:00 2001 From: Jihoon Son Date: Wed, 14 Feb 2018 10:30:14 -0800 Subject: [PATCH 04/21] add http version string --- .../initialization/jetty/JettyServerModule.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/server/src/main/java/io/druid/server/initialization/jetty/JettyServerModule.java b/server/src/main/java/io/druid/server/initialization/jetty/JettyServerModule.java index a0302f1f3eaa..346a277aaeda 100644 --- a/server/src/main/java/io/druid/server/initialization/jetty/JettyServerModule.java +++ b/server/src/main/java/io/druid/server/initialization/jetty/JettyServerModule.java @@ -33,10 +33,6 @@ import com.google.inject.Scopes; import com.google.inject.Singleton; import com.google.inject.multibindings.Multibinder; -import io.druid.java.util.emitter.service.ServiceEmitter; -import io.druid.java.util.emitter.service.ServiceMetricEvent; -import io.druid.java.util.metrics.AbstractMonitor; -import io.druid.java.util.metrics.MonitorUtils; import com.sun.jersey.api.core.DefaultResourceConfig; import com.sun.jersey.api.core.ResourceConfig; import com.sun.jersey.guice.JerseyServletModule; @@ -53,6 +49,10 @@ import io.druid.java.util.common.RE; import io.druid.java.util.common.lifecycle.Lifecycle; import io.druid.java.util.common.logger.Logger; +import io.druid.java.util.emitter.service.ServiceEmitter; +import io.druid.java.util.emitter.service.ServiceMetricEvent; +import io.druid.java.util.metrics.AbstractMonitor; +import io.druid.java.util.metrics.MonitorUtils; import io.druid.server.DruidNode; import io.druid.server.StatusResource; import io.druid.server.initialization.ServerConfig; @@ -60,7 +60,6 @@ import io.druid.server.metrics.DataSourceTaskIdHolder; import io.druid.server.metrics.MetricsModule; import io.druid.server.metrics.MonitorsConfig; -import io.netty.handler.codec.http.HttpVersion; import org.eclipse.jetty.server.ConnectionFactory; import org.eclipse.jetty.server.Handler; import org.eclipse.jetty.server.HttpConfiguration; @@ -91,6 +90,7 @@ public class JettyServerModule extends JerseyServletModule private static final Logger log = new Logger(JettyServerModule.class); private static final AtomicInteger activeConnections = new AtomicInteger(); + private static final String HTTP_1_1_STRING = "HTTP/1.1"; @Override protected void configureServlets() @@ -269,7 +269,7 @@ static Server makeAndInitializeServer( httpsConfiguration.setRequestHeaderSize(config.getMaxRequestHeaderSize()); final ServerConnector connector = new ServerConnector( server, - new SslConnectionFactory(sslContextFactory, HttpVersion.HTTP_1_1.toString()), + new SslConnectionFactory(sslContextFactory, HTTP_1_1_STRING), new HttpConnectionFactory(httpsConfiguration) ); connector.setPort(node.getTlsPort()); From 3dc2ec918c9c8afb985c6a343d34cf3743e51131 Mon Sep 17 00:00:00 2001 From: Jihoon Son Date: Wed, 14 Feb 2018 13:10:52 -0800 Subject: [PATCH 05/21] remove redundant dependencies, fix potential NPE, and fix test --- extensions-core/hdfs-storage/pom.xml | 4 -- extensions-core/s3-extensions/pom.xml | 5 -- .../firehose/s3/StaticS3FirehoseFactory.java | 19 +++++- indexing-hadoop/pom.xml | 30 ++++----- .../JavaScriptWorkerSelectStrategyTest.java | 2 +- pom.xml | 62 ------------------- 6 files changed, 33 insertions(+), 89 deletions(-) diff --git a/extensions-core/hdfs-storage/pom.xml b/extensions-core/hdfs-storage/pom.xml index f737da39c3e1..a653053d8cf4 100644 --- a/extensions-core/hdfs-storage/pom.xml +++ b/extensions-core/hdfs-storage/pom.xml @@ -152,10 +152,6 @@ - - com.amazonaws - aws-java-sdk-s3 - commons-io commons-io diff --git a/extensions-core/s3-extensions/pom.xml b/extensions-core/s3-extensions/pom.xml index 5513656f4151..ad4e468de670 100644 --- a/extensions-core/s3-extensions/pom.xml +++ b/extensions-core/s3-extensions/pom.xml @@ -45,11 +45,6 @@ ${project.parent.version} provided - - com.amazonaws - aws-java-sdk-s3 - provided - io.druid java-util diff --git a/extensions-core/s3-extensions/src/main/java/io/druid/firehose/s3/StaticS3FirehoseFactory.java b/extensions-core/s3-extensions/src/main/java/io/druid/firehose/s3/StaticS3FirehoseFactory.java index de93fc942197..e95c8db96df0 100644 --- a/extensions-core/s3-extensions/src/main/java/io/druid/firehose/s3/StaticS3FirehoseFactory.java +++ b/extensions-core/s3-extensions/src/main/java/io/druid/firehose/s3/StaticS3FirehoseFactory.java @@ -23,6 +23,7 @@ import com.amazonaws.services.s3.model.AmazonS3Exception; import com.amazonaws.services.s3.model.GetObjectRequest; import com.amazonaws.services.s3.model.ObjectMetadata; +import com.amazonaws.services.s3.model.S3Object; import com.amazonaws.services.s3.model.S3ObjectSummary; import com.fasterxml.jackson.annotation.JacksonInject; import com.fasterxml.jackson.annotation.JsonCreator; @@ -34,6 +35,7 @@ import io.druid.java.util.common.CompressionUtils; import io.druid.java.util.common.IAE; import io.druid.java.util.common.IOE; +import io.druid.java.util.common.ISE; import io.druid.java.util.common.logger.Logger; import io.druid.storage.s3.S3Utils; @@ -174,7 +176,11 @@ protected InputStream openObjectStream(S3ObjectSummary object) throws IOExceptio { try { // Get data of the given object and open an input stream - return s3Client.getObject(object.getBucketName(), object.getKey()).getObjectContent(); + final S3Object s3Object = s3Client.getObject(object.getBucketName(), object.getKey()); + if (s3Object == null) { + throw new ISE("Failed to get an s3 object for bucket[%s] and key[%s]", object.getBucketName(), object.getKey()); + } + return s3Object.getObjectContent(); } catch (AmazonS3Exception e) { throw new IOException(e); @@ -187,7 +193,16 @@ protected InputStream openObjectStream(S3ObjectSummary object, long start) throw final GetObjectRequest request = new GetObjectRequest(object.getBucketName(), object.getKey()); request.setRange(start); try { - return s3Client.getObject(request).getObjectContent(); + final S3Object s3Object = s3Client.getObject(request); + if (s3Object == null) { + throw new ISE( + "Failed to get an s3 object for bucket[%s], key[%s], and start[%d]", + object.getBucketName(), + object.getKey(), + start + ); + } + return s3Object.getObjectContent(); } catch (AmazonS3Exception e) { throw new IOException(e); diff --git a/indexing-hadoop/pom.xml b/indexing-hadoop/pom.xml index 75499854cf1e..3a9c7740a033 100644 --- a/indexing-hadoop/pom.xml +++ b/indexing-hadoop/pom.xml @@ -56,21 +56,6 @@ com.google.guava guava - - com.amazonaws - aws-java-sdk-s3 - test - - - org.apache.httpcomponents - httpclient - test - - - org.apache.httpcomponents - httpcore - test - org.apache.hadoop hadoop-client @@ -98,6 +83,21 @@ + + com.amazonaws + aws-java-sdk-bundle + test + + + org.apache.httpcomponents + httpclient + test + + + org.apache.httpcomponents + httpcore + test + junit junit diff --git a/indexing-service/src/test/java/io/druid/indexing/overlord/setup/JavaScriptWorkerSelectStrategyTest.java b/indexing-service/src/test/java/io/druid/indexing/overlord/setup/JavaScriptWorkerSelectStrategyTest.java index 174f7a85f1fd..0569d07f2b11 100644 --- a/indexing-service/src/test/java/io/druid/indexing/overlord/setup/JavaScriptWorkerSelectStrategyTest.java +++ b/indexing-service/src/test/java/io/druid/indexing/overlord/setup/JavaScriptWorkerSelectStrategyTest.java @@ -108,7 +108,7 @@ public void testDisabled() throws Exception expectedException.expectCause(CoreMatchers.instanceOf(IllegalStateException.class)); expectedException.expectMessage("JavaScript is disabled"); - mapper.readValue(strategyString, JavaScriptWorkerSelectStrategy.class); + mapper.readValue(strategyString, WorkerSelectStrategy.class); } @Test diff --git a/pom.xml b/pom.xml index a18262b6b418..b91339226d16 100644 --- a/pom.xml +++ b/pom.xml @@ -173,73 +173,11 @@ commons-lang 2.6 - - com.amazonaws - aws-java-sdk-ec2 - ${aws.sdk.version} - - - javax.mail - mail - - - com.fasterxml.jackson.core - jackson-databind - - - com.fasterxml.jackson.core - jackson-annotations - - - com.fasterxml.jackson.dataformat - jackson-dataformat-cbor - - - commons-codec - commons-codec - - - aws-java-sdk-core - com.amazonaws - - - com.amazonaws aws-java-sdk-bundle ${aws.sdk.bundle.version} - - com.amazonaws - aws-java-sdk-s3 - ${aws.sdk.version} - - - javax.mail - mail - - - com.fasterxml.jackson.core - jackson-databind - - - com.fasterxml.jackson.core - jackson-annotations - - - com.fasterxml.jackson.dataformat - jackson-dataformat-cbor - - - commons-codec - commons-codec - - - aws-java-sdk-core - com.amazonaws - - - com.ning compress-lzf From efb7cadfdf9c6569d00cb9c739958d12b57a2e7b Mon Sep 17 00:00:00 2001 From: Jihoon Son Date: Wed, 21 Feb 2018 13:15:11 -0800 Subject: [PATCH 06/21] resolve TODOs --- .../druid/storage/s3/S3DataSegmentPusher.java | 30 +++++++++++-------- .../java/io/druid/storage/s3/S3Utils.java | 3 -- 2 files changed, 17 insertions(+), 16 deletions(-) diff --git a/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3DataSegmentPusher.java b/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3DataSegmentPusher.java index e01fe0046fe9..31da44569ab5 100644 --- a/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3DataSegmentPusher.java +++ b/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3DataSegmentPusher.java @@ -27,9 +27,9 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.inject.Inject; -import io.druid.java.util.emitter.EmittingLogger; import io.druid.java.util.common.CompressionUtils; import io.druid.java.util.common.StringUtils; +import io.druid.java.util.emitter.EmittingLogger; import io.druid.segment.SegmentUtils; import io.druid.segment.loading.DataSegmentPusher; import io.druid.timeline.DataSegment; @@ -99,7 +99,7 @@ public DataSegment push(final File indexFilesDir, final DataSegment inSegment, f try { return S3Utils.retryS3Operation( () -> { - uploadFileAndClear(s3Client, config.getBucket(), s3Path, zipOutFile, replaceExisting); + uploadFileIfPossibleAndClear(s3Client, config.getBucket(), s3Path, zipOutFile, replaceExisting); final DataSegment outSegment = inSegment.withSize(indexSize) .withLoadSpec(makeLoadSpec(config.getBucket(), s3Path)) @@ -110,7 +110,7 @@ public DataSegment push(final File indexFilesDir, final DataSegment inSegment, f // runtime, and because Guava deletes methods over time, that causes incompatibilities. Files.write(descriptorFile.toPath(), jsonMapper.writeValueAsBytes(outSegment)); - uploadFileAndClear( + uploadFileIfPossibleAndClear( s3Client, config.getBucket(), S3Utils.descriptorPathForSegmentPath(s3Path), @@ -155,20 +155,24 @@ private Map makeLoadSpec(String bucket, String key) ); } - private void uploadFileAndClear(AmazonS3Client s3Client, String bucket, String key, File file, boolean replaceExisting) + private void uploadFileIfPossibleAndClear( + AmazonS3Client s3Client, + String bucket, + String key, + File file, + boolean replaceExisting + ) { - final PutObjectRequest indexFilePutRequest = new PutObjectRequest(bucket, key, file); - - if (!config.getDisableAcl()) { - indexFilePutRequest.setAccessControlList( - S3Utils.grantFullControlToBucketOwver(s3Client, bucket) - ); - } - - // TODO: maybe the below check can be moved out to the caller if (!replaceExisting && S3Utils.isObjectInBucketIgnoringPermission(s3Client, bucket, key)) { log.info("Skipping push because key [%s] exists && replaceExisting == false", key); } else { + final PutObjectRequest indexFilePutRequest = new PutObjectRequest(bucket, key, file); + + if (!config.getDisableAcl()) { + indexFilePutRequest.setAccessControlList( + S3Utils.grantFullControlToBucketOwver(s3Client, bucket) + ); + } log.info("Pushing [%s] to bucket[%s] and key[%s].", file, bucket, key); s3Client.putObject(indexFilePutRequest); } diff --git a/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3Utils.java b/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3Utils.java index c7b3feb21968..e31b18cc3b7e 100644 --- a/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3Utils.java +++ b/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3Utils.java @@ -52,7 +52,6 @@ public class S3Utils static boolean isServiceExceptionRecoverable(AmazonServiceException ex) { final boolean isIOException = ex.getCause() instanceof IOException; - // TODO: check the below error codes final boolean isTimeout = "RequestTimeout".equals(ex.getErrorCode()); return isIOException || isTimeout; } @@ -87,7 +86,6 @@ public static T retryS3Operation(Task f) throws Exception static boolean isObjectInBucketIgnoringPermission(AmazonS3Client s3Client, String bucketName, String objectKey) { try { - // TODO: check the below error codes return s3Client.doesObjectExist(bucketName, objectKey); } catch (AmazonS3Exception e) { @@ -205,7 +203,6 @@ public static String extractS3Key(URI uri) return uri.getPath().startsWith("/") ? uri.getPath().substring(1) : uri.getPath(); } - // TODO: check this is possible with the official sdk // Copied from org.jets3t.service.model.StorageObject.isDirectoryPlaceholder() public static boolean isDirectoryPlaceholder(String key, ObjectMetadata objectMetadata) { From cc639646d71fd54277444a4aa8f76b105e9c75de Mon Sep 17 00:00:00 2001 From: Jihoon Son Date: Wed, 21 Feb 2018 13:46:44 -0800 Subject: [PATCH 07/21] fix build --- .../java/io/druid/server/AsyncManagementForwardingServlet.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/io/druid/server/AsyncManagementForwardingServlet.java b/server/src/main/java/io/druid/server/AsyncManagementForwardingServlet.java index 21d952c7449e..5e84be355ded 100644 --- a/server/src/main/java/io/druid/server/AsyncManagementForwardingServlet.java +++ b/server/src/main/java/io/druid/server/AsyncManagementForwardingServlet.java @@ -19,6 +19,7 @@ package io.druid.server; +import com.amazonaws.thirdparty.apache.http.client.utils.URIBuilder; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Throwables; import com.google.common.collect.ImmutableMap; @@ -33,7 +34,6 @@ import io.druid.java.util.common.StringUtils; import io.druid.java.util.emitter.EmittingLogger; import io.druid.server.security.AuthConfig; -import org.apache.http.client.utils.URIBuilder; import org.eclipse.jetty.client.HttpClient; import org.eclipse.jetty.client.api.Request; import org.eclipse.jetty.proxy.AsyncProxyServlet; From 2b51ea70d3c507594f7ef4d9ab00fabb21686812 Mon Sep 17 00:00:00 2001 From: Jihoon Son Date: Wed, 21 Feb 2018 15:52:32 -0800 Subject: [PATCH 08/21] downgrade jackson version to 2.6.7 --- .../java/io/druid/guice/JsonConfigProvider.java | 4 ++-- .../druid/storage/hdfs/HdfsDataSegmentFinder.java | 8 +++----- .../segment/loading/HdfsDataSegmentFinderTest.java | 13 ++++++------- .../druid/indexer/DetermineHashedPartitionsJob.java | 3 ++- .../java/io/druid/indexer/IndexGeneratorJob.java | 3 +-- pom.xml | 3 ++- 6 files changed, 16 insertions(+), 18 deletions(-) diff --git a/api/src/main/java/io/druid/guice/JsonConfigProvider.java b/api/src/main/java/io/druid/guice/JsonConfigProvider.java index c3a9cfd64d80..609567b2dd7d 100644 --- a/api/src/main/java/io/druid/guice/JsonConfigProvider.java +++ b/api/src/main/java/io/druid/guice/JsonConfigProvider.java @@ -130,8 +130,8 @@ public static void bind( Key> supplierKey ) { - binder.bind(supplierKey).toProvider((Provider) of(propertyBase, clazz)).in(LazySingleton.class); - binder.bind(instanceKey).toProvider(new SupplierProvider(supplierKey)); + binder.bind(supplierKey).toProvider(of(propertyBase, clazz)).in(LazySingleton.class); + binder.bind(instanceKey).toProvider(new SupplierProvider<>(supplierKey)); } @SuppressWarnings("unchecked") diff --git a/extensions-core/hdfs-storage/src/main/java/io/druid/storage/hdfs/HdfsDataSegmentFinder.java b/extensions-core/hdfs-storage/src/main/java/io/druid/storage/hdfs/HdfsDataSegmentFinder.java index e5c2d3d52461..c960541e634e 100644 --- a/extensions-core/hdfs-storage/src/main/java/io/druid/storage/hdfs/HdfsDataSegmentFinder.java +++ b/extensions-core/hdfs-storage/src/main/java/io/druid/storage/hdfs/HdfsDataSegmentFinder.java @@ -22,19 +22,17 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.Sets; import com.google.inject.Inject; +import io.druid.java.util.common.StringUtils; import io.druid.java.util.common.logger.Logger; import io.druid.segment.loading.DataSegmentFinder; import io.druid.segment.loading.SegmentLoadingException; import io.druid.timeline.DataSegment; -import io.druid.java.util.common.StringUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.LocatedFileStatus; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.RemoteIterator; -import java.io.DataInput; -import java.io.DataOutput; import java.io.IOException; import java.util.Map; import java.util.Set; @@ -92,7 +90,7 @@ public Set findSegments(String workingDirPathStr, boolean updateDes indexZip = new Path(path.getParent(), "index.zip"); } if (fs.exists(indexZip)) { - final DataSegment dataSegment = mapper.readValue((DataInput) fs.open(path), DataSegment.class); + final DataSegment dataSegment = mapper.readValue(fs.open(path), DataSegment.class); log.info("Found segment [%s] located at [%s]", dataSegment.getIdentifier(), indexZip); final Map loadSpec = dataSegment.getLoadSpec(); @@ -104,7 +102,7 @@ public Set findSegments(String workingDirPathStr, boolean updateDes loadSpec.put("path", pathWithoutScheme); if (updateDescriptor) { log.info("Updating loadSpec in descriptor.json at [%s] with new path [%s]", path, pathWithoutScheme); - mapper.writeValue((DataOutput) fs.create(path, true), dataSegment); + mapper.writeValue(fs.create(path, true), dataSegment); } } segments.add(dataSegment); diff --git a/extensions-core/hdfs-storage/src/test/java/io/druid/segment/loading/HdfsDataSegmentFinderTest.java b/extensions-core/hdfs-storage/src/test/java/io/druid/segment/loading/HdfsDataSegmentFinderTest.java index 3581f33e5c25..626124622527 100644 --- a/extensions-core/hdfs-storage/src/test/java/io/druid/segment/loading/HdfsDataSegmentFinderTest.java +++ b/extensions-core/hdfs-storage/src/test/java/io/druid/segment/loading/HdfsDataSegmentFinderTest.java @@ -43,7 +43,6 @@ import org.junit.BeforeClass; import org.junit.Test; -import java.io.DataOutput; import java.io.File; import java.io.IOException; import java.net.URI; @@ -177,12 +176,12 @@ public void setUp() throws IOException indexZip5 = new Path(descriptor5.getParent(), "1_" + INDEX_ZIP); - mapper.writeValue((DataOutput) fs.create(descriptor1), SEGMENT_1); - mapper.writeValue((DataOutput) fs.create(descriptor2), SEGMENT_2); - mapper.writeValue((DataOutput) fs.create(descriptor3), SEGMENT_3); - mapper.writeValue((DataOutput) fs.create(descriptor4_0), SEGMENT_4_0); - mapper.writeValue((DataOutput) fs.create(descriptor4_1), SEGMENT_4_1); - mapper.writeValue((DataOutput) fs.create(descriptor5), SEGMENT_5); + mapper.writeValue(fs.create(descriptor1), SEGMENT_1); + mapper.writeValue(fs.create(descriptor2), SEGMENT_2); + mapper.writeValue(fs.create(descriptor3), SEGMENT_3); + mapper.writeValue(fs.create(descriptor4_0), SEGMENT_4_0); + mapper.writeValue(fs.create(descriptor4_1), SEGMENT_4_1); + mapper.writeValue(fs.create(descriptor5), SEGMENT_5); create(indexZip1); create(indexZip2); diff --git a/indexing-hadoop/src/main/java/io/druid/indexer/DetermineHashedPartitionsJob.java b/indexing-hadoop/src/main/java/io/druid/indexer/DetermineHashedPartitionsJob.java index 13efd97c1b2c..134bac4ce6f4 100644 --- a/indexing-hadoop/src/main/java/io/druid/indexer/DetermineHashedPartitionsJob.java +++ b/indexing-hadoop/src/main/java/io/druid/indexer/DetermineHashedPartitionsJob.java @@ -396,7 +396,8 @@ public static class DetermineHashedPartitionsPartitioner public int getPartition(LongWritable interval, BytesWritable text, int numPartitions) { - if (config.get("mapred.job.tracker").equals("local") || determineIntervals) { + final String jobTrackerString = config.get("mapred.job.tracker"); + if ((jobTrackerString != null && jobTrackerString.equals("local")) || determineIntervals) { return 0; } else { return reducerLookup.get(interval); diff --git a/indexing-hadoop/src/main/java/io/druid/indexer/IndexGeneratorJob.java b/indexing-hadoop/src/main/java/io/druid/indexer/IndexGeneratorJob.java index 8040eef57666..e094809cf3be 100644 --- a/indexing-hadoop/src/main/java/io/druid/indexer/IndexGeneratorJob.java +++ b/indexing-hadoop/src/main/java/io/druid/indexer/IndexGeneratorJob.java @@ -73,7 +73,6 @@ import org.joda.time.DateTime; import org.joda.time.Interval; -import java.io.DataInput; import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; @@ -114,7 +113,7 @@ public static List getPublishedSegments(HadoopDruidIndexerConfig co FileSystem fs = descriptorInfoDir.getFileSystem(conf); for (FileStatus status : fs.listStatus(descriptorInfoDir)) { - final DataSegment segment = jsonMapper.readValue((DataInput) fs.open(status.getPath()), DataSegment.class); + final DataSegment segment = jsonMapper.readValue(fs.open(status.getPath()), DataSegment.class); publishedSegmentsBuilder.add(segment); log.info("Adding segment %s to the list of published segments", segment.getIdentifier()); } diff --git a/pom.xml b/pom.xml index e5572ad9c170..c05260d9a630 100644 --- a/pom.xml +++ b/pom.xml @@ -68,7 +68,8 @@ 4.1.0 9.3.19.v20170502 1.19.3 - 2.8.10 + + 2.6.7 2.5 3.10.6.Final From c9db6f5da8748cebb7d77eee127c828b7b965d4c Mon Sep 17 00:00:00 2001 From: Jihoon Son Date: Thu, 22 Feb 2018 11:03:55 -0800 Subject: [PATCH 09/21] fix test --- .../src/test/java/io/druid/segment/indexing/DataSchemaTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/test/java/io/druid/segment/indexing/DataSchemaTest.java b/server/src/test/java/io/druid/segment/indexing/DataSchemaTest.java index e7902256437a..65df40f60938 100644 --- a/server/src/test/java/io/druid/segment/indexing/DataSchemaTest.java +++ b/server/src/test/java/io/druid/segment/indexing/DataSchemaTest.java @@ -279,7 +279,7 @@ public void testSerdeWithInvalidParserMap() throws Exception expectedException.expect(CoreMatchers.instanceOf(IllegalArgumentException.class)); expectedException.expectCause(CoreMatchers.instanceOf(JsonMappingException.class)); expectedException.expectMessage( - "Can not construct instance of io.druid.data.input.impl.StringInputRowParser, problem: parseSpec" + "Instantiation of [simple type, class io.druid.data.input.impl.StringInputRowParser] value failed: parseSpec" ); // Jackson creates a default type parser (StringInputRowParser) for an invalid type. From 1062a9227e445ee090dc0a8520594adbb681deaa Mon Sep 17 00:00:00 2001 From: Jihoon Son Date: Thu, 22 Feb 2018 16:05:58 -0800 Subject: [PATCH 10/21] resolve the last TODO --- .../io/druid/storage/s3/S3DataSegmentPuller.java | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3DataSegmentPuller.java b/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3DataSegmentPuller.java index 3aba36a5be75..da447fb45f01 100644 --- a/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3DataSegmentPuller.java +++ b/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3DataSegmentPuller.java @@ -71,7 +71,6 @@ private static FileObject buildFileObject(final URI uri, final AmazonS3Client s3 return new FileObject() { - final Object inputStreamOpener = new Object(); S3Object s3Object = null; @Override @@ -87,18 +86,16 @@ public String getName() return Files.getNameWithoutExtension(path) + (Strings.isNullOrEmpty(ext) ? "" : ("." + ext)); } + /** + * Returns an input stream for a s3 object. The returned input stream is not thread-safe. + */ @Override public InputStream openInputStream() throws IOException { try { - // TODO: check this method can be called by differernt threads if (s3Object == null) { - synchronized (inputStreamOpener) { - if (s3Object == null) { - // lazily promote to full GET - s3Object = s3Client.getObject(objectSummary.getBucketName(), objectSummary.getKey()); - } - } + // lazily promote to full GET + s3Object = s3Client.getObject(objectSummary.getBucketName(), objectSummary.getKey()); } final InputStream in = s3Object.getObjectContent(); From 60419aadcde5f167b4393cf68bb9187f7f82ae4a Mon Sep 17 00:00:00 2001 From: Jihoon Son Date: Thu, 1 Mar 2018 16:28:55 -0800 Subject: [PATCH 11/21] support proxy and endpoint configurations --- .../druid/common/aws/AWSEndpointConfig.java | 43 +++++++++++++ .../io/druid/common/aws/AWSProxyConfig.java | 61 ++++++++++++++++++ .../content/development/extensions-core/s3.md | 18 ++++-- .../firehose/s3/StaticS3FirehoseFactory.java | 6 +- .../storage/s3/S3DataSegmentArchiver.java | 4 +- .../druid/storage/s3/S3DataSegmentFinder.java | 6 +- .../druid/storage/s3/S3DataSegmentKiller.java | 6 +- .../druid/storage/s3/S3DataSegmentMover.java | 6 +- .../druid/storage/s3/S3DataSegmentPuller.java | 8 +-- .../druid/storage/s3/S3DataSegmentPusher.java | 8 +-- .../storage/s3/S3StorageDruidModule.java | 62 ++++++++++++++++++- .../java/io/druid/storage/s3/S3TaskLogs.java | 6 +- .../s3/S3TimestampVersionedDataFinder.java | 4 +- .../java/io/druid/storage/s3/S3Utils.java | 10 +-- .../s3/TestAWSCredentialsProvider.java | 6 +- .../main/java/io/druid/guice/AWSModule.java | 4 ++ 16 files changed, 215 insertions(+), 43 deletions(-) create mode 100644 aws-common/src/main/java/io/druid/common/aws/AWSEndpointConfig.java create mode 100644 aws-common/src/main/java/io/druid/common/aws/AWSProxyConfig.java diff --git a/aws-common/src/main/java/io/druid/common/aws/AWSEndpointConfig.java b/aws-common/src/main/java/io/druid/common/aws/AWSEndpointConfig.java new file mode 100644 index 000000000000..14cd048816d8 --- /dev/null +++ b/aws-common/src/main/java/io/druid/common/aws/AWSEndpointConfig.java @@ -0,0 +1,43 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.common.aws; + +import com.fasterxml.jackson.annotation.JsonProperty; + +public class AWSEndpointConfig +{ + @JsonProperty + private String url; + + @JsonProperty + private String signingRegion; + + @JsonProperty + public String getUrl() + { + return url; + } + + @JsonProperty + public String getSigningRegion() + { + return signingRegion; + } +} diff --git a/aws-common/src/main/java/io/druid/common/aws/AWSProxyConfig.java b/aws-common/src/main/java/io/druid/common/aws/AWSProxyConfig.java new file mode 100644 index 000000000000..90c7665286f4 --- /dev/null +++ b/aws-common/src/main/java/io/druid/common/aws/AWSProxyConfig.java @@ -0,0 +1,61 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.common.aws; + +import com.fasterxml.jackson.annotation.JsonProperty; + +public class AWSProxyConfig +{ + @JsonProperty + private String host; + + @JsonProperty + private int port = -1; + + @JsonProperty + private String username; + + @JsonProperty + private String password; + + @JsonProperty + public String getHost() + { + return host; + } + + @JsonProperty + public int getPort() + { + return port; + } + + @JsonProperty + public String getUsername() + { + return username; + } + + @JsonProperty + public String getPassword() + { + return password; + } +} diff --git a/docs/content/development/extensions-core/s3.md b/docs/content/development/extensions-core/s3.md index f24a406946b6..3d32c9de8953 100644 --- a/docs/content/development/extensions-core/s3.md +++ b/docs/content/development/extensions-core/s3.md @@ -12,12 +12,18 @@ S3-compatible deep storage is basically either S3 or something like Google Stora ### Configuration -|Property|Possible Values|Description|Default| -|--------|---------------|-----------|-------| -|`druid.s3.accessKey`||S3 access key.|Must be set.| -|`druid.s3.secretKey`||S3 secret key.|Must be set.| -|`druid.storage.bucket`||Bucket to store in.|Must be set.| -|`druid.storage.baseKey`||Base key prefix to use, i.e. what directory.|Must be set.| +|Property|Description|Default| +|--------|-----------|-------| +|`druid.s3.accessKey`|S3 access key.|Must be set.| +|`druid.s3.secretKey`|S3 secret key.|Must be set.| +|`druid.storage.bucket`|Bucket to store in.|Must be set.| +|`druid.storage.baseKey`|Base key prefix to use, i.e. what directory.|Must be set.| +|`druid.s3.endpoint.url`|Service endpoint either with or without the protocol.|None| +|`druid.s3.endpoint.signingRegion`|Region to use for SigV4 signing of requests (e.g. us-west-1).|None| +|`druid.s3.proxy.host`|Proxy host to connect through.|None| +|`druid.s3.proxy.port`|Port on the proxy host to connect through.|None| +|`druid.s3.proxy.username`|User name to use when connecting through a proxy.|None| +|`druid.s3.proxy.password`|Password to use when connecting through a proxy.|None| ## StaticS3Firehose diff --git a/extensions-core/s3-extensions/src/main/java/io/druid/firehose/s3/StaticS3FirehoseFactory.java b/extensions-core/s3-extensions/src/main/java/io/druid/firehose/s3/StaticS3FirehoseFactory.java index e95c8db96df0..8827fc9ae31d 100644 --- a/extensions-core/s3-extensions/src/main/java/io/druid/firehose/s3/StaticS3FirehoseFactory.java +++ b/extensions-core/s3-extensions/src/main/java/io/druid/firehose/s3/StaticS3FirehoseFactory.java @@ -19,7 +19,7 @@ package io.druid.firehose.s3; -import com.amazonaws.services.s3.AmazonS3Client; +import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.model.AmazonS3Exception; import com.amazonaws.services.s3.model.GetObjectRequest; import com.amazonaws.services.s3.model.ObjectMetadata; @@ -57,13 +57,13 @@ public class StaticS3FirehoseFactory extends PrefetchableTextFilesFirehoseFactor private static final Logger log = new Logger(StaticS3FirehoseFactory.class); private static final int MAX_LISTING_LENGTH = 1024; - private final AmazonS3Client s3Client; + private final AmazonS3 s3Client; private final List uris; private final List prefixes; @JsonCreator public StaticS3FirehoseFactory( - @JacksonInject("s3Client") AmazonS3Client s3Client, + @JacksonInject("s3Client") AmazonS3 s3Client, @JsonProperty("uris") List uris, @JsonProperty("prefixes") List prefixes, @JsonProperty("maxCacheCapacityBytes") Long maxCacheCapacityBytes, diff --git a/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3DataSegmentArchiver.java b/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3DataSegmentArchiver.java index af7c6e4afff5..42eef5ce819e 100644 --- a/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3DataSegmentArchiver.java +++ b/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3DataSegmentArchiver.java @@ -19,7 +19,7 @@ package io.druid.storage.s3; -import com.amazonaws.services.s3.AmazonS3Client; +import com.amazonaws.services.s3.AmazonS3; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Objects; import com.google.common.collect.ImmutableMap; @@ -40,7 +40,7 @@ public class S3DataSegmentArchiver extends S3DataSegmentMover implements DataSeg @Inject public S3DataSegmentArchiver( @Json ObjectMapper mapper, - AmazonS3Client s3Client, + AmazonS3 s3Client, S3DataSegmentArchiverConfig archiveConfig, S3DataSegmentPusherConfig restoreConfig ) diff --git a/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3DataSegmentFinder.java b/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3DataSegmentFinder.java index 42d87c73d14b..649554e7564b 100644 --- a/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3DataSegmentFinder.java +++ b/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3DataSegmentFinder.java @@ -20,7 +20,7 @@ package io.druid.storage.s3; import com.amazonaws.AmazonServiceException; -import com.amazonaws.services.s3.AmazonS3Client; +import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.model.ObjectMetadata; import com.amazonaws.services.s3.model.S3Object; import com.amazonaws.services.s3.model.S3ObjectInputStream; @@ -45,13 +45,13 @@ public class S3DataSegmentFinder implements DataSegmentFinder { private static final Logger log = new Logger(S3DataSegmentFinder.class); - private final AmazonS3Client s3Client; + private final AmazonS3 s3Client; private final ObjectMapper jsonMapper; private final S3DataSegmentPusherConfig config; @Inject public S3DataSegmentFinder( - AmazonS3Client s3Client, + AmazonS3 s3Client, S3DataSegmentPusherConfig config, ObjectMapper jsonMapper ) diff --git a/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3DataSegmentKiller.java b/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3DataSegmentKiller.java index 4947d60fdc57..6c2825eac38a 100644 --- a/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3DataSegmentKiller.java +++ b/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3DataSegmentKiller.java @@ -20,7 +20,7 @@ package io.druid.storage.s3; import com.amazonaws.AmazonServiceException; -import com.amazonaws.services.s3.AmazonS3Client; +import com.amazonaws.services.s3.AmazonS3; import com.google.inject.Inject; import io.druid.java.util.common.MapUtils; import io.druid.java.util.common.logger.Logger; @@ -37,11 +37,11 @@ public class S3DataSegmentKiller implements DataSegmentKiller { private static final Logger log = new Logger(S3DataSegmentKiller.class); - private final AmazonS3Client s3Client; + private final AmazonS3 s3Client; @Inject public S3DataSegmentKiller( - AmazonS3Client s3Client + AmazonS3 s3Client ) { this.s3Client = s3Client; diff --git a/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3DataSegmentMover.java b/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3DataSegmentMover.java index ea11d95a2da3..fdc5a756afc1 100644 --- a/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3DataSegmentMover.java +++ b/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3DataSegmentMover.java @@ -20,7 +20,7 @@ package io.druid.storage.s3; import com.amazonaws.AmazonServiceException; -import com.amazonaws.services.s3.AmazonS3Client; +import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.model.CopyObjectRequest; import com.amazonaws.services.s3.model.ListObjectsV2Request; import com.amazonaws.services.s3.model.ListObjectsV2Result; @@ -49,12 +49,12 @@ public class S3DataSegmentMover implements DataSegmentMover { private static final Logger log = new Logger(S3DataSegmentMover.class); - private final AmazonS3Client s3Client; + private final AmazonS3 s3Client; private final S3DataSegmentPusherConfig config; @Inject public S3DataSegmentMover( - AmazonS3Client s3Client, + AmazonS3 s3Client, S3DataSegmentPusherConfig config ) { diff --git a/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3DataSegmentPuller.java b/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3DataSegmentPuller.java index da447fb45f01..059b6d39bfd4 100644 --- a/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3DataSegmentPuller.java +++ b/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3DataSegmentPuller.java @@ -20,7 +20,7 @@ package io.druid.storage.s3; import com.amazonaws.AmazonServiceException; -import com.amazonaws.services.s3.AmazonS3Client; +import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.model.AmazonS3Exception; import com.amazonaws.services.s3.model.S3Object; import com.amazonaws.services.s3.model.S3ObjectSummary; @@ -63,7 +63,7 @@ public class S3DataSegmentPuller implements DataSegmentPuller, URIDataPuller { public static final int DEFAULT_RETRY_COUNT = 3; - private static FileObject buildFileObject(final URI uri, final AmazonS3Client s3Client) throws AmazonServiceException + private static FileObject buildFileObject(final URI uri, final AmazonS3 s3Client) throws AmazonServiceException { final S3Coords coords = new S3Coords(checkURI(uri)); final S3ObjectSummary objectSummary = S3Utils.getSingleObjectSummary(s3Client, coords.bucket, coords.path); @@ -162,11 +162,11 @@ public boolean delete() protected static final String BUCKET = "bucket"; protected static final String KEY = "key"; - protected final AmazonS3Client s3Client; + protected final AmazonS3 s3Client; @Inject public S3DataSegmentPuller( - AmazonS3Client s3Client + AmazonS3 s3Client ) { this.s3Client = s3Client; diff --git a/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3DataSegmentPusher.java b/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3DataSegmentPusher.java index 31da44569ab5..31b650425b72 100644 --- a/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3DataSegmentPusher.java +++ b/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3DataSegmentPusher.java @@ -20,7 +20,7 @@ package io.druid.storage.s3; import com.amazonaws.AmazonServiceException; -import com.amazonaws.services.s3.AmazonS3Client; +import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.model.PutObjectRequest; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Throwables; @@ -45,13 +45,13 @@ public class S3DataSegmentPusher implements DataSegmentPusher { private static final EmittingLogger log = new EmittingLogger(S3DataSegmentPusher.class); - private final AmazonS3Client s3Client; + private final AmazonS3 s3Client; private final S3DataSegmentPusherConfig config; private final ObjectMapper jsonMapper; @Inject public S3DataSegmentPusher( - AmazonS3Client s3Client, + AmazonS3 s3Client, S3DataSegmentPusherConfig config, ObjectMapper jsonMapper ) @@ -156,7 +156,7 @@ private Map makeLoadSpec(String bucket, String key) } private void uploadFileIfPossibleAndClear( - AmazonS3Client s3Client, + AmazonS3 s3Client, String bucket, String key, File file, diff --git a/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3StorageDruidModule.java b/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3StorageDruidModule.java index f091f13cd880..19ba32bbbc5e 100644 --- a/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3StorageDruidModule.java +++ b/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3StorageDruidModule.java @@ -19,8 +19,11 @@ package io.druid.storage.s3; +import com.amazonaws.ClientConfiguration; import com.amazonaws.auth.AWSCredentialsProvider; -import com.amazonaws.services.s3.AmazonS3Client; +import com.amazonaws.client.builder.AwsClientBuilder.EndpointConfiguration; +import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.AmazonS3ClientBuilder; import com.fasterxml.jackson.core.Version; import com.fasterxml.jackson.databind.Module; import com.google.common.collect.ImmutableList; @@ -28,11 +31,15 @@ import com.google.inject.Provides; import com.google.inject.multibindings.MapBinder; import io.druid.common.aws.AWSCredentialsConfig; +import io.druid.common.aws.AWSEndpointConfig; +import io.druid.common.aws.AWSProxyConfig; import io.druid.data.SearchableVersionedDataFinder; import io.druid.guice.Binders; import io.druid.guice.JsonConfigProvider; import io.druid.guice.LazySingleton; import io.druid.initialization.DruidModule; +import io.druid.java.util.common.logger.Logger; +import org.apache.commons.lang.StringUtils; import java.util.List; @@ -42,6 +49,8 @@ public class S3StorageDruidModule implements DruidModule { public static final String SCHEME = "s3_zip"; + private static final Logger log = new Logger(S3StorageDruidModule.class); + @Override public List getJacksonModules() { @@ -73,6 +82,8 @@ public void setupModule(SetupContext context) public void configure(Binder binder) { JsonConfigProvider.bind(binder, "druid.s3", AWSCredentialsConfig.class); + JsonConfigProvider.bind(binder, "druid.s3.proxy", AWSProxyConfig.class); + JsonConfigProvider.bind(binder, "druid.s3.endpoint", AWSEndpointConfig.class); MapBinder.newMapBinder(binder, String.class, SearchableVersionedDataFinder.class) .addBinding("s3") .to(S3TimestampVersionedDataFinder.class) @@ -100,8 +111,53 @@ public void configure(Binder binder) @Provides @LazySingleton - public AmazonS3Client getAmazonS3Client(AWSCredentialsProvider provider) + public AmazonS3 getAmazonS3Client( + AWSCredentialsProvider provider, + AWSProxyConfig proxyConfig, + AWSEndpointConfig endpointConfig + ) + { + final AmazonS3ClientBuilder builder = AmazonS3ClientBuilder.standard().withCredentials(provider); + final ClientConfiguration configuration = new ClientConfiguration(); + + builder.withClientConfiguration(setProxyConfig(configuration, proxyConfig)); + if (toEndpointConfiguration(endpointConfig) != null) { + builder.withEndpointConfiguration(toEndpointConfiguration(endpointConfig)); + } + + log.info(builder.getEndpoint().getServiceEndpoint()); + log.info(builder.getEndpoint().getSigningRegion()); + log.info(builder.getClientConfiguration().getProxyHost()); + log.info(builder.getClientConfiguration().getProxyPassword()); + return builder.build(); + } + + private static ClientConfiguration setProxyConfig(ClientConfiguration conf, AWSProxyConfig proxyConfig) + { + if (StringUtils.isNotEmpty(proxyConfig.getHost())) { + conf.setProxyHost(proxyConfig.getHost()); + } + if (proxyConfig.getPort() != -1) { + conf.setProxyPort(proxyConfig.getPort()); + } + if (StringUtils.isNotEmpty(proxyConfig.getUsername())) { + conf.setProxyUsername(proxyConfig.getUsername()); + } + if (StringUtils.isNotEmpty(proxyConfig.getPassword())) { + conf.setProxyPassword(proxyConfig.getPassword()); + } + if (StringUtils.isNotEmpty(proxyConfig.getDomain())) { + conf.setProxyDomain(proxyConfig.getDomain()); + } + return conf; + } + + private static EndpointConfiguration toEndpointConfiguration(AWSEndpointConfig endpointConfig) { - return new AmazonS3Client(provider); + if (StringUtils.isNotEmpty(endpointConfig.getUrl())) { + return new EndpointConfiguration(endpointConfig.getUrl(), endpointConfig.getSigningRegion()); + } else { + return null; + } } } diff --git a/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3TaskLogs.java b/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3TaskLogs.java index 89a41b093ae3..b89d7a5bde3f 100644 --- a/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3TaskLogs.java +++ b/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3TaskLogs.java @@ -20,7 +20,7 @@ package io.druid.storage.s3; import com.amazonaws.AmazonServiceException; -import com.amazonaws.services.s3.AmazonS3Client; +import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.model.AmazonS3Exception; import com.amazonaws.services.s3.model.GetObjectRequest; import com.amazonaws.services.s3.model.ObjectMetadata; @@ -44,11 +44,11 @@ public class S3TaskLogs implements TaskLogs { private static final Logger log = new Logger(S3TaskLogs.class); - private final AmazonS3Client service; + private final AmazonS3 service; private final S3TaskLogsConfig config; @Inject - public S3TaskLogs(S3TaskLogsConfig config, AmazonS3Client service) + public S3TaskLogs(S3TaskLogsConfig config, AmazonS3 service) { this.config = config; this.service = service; diff --git a/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3TimestampVersionedDataFinder.java b/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3TimestampVersionedDataFinder.java index 914546cba8fc..2d4724851b7c 100644 --- a/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3TimestampVersionedDataFinder.java +++ b/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3TimestampVersionedDataFinder.java @@ -19,7 +19,7 @@ package io.druid.storage.s3; -import com.amazonaws.services.s3.AmazonS3Client; +import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.model.S3ObjectSummary; import com.google.common.base.Throwables; import com.google.inject.Inject; @@ -41,7 +41,7 @@ public class S3TimestampVersionedDataFinder extends S3DataSegmentPuller implemen private static final int MAX_LISTING_KEYS = 1000; @Inject - public S3TimestampVersionedDataFinder(AmazonS3Client s3Client) + public S3TimestampVersionedDataFinder(AmazonS3 s3Client) { super(s3Client); } diff --git a/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3Utils.java b/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3Utils.java index e31b18cc3b7e..35edd89bb488 100644 --- a/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3Utils.java +++ b/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3Utils.java @@ -20,7 +20,7 @@ package io.druid.storage.s3; import com.amazonaws.AmazonServiceException; -import com.amazonaws.services.s3.AmazonS3Client; +import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.model.AccessControlList; import com.amazonaws.services.s3.model.AmazonS3Exception; import com.amazonaws.services.s3.model.CanonicalGrantee; @@ -83,7 +83,7 @@ public static T retryS3Operation(Task f) throws Exception return RetryUtils.retry(f, S3RETRY, maxTries); } - static boolean isObjectInBucketIgnoringPermission(AmazonS3Client s3Client, String bucketName, String objectKey) + static boolean isObjectInBucketIgnoringPermission(AmazonS3 s3Client, String bucketName, String objectKey) { try { return s3Client.doesObjectExist(bucketName, objectKey); @@ -99,7 +99,7 @@ static boolean isObjectInBucketIgnoringPermission(AmazonS3Client s3Client, Strin } public static Iterator objectSummaryIterator( - final AmazonS3Client s3Client, + final AmazonS3 s3Client, final String bucket, final String prefix, final int numMaxKeys @@ -191,7 +191,7 @@ static String toFilename(String key, final String suffix) return filename; } - static AccessControlList grantFullControlToBucketOwver(AmazonS3Client s3Client, String bucket) + static AccessControlList grantFullControlToBucketOwver(AmazonS3 s3Client, String bucket) { final AccessControlList acl = s3Client.getBucketAcl(bucket); acl.grantAllPermissions(new Grant(new CanonicalGrantee(acl.getOwner().getId()), Permission.FullControl)); @@ -240,7 +240,7 @@ public static boolean isDirectoryPlaceholder(String key, ObjectMetadata objectMe * @param bucket s3 bucket * @param key unique key for the object to be retrieved */ - public static S3ObjectSummary getSingleObjectSummary(AmazonS3Client s3Client, String bucket, String key) + public static S3ObjectSummary getSingleObjectSummary(AmazonS3 s3Client, String bucket, String key) { final ListObjectsV2Request request = new ListObjectsV2Request() .withBucketName(bucket) diff --git a/extensions-core/s3-extensions/src/test/java/io/druid/storage/s3/TestAWSCredentialsProvider.java b/extensions-core/s3-extensions/src/test/java/io/druid/storage/s3/TestAWSCredentialsProvider.java index db97c424bc18..a7716e2a5ed8 100644 --- a/extensions-core/s3-extensions/src/test/java/io/druid/storage/s3/TestAWSCredentialsProvider.java +++ b/extensions-core/s3-extensions/src/test/java/io/druid/storage/s3/TestAWSCredentialsProvider.java @@ -24,6 +24,8 @@ import com.amazonaws.auth.AWSSessionCredentials; import com.google.common.io.Files; import io.druid.common.aws.AWSCredentialsConfig; +import io.druid.common.aws.AWSEndpointConfig; +import io.druid.common.aws.AWSProxyConfig; import io.druid.guice.AWSModule; import io.druid.metadata.DefaultPasswordProvider; import org.easymock.EasyMock; @@ -58,7 +60,7 @@ public void testWithFixedAWSKeys() assertEquals(credentials.getAWSSecretKey(), "secretKeySample"); // try to create - s3Module.getAmazonS3Client(provider); + s3Module.getAmazonS3Client(provider, new AWSProxyConfig(), new AWSEndpointConfig()); } @Rule @@ -86,6 +88,6 @@ public void testWithFileSessionCredentials() throws IOException assertEquals(sessionCredentials.getSessionToken(), "sessionTokenSample"); // try to create - s3Module.getAmazonS3Client(provider); + s3Module.getAmazonS3Client(provider, new AWSProxyConfig(), new AWSEndpointConfig()); } } diff --git a/server/src/main/java/io/druid/guice/AWSModule.java b/server/src/main/java/io/druid/guice/AWSModule.java index 15925816a28c..4127f3cb2000 100644 --- a/server/src/main/java/io/druid/guice/AWSModule.java +++ b/server/src/main/java/io/druid/guice/AWSModule.java @@ -27,6 +27,8 @@ import com.google.inject.Provides; import io.druid.common.aws.AWSCredentialsConfig; import io.druid.common.aws.AWSCredentialsUtils; +import io.druid.common.aws.AWSEndpointConfig; +import io.druid.common.aws.AWSProxyConfig; /** */ @@ -36,6 +38,8 @@ public class AWSModule implements Module public void configure(Binder binder) { JsonConfigProvider.bind(binder, "druid.s3", AWSCredentialsConfig.class); + JsonConfigProvider.bind(binder, "druid.s3.proxy", AWSProxyConfig.class); + JsonConfigProvider.bind(binder, "druid.s3.endpoint", AWSEndpointConfig.class); } @Provides From b151371a93cc09d26a2625577eb52e3e19465532 Mon Sep 17 00:00:00 2001 From: Jihoon Son Date: Thu, 1 Mar 2018 17:04:40 -0800 Subject: [PATCH 12/21] fix build --- .../main/java/io/druid/storage/s3/S3StorageDruidModule.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3StorageDruidModule.java b/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3StorageDruidModule.java index 19ba32bbbc5e..f7304f6968c6 100644 --- a/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3StorageDruidModule.java +++ b/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3StorageDruidModule.java @@ -146,9 +146,6 @@ private static ClientConfiguration setProxyConfig(ClientConfiguration conf, AWSP if (StringUtils.isNotEmpty(proxyConfig.getPassword())) { conf.setProxyPassword(proxyConfig.getPassword()); } - if (StringUtils.isNotEmpty(proxyConfig.getDomain())) { - conf.setProxyDomain(proxyConfig.getDomain()); - } return conf; } From 97e2db0231a6835b72033157b3d51cfafbf2f4c0 Mon Sep 17 00:00:00 2001 From: Jihoon Son Date: Fri, 2 Mar 2018 02:32:06 -0800 Subject: [PATCH 13/21] remove debugging log --- .../io/druid/storage/s3/S3StorageDruidModule.java | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3StorageDruidModule.java b/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3StorageDruidModule.java index f7304f6968c6..12fa10459559 100644 --- a/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3StorageDruidModule.java +++ b/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3StorageDruidModule.java @@ -20,6 +20,7 @@ package io.druid.storage.s3; import com.amazonaws.ClientConfiguration; +import com.amazonaws.ClientConfigurationFactory; import com.amazonaws.auth.AWSCredentialsProvider; import com.amazonaws.client.builder.AwsClientBuilder.EndpointConfiguration; import com.amazonaws.services.s3.AmazonS3; @@ -118,17 +119,14 @@ public AmazonS3 getAmazonS3Client( ) { final AmazonS3ClientBuilder builder = AmazonS3ClientBuilder.standard().withCredentials(provider); - final ClientConfiguration configuration = new ClientConfiguration(); + final ClientConfiguration configuration = new ClientConfigurationFactory().getConfig(); - builder.withClientConfiguration(setProxyConfig(configuration, proxyConfig)); - if (toEndpointConfiguration(endpointConfig) != null) { - builder.withEndpointConfiguration(toEndpointConfiguration(endpointConfig)); + builder.setClientConfiguration(setProxyConfig(configuration, proxyConfig)); + final EndpointConfiguration endpointConfiguration = toEndpointConfiguration(endpointConfig); + if (endpointConfiguration != null) { + builder.setEndpointConfiguration(endpointConfiguration); } - log.info(builder.getEndpoint().getServiceEndpoint()); - log.info(builder.getEndpoint().getSigningRegion()); - log.info(builder.getClientConfiguration().getProxyHost()); - log.info(builder.getClientConfiguration().getProxyPassword()); return builder.build(); } From 5f2c465b605002741e9e315a16059c7b9fdc0fff Mon Sep 17 00:00:00 2001 From: Jihoon Son Date: Mon, 5 Mar 2018 17:49:02 -0800 Subject: [PATCH 14/21] downgrade hadoop version to 2.8.3 --- .../main/java/io/druid/storage/s3/S3StorageDruidModule.java | 2 +- .../java/io/druid/indexing/common/config/TaskConfig.java | 2 +- .../main/java/io/druid/indexing/common/task/HadoopTask.java | 6 ++---- pom.xml | 4 ++-- 4 files changed, 6 insertions(+), 8 deletions(-) diff --git a/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3StorageDruidModule.java b/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3StorageDruidModule.java index 12fa10459559..bba6be668110 100644 --- a/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3StorageDruidModule.java +++ b/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3StorageDruidModule.java @@ -127,7 +127,7 @@ public AmazonS3 getAmazonS3Client( builder.setEndpointConfiguration(endpointConfiguration); } - return builder.build(); + return builder.enableForceGlobalBucketAccess().build(); } private static ClientConfiguration setProxyConfig(ClientConfiguration conf, AWSProxyConfig proxyConfig) diff --git a/indexing-service/src/main/java/io/druid/indexing/common/config/TaskConfig.java b/indexing-service/src/main/java/io/druid/indexing/common/config/TaskConfig.java index fcf8aa3f4f63..9152c8732109 100644 --- a/indexing-service/src/main/java/io/druid/indexing/common/config/TaskConfig.java +++ b/indexing-service/src/main/java/io/druid/indexing/common/config/TaskConfig.java @@ -31,7 +31,7 @@ public class TaskConfig { public static final List DEFAULT_DEFAULT_HADOOP_COORDINATES = ImmutableList.of( - "org.apache.hadoop:hadoop-client:2.9.0" + "org.apache.hadoop:hadoop-client:2.8.3" ); private static final Period DEFAULT_DIRECTORY_LOCK_TIMEOUT = new Period("PT10M"); diff --git a/indexing-service/src/main/java/io/druid/indexing/common/task/HadoopTask.java b/indexing-service/src/main/java/io/druid/indexing/common/task/HadoopTask.java index 94ac078c3ea8..8963559e3123 100644 --- a/indexing-service/src/main/java/io/druid/indexing/common/task/HadoopTask.java +++ b/indexing-service/src/main/java/io/druid/indexing/common/task/HadoopTask.java @@ -35,7 +35,6 @@ import java.io.File; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; -import java.net.MalformedURLException; import java.net.URISyntaxException; import java.net.URL; import java.net.URLClassLoader; @@ -127,15 +126,14 @@ public boolean apply(@Nullable URL input) * * @param toolbox The toolbox to pull the default coordinates from if not present in the task * @return An isolated URLClassLoader not tied by parent chain to the ApplicationClassLoader - * @throws MalformedURLException from Initialization.getClassLoaderForExtension */ - protected ClassLoader buildClassLoader(final TaskToolbox toolbox) throws MalformedURLException + protected ClassLoader buildClassLoader(final TaskToolbox toolbox) { return buildClassLoader(hadoopDependencyCoordinates, toolbox.getConfig().getDefaultHadoopCoordinates()); } public static ClassLoader buildClassLoader(final List hadoopDependencyCoordinates, - final List defaultHadoopCoordinates) throws MalformedURLException + final List defaultHadoopCoordinates) { final List finalHadoopDependencyCoordinates = hadoopDependencyCoordinates != null ? hadoopDependencyCoordinates diff --git a/pom.xml b/pom.xml index c05260d9a630..48c3192243d3 100644 --- a/pom.xml +++ b/pom.xml @@ -69,7 +69,7 @@ 9.3.19.v20170502 1.19.3 - 2.6.7 + 2.4.6 2.5 3.10.6.Final @@ -78,7 +78,7 @@ 4.0.52.Final 1.7.12 - 2.9.0 + 2.8.3 2.0.0 1.6.6 1.11.29 From a8602f3452de52f67c95c00849bba29c7dfe2dbf Mon Sep 17 00:00:00 2001 From: Jihoon Son Date: Tue, 6 Mar 2018 13:04:12 -0800 Subject: [PATCH 15/21] fix tests --- .travis.yml | 1 + .../firehose/s3/StaticS3FirehoseFactoryTest.java | 12 +++++------- pom.xml | 1 - 3 files changed, 6 insertions(+), 8 deletions(-) diff --git a/.travis.yml b/.travis.yml index 4fb64a7bf9af..4ec30507b4f4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -47,6 +47,7 @@ matrix: - sudo: false env: - NAME="other modules test" + - AWS_REGION=us-east-1 # set a aws region for unit tests install: echo "MAVEN_OPTS='-Xmx3000m'" > ~/.mavenrc && mvn install -q -ff -DskipTests -B before_script: - unset _JAVA_OPTIONS diff --git a/extensions-core/s3-extensions/src/test/java/io/druid/firehose/s3/StaticS3FirehoseFactoryTest.java b/extensions-core/s3-extensions/src/test/java/io/druid/firehose/s3/StaticS3FirehoseFactoryTest.java index 7b70d874e4e6..e60210126943 100644 --- a/extensions-core/s3-extensions/src/test/java/io/druid/firehose/s3/StaticS3FirehoseFactoryTest.java +++ b/extensions-core/s3-extensions/src/test/java/io/druid/firehose/s3/StaticS3FirehoseFactoryTest.java @@ -19,6 +19,7 @@ package io.druid.firehose.s3; +import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.AmazonS3Client; import com.fasterxml.jackson.core.JsonParser; import com.fasterxml.jackson.databind.DeserializationContext; @@ -36,7 +37,6 @@ import org.junit.Assert; import org.junit.Test; -import java.io.IOException; import java.net.URI; import java.util.Arrays; import java.util.List; @@ -95,7 +95,7 @@ public List getJacksonModules() { // Deserializer is need for AmazonS3Client even though it is injected. // See https://github.com/FasterXML/jackson-databind/issues/962. - return ImmutableList.of(new SimpleModule().addDeserializer(AmazonS3Client.class, new ItemDeserializer())); + return ImmutableList.of(new SimpleModule().addDeserializer(AmazonS3.class, new ItemDeserializer())); } @Override @@ -105,13 +105,13 @@ public void configure(Binder binder) } @Provides - public AmazonS3Client getAmazonS3Client() + public AmazonS3 getAmazonS3Client() { return SERVICE; } } - public static class ItemDeserializer extends StdDeserializer + public static class ItemDeserializer extends StdDeserializer { public ItemDeserializer() { @@ -124,9 +124,7 @@ public ItemDeserializer(Class vc) } @Override - public AmazonS3Client deserialize( - JsonParser jp, DeserializationContext ctxt - ) throws IOException + public AmazonS3 deserialize(JsonParser jp, DeserializationContext ctxt) { throw new UnsupportedOperationException(); } diff --git a/pom.xml b/pom.xml index 48c3192243d3..8c26bdee3e1b 100644 --- a/pom.xml +++ b/pom.xml @@ -81,7 +81,6 @@ 2.8.3 2.0.0 1.6.6 - 1.11.29 1.11.199 2.5.5 From 97c6b03e87dba20844111baed83ac17a6a4aaa83 Mon Sep 17 00:00:00 2001 From: Jihoon Son Date: Tue, 6 Mar 2018 18:30:09 -0800 Subject: [PATCH 16/21] remove unused log --- .../main/java/io/druid/storage/s3/S3StorageDruidModule.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3StorageDruidModule.java b/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3StorageDruidModule.java index bba6be668110..be12907df7be 100644 --- a/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3StorageDruidModule.java +++ b/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3StorageDruidModule.java @@ -39,7 +39,6 @@ import io.druid.guice.JsonConfigProvider; import io.druid.guice.LazySingleton; import io.druid.initialization.DruidModule; -import io.druid.java.util.common.logger.Logger; import org.apache.commons.lang.StringUtils; import java.util.List; @@ -50,8 +49,6 @@ public class S3StorageDruidModule implements DruidModule { public static final String SCHEME = "s3_zip"; - private static final Logger log = new Logger(S3StorageDruidModule.class); - @Override public List getJacksonModules() { From adf3cbb8868c53a1023a8d5a919c845911493fe1 Mon Sep 17 00:00:00 2001 From: Jihoon Son Date: Wed, 7 Mar 2018 01:03:24 -0800 Subject: [PATCH 17/21] fix it test --- .../druid/common/aws/AWSEndpointConfig.java | 9 ++++++ .../storage/s3/S3StorageDruidModule.java | 31 +++++++++---------- pom.xml | 2 +- 3 files changed, 24 insertions(+), 18 deletions(-) diff --git a/aws-common/src/main/java/io/druid/common/aws/AWSEndpointConfig.java b/aws-common/src/main/java/io/druid/common/aws/AWSEndpointConfig.java index 14cd048816d8..773a2ab15013 100644 --- a/aws-common/src/main/java/io/druid/common/aws/AWSEndpointConfig.java +++ b/aws-common/src/main/java/io/druid/common/aws/AWSEndpointConfig.java @@ -26,6 +26,9 @@ public class AWSEndpointConfig @JsonProperty private String url; + @JsonProperty + private String serviceName; + @JsonProperty private String signingRegion; @@ -35,6 +38,12 @@ public String getUrl() return url; } + @JsonProperty + public String getServiceName() + { + return serviceName; + } + @JsonProperty public String getSigningRegion() { diff --git a/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3StorageDruidModule.java b/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3StorageDruidModule.java index be12907df7be..6ba230881f41 100644 --- a/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3StorageDruidModule.java +++ b/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3StorageDruidModule.java @@ -22,9 +22,9 @@ import com.amazonaws.ClientConfiguration; import com.amazonaws.ClientConfigurationFactory; import com.amazonaws.auth.AWSCredentialsProvider; -import com.amazonaws.client.builder.AwsClientBuilder.EndpointConfiguration; import com.amazonaws.services.s3.AmazonS3; -import com.amazonaws.services.s3.AmazonS3ClientBuilder; +import com.amazonaws.services.s3.AmazonS3Client; +import com.amazonaws.services.s3.S3ClientOptions; import com.fasterxml.jackson.core.Version; import com.fasterxml.jackson.databind.Module; import com.google.common.collect.ImmutableList; @@ -115,16 +115,22 @@ public AmazonS3 getAmazonS3Client( AWSEndpointConfig endpointConfig ) { - final AmazonS3ClientBuilder builder = AmazonS3ClientBuilder.standard().withCredentials(provider); + // AmazonS3ClientBuilder can't be used because it makes integration tests failed final ClientConfiguration configuration = new ClientConfigurationFactory().getConfig(); + final AmazonS3Client client = new AmazonS3Client(provider, setProxyConfig(configuration, proxyConfig)); - builder.setClientConfiguration(setProxyConfig(configuration, proxyConfig)); - final EndpointConfiguration endpointConfiguration = toEndpointConfiguration(endpointConfig); - if (endpointConfiguration != null) { - builder.setEndpointConfiguration(endpointConfiguration); + if (StringUtils.isNotEmpty(endpointConfig.getUrl())) { + if (StringUtils.isNotEmpty(endpointConfig.getServiceName()) && + StringUtils.isNotEmpty(endpointConfig.getSigningRegion())) { + client.setEndpoint(endpointConfig.getUrl(), endpointConfig.getServiceName(), endpointConfig.getSigningRegion()); + } else { + client.setEndpoint(endpointConfig.getUrl()); + } } - return builder.enableForceGlobalBucketAccess().build(); + client.setS3ClientOptions(S3ClientOptions.builder().enableForceGlobalBucketAccess().build()); + + return client; } private static ClientConfiguration setProxyConfig(ClientConfiguration conf, AWSProxyConfig proxyConfig) @@ -143,13 +149,4 @@ private static ClientConfiguration setProxyConfig(ClientConfiguration conf, AWSP } return conf; } - - private static EndpointConfiguration toEndpointConfiguration(AWSEndpointConfig endpointConfig) - { - if (StringUtils.isNotEmpty(endpointConfig.getUrl())) { - return new EndpointConfiguration(endpointConfig.getUrl(), endpointConfig.getSigningRegion()); - } else { - return null; - } - } } diff --git a/pom.xml b/pom.xml index aa874f16086d..097673dcb96b 100644 --- a/pom.xml +++ b/pom.xml @@ -69,7 +69,7 @@ 9.3.19.v20170502 1.19.3 - 2.4.6 + 2.6.7 2.5 3.10.6.Final From e8847a1003cba9b3e3d32db5e322b26312855085 Mon Sep 17 00:00:00 2001 From: Jihoon Son Date: Wed, 7 Mar 2018 12:34:48 -0800 Subject: [PATCH 18/21] revert KerberosAuthenticator change --- .../kerberos/KerberosAuthenticator.java | 69 +++++++++++++++++-- 1 file changed, 65 insertions(+), 4 deletions(-) diff --git a/extensions-core/druid-kerberos/src/main/java/io/druid/security/kerberos/KerberosAuthenticator.java b/extensions-core/druid-kerberos/src/main/java/io/druid/security/kerberos/KerberosAuthenticator.java index 66f93ffee98e..dbb142864459 100644 --- a/extensions-core/druid-kerberos/src/main/java/io/druid/security/kerberos/KerberosAuthenticator.java +++ b/extensions-core/druid-kerberos/src/main/java/io/druid/security/kerberos/KerberosAuthenticator.java @@ -73,15 +73,19 @@ import java.io.File; import java.io.IOException; import java.security.Principal; +import java.text.SimpleDateFormat; import java.util.Collections; +import java.util.Date; import java.util.EnumSet; import java.util.HashMap; import java.util.HashSet; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Properties; import java.util.Random; import java.util.Set; +import java.util.TimeZone; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -330,13 +334,13 @@ public Principal getUserPrincipal() }; if (newToken && !token.isExpired() && token != AuthenticationToken.ANONYMOUS) { String signedToken = mySigner.sign(token.toString()); - createAuthCookie( + tokenToAuthCookie( httpResponse, signedToken, getCookieDomain(), getCookiePath(), token.getExpires(), - isCookiePersistent(), + !token.isExpired() && token.getExpires() > 0, isHttps ); } @@ -358,13 +362,13 @@ public Principal getUserPrincipal() } if (unauthorizedResponse) { if (!httpResponse.isCommitted()) { - createAuthCookie( + tokenToAuthCookie( httpResponse, "", getCookieDomain(), getCookiePath(), 0, - isCookiePersistent(), + false, isHttps ); // If response code is 401. Then WWW-Authenticate Header should be @@ -617,4 +621,61 @@ private void initializeKerberosLogin() throws ServletException throw new ServletException(ex); } } + + + /** + * Creates the Hadoop authentication HTTP cookie. + * + * @param resp the response object. + * @param token authentication token for the cookie. + * @param domain the cookie domain. + * @param path the cookie path. + * @param expires UNIX timestamp that indicates the expire date of the + * cookie. It has no effect if its value < 0. + * @param isSecure is the cookie secure? + * @param isCookiePersistent whether the cookie is persistent or not. + *the following code copy/past from Hadoop 3.0.0 copied to avoid compilation issue due to new signature, + * org.apache.hadoop.security.authentication.server.AuthenticationFilter#createAuthCookie + * ( + * javax.servlet.http.HttpServletResponse, + * java.lang.String, + * java.lang.String, + * java.lang.String, + * long, boolean, boolean) + */ + private static void tokenToAuthCookie( + HttpServletResponse resp, String token, + String domain, String path, long expires, + boolean isCookiePersistent, + boolean isSecure + ) + { + StringBuilder sb = new StringBuilder(AuthenticatedURL.AUTH_COOKIE) + .append("="); + if (token != null && token.length() > 0) { + sb.append("\"").append(token).append("\""); + } + + if (path != null) { + sb.append("; Path=").append(path); + } + + if (domain != null) { + sb.append("; Domain=").append(domain); + } + + if (expires >= 0 && isCookiePersistent) { + Date date = new Date(expires); + SimpleDateFormat df = new SimpleDateFormat("EEE, dd-MMM-yyyy HH:mm:ss zzz", Locale.ENGLISH); + df.setTimeZone(TimeZone.getTimeZone("GMT")); + sb.append("; Expires=").append(df.format(date)); + } + + if (isSecure) { + sb.append("; Secure"); + } + + sb.append("; HttpOnly"); + resp.addHeader("Set-Cookie", sb.toString()); + } } From 87742113bfac99cefecd7216424f9d2e7584e69f Mon Sep 17 00:00:00 2001 From: Jihoon Son Date: Wed, 7 Mar 2018 12:37:00 -0800 Subject: [PATCH 19/21] change hadoop-aws scope to provided in hdfs-storage --- extensions-core/hdfs-storage/pom.xml | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/extensions-core/hdfs-storage/pom.xml b/extensions-core/hdfs-storage/pom.xml index a653053d8cf4..4c1a853744fb 100644 --- a/extensions-core/hdfs-storage/pom.xml +++ b/extensions-core/hdfs-storage/pom.xml @@ -145,12 +145,7 @@ org.apache.hadoop hadoop-aws ${hadoop.compile.version} - - - com.amazonaws - aws-java-sdk - - + provided commons-io From b73d61ee52482056b9cc6bab298538ca4d2be7c7 Mon Sep 17 00:00:00 2001 From: Jihoon Son Date: Tue, 13 Mar 2018 15:26:45 -0700 Subject: [PATCH 20/21] address comments --- .../src/main/java/io/druid/common/aws/AWSProxyConfig.java | 2 +- .../src/main/java/io/druid/storage/s3/S3Utils.java | 2 +- .../java/io/druid/indexer/DetermineHashedPartitionsJob.java | 3 +-- server/pom.xml | 4 ++++ .../io/druid/server/AsyncManagementForwardingServlet.java | 2 +- .../java/io/druid/server/AsyncQueryForwardingServlet.java | 2 +- 6 files changed, 9 insertions(+), 6 deletions(-) diff --git a/aws-common/src/main/java/io/druid/common/aws/AWSProxyConfig.java b/aws-common/src/main/java/io/druid/common/aws/AWSProxyConfig.java index 90c7665286f4..eda04bb37152 100644 --- a/aws-common/src/main/java/io/druid/common/aws/AWSProxyConfig.java +++ b/aws-common/src/main/java/io/druid/common/aws/AWSProxyConfig.java @@ -27,7 +27,7 @@ public class AWSProxyConfig private String host; @JsonProperty - private int port = -1; + private int port = -1; // AWS's default proxy port is -1 @JsonProperty private String username; diff --git a/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3Utils.java b/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3Utils.java index 35edd89bb488..c71deeb6c76c 100644 --- a/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3Utils.java +++ b/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3Utils.java @@ -89,7 +89,7 @@ static boolean isObjectInBucketIgnoringPermission(AmazonS3 s3Client, String buck return s3Client.doesObjectExist(bucketName, objectKey); } catch (AmazonS3Exception e) { - if (e.getStatusCode() == 403) { + if (e.getStatusCode() == 404) { // Object is inaccessible to current user, but does exist. return true; } diff --git a/indexing-hadoop/src/main/java/io/druid/indexer/DetermineHashedPartitionsJob.java b/indexing-hadoop/src/main/java/io/druid/indexer/DetermineHashedPartitionsJob.java index 134bac4ce6f4..385c6203c5e9 100644 --- a/indexing-hadoop/src/main/java/io/druid/indexer/DetermineHashedPartitionsJob.java +++ b/indexing-hadoop/src/main/java/io/druid/indexer/DetermineHashedPartitionsJob.java @@ -396,8 +396,7 @@ public static class DetermineHashedPartitionsPartitioner public int getPartition(LongWritable interval, BytesWritable text, int numPartitions) { - final String jobTrackerString = config.get("mapred.job.tracker"); - if ((jobTrackerString != null && jobTrackerString.equals("local")) || determineIntervals) { + if ("local".equals(config.get("mapred.job.tracker")) || determineIntervals) { return 0; } else { return reducerLookup.get(interval); diff --git a/server/pom.xml b/server/pom.xml index 0b9a07a3b9ef..4173650b1b0b 100644 --- a/server/pom.xml +++ b/server/pom.xml @@ -70,6 +70,10 @@ org.apache.zookeeper zookeeper + + org.apache.httpcomponents + httpclient + org.apache.curator curator-framework diff --git a/server/src/main/java/io/druid/server/AsyncManagementForwardingServlet.java b/server/src/main/java/io/druid/server/AsyncManagementForwardingServlet.java index 5e84be355ded..21d952c7449e 100644 --- a/server/src/main/java/io/druid/server/AsyncManagementForwardingServlet.java +++ b/server/src/main/java/io/druid/server/AsyncManagementForwardingServlet.java @@ -19,7 +19,6 @@ package io.druid.server; -import com.amazonaws.thirdparty.apache.http.client.utils.URIBuilder; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Throwables; import com.google.common.collect.ImmutableMap; @@ -34,6 +33,7 @@ import io.druid.java.util.common.StringUtils; import io.druid.java.util.emitter.EmittingLogger; import io.druid.server.security.AuthConfig; +import org.apache.http.client.utils.URIBuilder; import org.eclipse.jetty.client.HttpClient; import org.eclipse.jetty.client.api.Request; import org.eclipse.jetty.proxy.AsyncProxyServlet; diff --git a/server/src/main/java/io/druid/server/AsyncQueryForwardingServlet.java b/server/src/main/java/io/druid/server/AsyncQueryForwardingServlet.java index f93706784ac7..6a20e815cbed 100644 --- a/server/src/main/java/io/druid/server/AsyncQueryForwardingServlet.java +++ b/server/src/main/java/io/druid/server/AsyncQueryForwardingServlet.java @@ -19,7 +19,6 @@ package io.druid.server; -import com.amazonaws.thirdparty.apache.http.client.utils.URIBuilder; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.jaxrs.smile.SmileMediaTypes; @@ -45,6 +44,7 @@ import io.druid.server.router.QueryHostFinder; import io.druid.server.router.Router; import io.druid.server.security.AuthConfig; +import org.apache.http.client.utils.URIBuilder; import org.eclipse.jetty.client.HttpClient; import org.eclipse.jetty.client.api.Request; import org.eclipse.jetty.client.api.Response; From 66dae387b95981aa4981f5353d45369bc95abf9c Mon Sep 17 00:00:00 2001 From: Jihoon Son Date: Tue, 20 Mar 2018 14:44:52 -0700 Subject: [PATCH 21/21] address comments --- extensions-core/s3-extensions/pom.xml | 132 +++++++++--------- .../druid/storage/s3/S3DataSegmentMover.java | 2 +- .../druid/storage/s3/S3DataSegmentPusher.java | 36 ++--- .../java/io/druid/storage/s3/S3Utils.java | 6 +- 4 files changed, 89 insertions(+), 87 deletions(-) diff --git a/extensions-core/s3-extensions/pom.xml b/extensions-core/s3-extensions/pom.xml index ad4e468de670..487d33782542 100644 --- a/extensions-core/s3-extensions/pom.xml +++ b/extensions-core/s3-extensions/pom.xml @@ -18,75 +18,75 @@ - 4.0.0 + 4.0.0 - io.druid.extensions - druid-s3-extensions - druid-s3-extensions - druid-s3-extensions + io.druid.extensions + druid-s3-extensions + druid-s3-extensions + druid-s3-extensions - - io.druid - druid - 0.13.0-SNAPSHOT - ../../pom.xml - + + io.druid + druid + 0.13.0-SNAPSHOT + ../../pom.xml + - - - io.druid - druid-api - ${project.parent.version} - provided - - - io.druid - druid-aws-common - ${project.parent.version} - provided - - - io.druid - java-util - ${project.parent.version} - provided - - - commons-io - commons-io - provided - - - com.fasterxml.jackson.module - jackson-module-guice - ${jackson.version} - provided - + + + io.druid + druid-api + ${project.parent.version} + provided + + + io.druid + druid-aws-common + ${project.parent.version} + provided + + + io.druid + java-util + ${project.parent.version} + provided + + + commons-io + commons-io + provided + + + com.fasterxml.jackson.module + jackson-module-guice + ${jackson.version} + provided + - - - io.druid - druid-server - ${project.parent.version} - test - - - io.druid - druid-processing - ${project.parent.version} - test-jar - test - - - junit - junit - test - - - org.easymock - easymock - test - - + + + io.druid + druid-server + ${project.parent.version} + test + + + io.druid + druid-processing + ${project.parent.version} + test-jar + test + + + junit + junit + test + + + org.easymock + easymock + test + + diff --git a/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3DataSegmentMover.java b/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3DataSegmentMover.java index fdc5a756afc1..e50ea2cca700 100644 --- a/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3DataSegmentMover.java +++ b/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3DataSegmentMover.java @@ -189,7 +189,7 @@ private void selfCheckingMove( log.info("Moving file %s", copyMsg); final CopyObjectRequest copyRequest = new CopyObjectRequest(s3Bucket, s3Path, targetS3Bucket, targetS3Path); if (!config.getDisableAcl()) { - copyRequest.setAccessControlList(S3Utils.grantFullControlToBucketOwver(s3Client, targetS3Bucket)); + copyRequest.setAccessControlList(S3Utils.grantFullControlToBucketOwner(s3Client, targetS3Bucket)); } s3Client.copyObject(copyRequest); if (!s3Client.doesObjectExist(targetS3Bucket, targetS3Path)) { diff --git a/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3DataSegmentPusher.java b/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3DataSegmentPusher.java index 31b650425b72..981d24a7ef5e 100644 --- a/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3DataSegmentPusher.java +++ b/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3DataSegmentPusher.java @@ -96,21 +96,20 @@ public DataSegment push(final File indexFilesDir, final DataSegment inSegment, f final File zipOutFile = File.createTempFile("druid", "index.zip"); final long indexSize = CompressionUtils.zip(indexFilesDir, zipOutFile); + final DataSegment outSegment = inSegment.withSize(indexSize) + .withLoadSpec(makeLoadSpec(config.getBucket(), s3Path)) + .withBinaryVersion(SegmentUtils.getVersionFromDir(indexFilesDir)); + + final File descriptorFile = File.createTempFile("druid", "descriptor.json"); + // Avoid using Guava in DataSegmentPushers because they might be used with very diverse Guava versions in + // runtime, and because Guava deletes methods over time, that causes incompatibilities. + Files.write(descriptorFile.toPath(), jsonMapper.writeValueAsBytes(outSegment)); + try { return S3Utils.retryS3Operation( () -> { - uploadFileIfPossibleAndClear(s3Client, config.getBucket(), s3Path, zipOutFile, replaceExisting); - - final DataSegment outSegment = inSegment.withSize(indexSize) - .withLoadSpec(makeLoadSpec(config.getBucket(), s3Path)) - .withBinaryVersion(SegmentUtils.getVersionFromDir(indexFilesDir)); - - File descriptorFile = File.createTempFile("druid", "descriptor.json"); - // Avoid using Guava in DataSegmentPushers because they might be used with very diverse Guava versions in - // runtime, and because Guava deletes methods over time, that causes incompatibilities. - Files.write(descriptorFile.toPath(), jsonMapper.writeValueAsBytes(outSegment)); - - uploadFileIfPossibleAndClear( + uploadFileIfPossible(s3Client, config.getBucket(), s3Path, zipOutFile, replaceExisting); + uploadFileIfPossible( s3Client, config.getBucket(), S3Utils.descriptorPathForSegmentPath(s3Path), @@ -128,6 +127,12 @@ public DataSegment push(final File indexFilesDir, final DataSegment inSegment, f catch (Exception e) { throw Throwables.propagate(e); } + finally { + log.info("Deleting temporary cached index.zip"); + zipOutFile.delete(); + log.info("Deleting temporary cached descriptor.json"); + descriptorFile.delete(); + } } @Override @@ -155,7 +160,7 @@ private Map makeLoadSpec(String bucket, String key) ); } - private void uploadFileIfPossibleAndClear( + private void uploadFileIfPossible( AmazonS3 s3Client, String bucket, String key, @@ -170,14 +175,11 @@ private void uploadFileIfPossibleAndClear( if (!config.getDisableAcl()) { indexFilePutRequest.setAccessControlList( - S3Utils.grantFullControlToBucketOwver(s3Client, bucket) + S3Utils.grantFullControlToBucketOwner(s3Client, bucket) ); } log.info("Pushing [%s] to bucket[%s] and key[%s].", file, bucket, key); s3Client.putObject(indexFilePutRequest); } - - log.info("Deleting temporary cached file [%s]", file.getAbsolutePath()); - file.delete(); } } diff --git a/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3Utils.java b/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3Utils.java index c71deeb6c76c..c4fa15761066 100644 --- a/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3Utils.java +++ b/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3Utils.java @@ -191,7 +191,7 @@ static String toFilename(String key, final String suffix) return filename; } - static AccessControlList grantFullControlToBucketOwver(AmazonS3 s3Client, String bucket) + static AccessControlList grantFullControlToBucketOwner(AmazonS3 s3Client, String bucket) { final AccessControlList acl = s3Client.getBucketAcl(bucket); acl.grantAllPermissions(new Grant(new CanonicalGrantee(acl.getOwner().getId()), Permission.FullControl)); @@ -232,8 +232,8 @@ public static boolean isDirectoryPlaceholder(String key, ObjectMetadata objectMe } /** - * Gets a single {@link S3ObjectSummary} from s3. Since this method might throw an exception if there are multiple - * objets that match the given key, this method should be used only when it's guaranteed that the given key is unique + * Gets a single {@link S3ObjectSummary} from s3. Since this method might return a wrong object if there are multiple + * objects that match the given key, this method should be used only when it's guaranteed that the given key is unique * in the given bucket. * * @param s3Client s3 client