Backport 6888 to 0.14.0 incubating

core/src/main/java/org/apache/druid/timeline/LogicalSegment.java

@@ -22,8 +22,26 @@
 import org.apache.druid.guice.annotations.PublicApi;
 import org.joda.time.Interval;
 
+/**
+ * A logical segment can represent an entire segment or a part of a segment. As a result, it can have a different
+ * interval from its actual base segment. {@link #getInterval()} and {@link #getTrueInterval()} return the interval of
+ * this logical segment and the interval of the base segment, respectively.
+ *
+ * For example, suppose we have 2 segments as below:
+ *
+ * - Segment A has an interval of 2017/2018.
+ * - Segment B has an interval of 2017-08-01/2017-08-02.
+ *
+ * For these segments, {@link VersionedIntervalTimeline#lookup} returns 3 segments as below:
+ *
+ * - interval of 2017/2017-08-01 (trueInterval: 2017/2018)
+ * - interval of 2017-08-01/2017-08-02 (trueInterval: 2017-08-01/2017-08-02)
+ * - interval of 2017-08-02/2018 (trueInterval: 2017/2018)
+ */
 @PublicApi
 public interface LogicalSegment
 {
   Interval getInterval();
+
+  Interval getTrueInterval();
 }

core/src/main/java/org/apache/druid/timeline/TimelineObjectHolder.java

@@ -19,6 +19,7 @@
 
 package org.apache.druid.timeline;
 
+import com.google.common.annotations.VisibleForTesting;
 import org.apache.druid.timeline.partition.PartitionHolder;
 import org.joda.time.Interval;
 
@@ -27,16 +28,25 @@
 public class TimelineObjectHolder<VersionType, ObjectType> implements LogicalSegment
 {
   private final Interval interval;
+  private final Interval trueInterval;
   private final VersionType version;
   private final PartitionHolder<ObjectType> object;
 
+  @VisibleForTesting
+  public TimelineObjectHolder(Interval interval, VersionType version, PartitionHolder<ObjectType> object)
+  {
+    this(interval, interval, version, object);
+  }
+
   public TimelineObjectHolder(
       Interval interval,
+      Interval trueInterval,
       VersionType version,
       PartitionHolder<ObjectType> object
   )
   {
     this.interval = interval;
+    this.trueInterval = trueInterval;
     this.version = version;
     this.object = object;
   }
@@ -47,6 +57,12 @@ public Interval getInterval()
     return interval;
   }
 
+  @Override
+  public Interval getTrueInterval()
+  {
+    return trueInterval;
+  }
+
   public VersionType getVersion()
   {
     return version;
@@ -62,6 +78,7 @@ public String toString()
   {
     return "TimelineObjectHolder{" +
            "interval=" + interval +
+           ", trueInterval=" + trueInterval +
            ", version=" + version +
            ", object=" + object +
            '}';

core/src/main/java/org/apache/druid/timeline/VersionedIntervalTimeline.java

@@ -300,6 +300,7 @@ public TimelineObjectHolder<VersionType, ObjectType> last()
   private TimelineObjectHolder<VersionType, ObjectType> timelineEntryToObjectHolder(TimelineEntry entry)
   {
     return new TimelineObjectHolder<>(
+        entry.getTrueInterval(),
         entry.getTrueInterval(),
         entry.getVersion(),
         new PartitionHolder<>(entry.getPartitionHolder())
@@ -586,10 +587,11 @@ private List<TimelineObjectHolder<VersionType, ObjectType>> lookup(Interval inte
 
       if (timelineInterval.overlaps(interval)) {
         retVal.add(
-            new TimelineObjectHolder<VersionType, ObjectType>(
+            new TimelineObjectHolder<>(
                 timelineInterval,
+                val.getTrueInterval(),
                 val.getVersion(),
-                new PartitionHolder<ObjectType>(val.getPartitionHolder())
+                new PartitionHolder<>(val.getPartitionHolder())
             )
         );
       }
@@ -604,8 +606,9 @@ private List<TimelineObjectHolder<VersionType, ObjectType>> lookup(Interval inte
                                                                .isAfter(firstEntry.getInterval().getStart())) {
       retVal.set(
           0,
-          new TimelineObjectHolder<VersionType, ObjectType>(
+          new TimelineObjectHolder<>(
               new Interval(interval.getStart(), firstEntry.getInterval().getEnd()),
+              firstEntry.getTrueInterval(),
               firstEntry.getVersion(),
               firstEntry.getObject()
           )
@@ -616,8 +619,9 @@ private List<TimelineObjectHolder<VersionType, ObjectType>> lookup(Interval inte
     if (interval.overlaps(lastEntry.getInterval()) && interval.getEnd().isBefore(lastEntry.getInterval().getEnd())) {
       retVal.set(
           retVal.size() - 1,
-          new TimelineObjectHolder<VersionType, ObjectType>(
+          new TimelineObjectHolder<>(
               new Interval(lastEntry.getInterval().getStart(), interval.getEnd()),
+              lastEntry.getTrueInterval(),
               lastEntry.getVersion(),
               lastEntry.getObject()
           )

extensions-core/druid-kerberos/pom.xml

@@ -71,6 +71,7 @@
       <groupId>org.apache.hadoop</groupId>
       <artifactId>hadoop-common</artifactId>
       <version>${hadoop.compile.version}</version>
+      <scope>compile</scope>
       <exclusions>
         <exclusion>
           <groupId>commons-cli</groupId>

extensions-core/hdfs-storage/pom.xml

@@ -151,6 +151,130 @@
               </exclusion>
             </exclusions>
         </dependency>
+        <dependency>
+          <groupId>org.apache.hadoop</groupId>
+          <artifactId>hadoop-common</artifactId>
+          <version>${hadoop.compile.version}</version>
+          <scope>compile</scope>
+          <exclusions>
+            <exclusion>
+              <groupId>commons-cli</groupId>
+              <artifactId>commons-cli</artifactId>
+            </exclusion>
+            <exclusion>
+              <groupId>commons-httpclient</groupId>
+              <artifactId>commons-httpclient</artifactId>
+            </exclusion>
+            <exclusion>
+              <groupId>log4j</groupId>
+              <artifactId>log4j</artifactId>
+            </exclusion>
+            <exclusion>
+              <groupId>commons-codec</groupId>
+              <artifactId>commons-codec</artifactId>
+            </exclusion>
+            <exclusion>
+              <groupId>commons-logging</groupId>
+              <artifactId>commons-logging</artifactId>
+            </exclusion>
+            <exclusion>
+              <groupId>commons-io</groupId>
+              <artifactId>commons-io</artifactId>
+            </exclusion>
+            <exclusion>
+              <groupId>commons-lang</groupId>
+              <artifactId>commons-lang</artifactId>
+            </exclusion>
+            <exclusion>
+              <groupId>org.apache.httpcomponents</groupId>
+              <artifactId>httpclient</artifactId>
+            </exclusion>
+            <exclusion>
+              <groupId>org.apache.httpcomponents</groupId>
+              <artifactId>httpcore</artifactId>
+            </exclusion>
+            <exclusion>
+              <groupId>org.codehaus.jackson</groupId>
+              <artifactId>jackson-core-asl</artifactId>
+            </exclusion>
+            <exclusion>
+              <groupId>org.codehaus.jackson</groupId>
+              <artifactId>jackson-mapper-asl</artifactId>
+            </exclusion>
+            <exclusion>
+              <groupId>org.apache.zookeeper</groupId>
+              <artifactId>zookeeper</artifactId>
+            </exclusion>
+            <exclusion>
+              <groupId>org.slf4j</groupId>
+              <artifactId>slf4j-api</artifactId>
+            </exclusion>
+            <exclusion>
+              <groupId>org.slf4j</groupId>
+              <artifactId>slf4j-log4j12</artifactId>
+            </exclusion>
+            <exclusion>
+              <groupId>javax.ws.rs</groupId>
+              <artifactId>jsr311-api</artifactId>
+            </exclusion>
+            <exclusion>
+              <groupId>com.google.code.findbugs</groupId>
+              <artifactId>jsr305</artifactId>
+            </exclusion>
+            <exclusion>
+              <groupId>org.mortbay.jetty</groupId>
+              <artifactId>jetty-util</artifactId>
+            </exclusion>
+            <exclusion>
+              <groupId>org.apache.hadoop</groupId>
+              <artifactId>hadoop-annotations</artifactId>
+            </exclusion>
+            <exclusion>
+              <groupId>com.google.protobuf</groupId>
+              <artifactId>protobuf-java</artifactId>
+            </exclusion>
+            <exclusion>
+              <groupId>com.sun.jersey</groupId>
+              <artifactId>jersey-core</artifactId>
+            </exclusion>
+            <exclusion>
+              <groupId>org.apache.curator</groupId>
+              <artifactId>curator-client</artifactId>
+            </exclusion>
+            <exclusion>
+              <groupId>org.apache.commons</groupId>
+              <artifactId>commons-math3</artifactId>
+            </exclusion>
+            <exclusion>
+              <groupId>com.google.guava</groupId>
+              <artifactId>guava</artifactId>
+            </exclusion>
+            <exclusion>
+              <groupId>org.apache.avro</groupId>
+              <artifactId>avro</artifactId>
+            </exclusion>
+            <exclusion>
+              <groupId>net.java.dev.jets3t</groupId>
+              <artifactId>jets3t</artifactId>
+            </exclusion>
+            <exclusion>
+              <groupId>com.sun.jersey</groupId>
+              <artifactId>jersey-json</artifactId>
+            </exclusion>
+            <exclusion>
+              <groupId>com.jcraft</groupId>
+              <artifactId>jsch</artifactId>
+            </exclusion>
+            <exclusion>
+              <groupId>org.mortbay.jetty</groupId>
+              <artifactId>jetty</artifactId>
+            </exclusion>
+            <exclusion>
+              <groupId>com.sun.jersey</groupId>
+              <artifactId>jersey-server</artifactId>
+            </exclusion>
+          </exclusions>
+        </dependency>
         <dependency>
           <groupId>org.apache.hadoop</groupId>
           <artifactId>hadoop-aws</artifactId>
@@ -164,6 +288,13 @@
         </dependency>
 
         <!-- Tests -->
+        <dependency>
+          <groupId>org.apache.hadoop</groupId>
+          <artifactId>hadoop-common</artifactId>
+          <version>${hadoop.compile.version}</version>
+          <classifier>tests</classifier>
+          <scope>test</scope>
+        </dependency>
         <dependency>
           <groupId>junit</groupId>
           <artifactId>junit</artifactId>
@@ -189,13 +320,6 @@
           <classifier>tests</classifier>
           <scope>test</scope>
         </dependency>
-        <dependency>
-          <groupId>org.apache.hadoop</groupId>
-          <artifactId>hadoop-common</artifactId>
-          <version>${hadoop.compile.version}</version>
-          <classifier>tests</classifier>
-          <scope>test</scope>
-        </dependency>
         <dependency>
           <groupId>org.apache.hadoop</groupId>
           <artifactId>hadoop-hdfs</artifactId>

extensions-core/histogram/src/main/java/org/apache/druid/query/aggregation/histogram/BucketsPostAggregator.java

@@ -82,7 +82,7 @@ public float getBucketSize()
   @JsonProperty
   public float getOffset()
   {
-    return bucketSize;
+    return offset;
   }
 
   @Override

extensions-core/histogram/src/test/java/org/apache/druid/query/aggregation/histogram/BucketsPostAggregatorTest.java

@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.query.aggregation.histogram;
+
+import org.apache.druid.jackson.DefaultObjectMapper;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class BucketsPostAggregatorTest
+{
+  @Test
+  public void testSerde() throws Exception
+  {
+    BucketsPostAggregator aggregator1 =
+        new BucketsPostAggregator("buckets_post_aggregator", "test_field", 2f, 4f);
+
+    DefaultObjectMapper mapper = new DefaultObjectMapper();
+    BucketsPostAggregator aggregator2 = mapper.readValue(
+        mapper.writeValueAsString(aggregator1),
+        BucketsPostAggregator.class
+    );
+
+    Assert.assertEquals(aggregator1.getBucketSize(), aggregator2.getBucketSize(), 0.0001);
+    Assert.assertEquals(aggregator1.getOffset(), aggregator2.getOffset(), 0.0001);
+    Assert.assertArrayEquals(aggregator1.getCacheKey(), aggregator2.getCacheKey());
+    Assert.assertEquals(aggregator1.getDependentFields(), aggregator2.getDependentFields());
+  }
+}

extensions-core/s3-extensions/src/main/java/org/apache/druid/storage/s3/S3DataSegmentMover.java

@@ -174,7 +174,10 @@ private void selfCheckingMove(
               .withPrefix(s3Path)
               .withMaxKeys(1)
       );
-      if (listResult.getKeyCount() == 0) {
+      // Using getObjectSummaries().size() instead of getKeyCount as, in some cases
+      // it is observed that even though the getObjectSummaries returns some data
+      // keyCount is still zero.
+      if (listResult.getObjectSummaries().size() == 0) {
         // should never happen
         throw new ISE("Unable to list object [s3://%s/%s]", s3Bucket, s3Path);
       }

extensions-core/s3-extensions/src/main/java/org/apache/druid/storage/s3/S3Utils.java

@@ -251,7 +251,10 @@ public static S3ObjectSummary getSingleObjectSummary(ServerSideEncryptingAmazonS
         .withMaxKeys(1);
     final ListObjectsV2Result result = s3Client.listObjectsV2(request);
 
-    if (result.getKeyCount() == 0) {
+    // Using getObjectSummaries().size() instead of getKeyCount as, in some cases
+    // it is observed that even though the getObjectSummaries returns some data
+    // keyCount is still zero.
+    if (result.getObjectSummaries().size() == 0) {
       throw new ISE("Cannot find object for bucket[%s] and key[%s]", bucket, key);
     }
     final S3ObjectSummary objectSummary = result.getObjectSummaries().get(0);