A collection of PRs to backport to 0.9.0

docs/content/configuration/index.md

@@ -74,6 +74,16 @@ The following path is used for service discovery. It is **not** affected by `dru
 |--------|-----------|-------|
 |`druid.discovery.curator.path`|Services announce themselves under this ZooKeeper path.|`/druid/discovery`|
 
+### Startup Logging
+
+All nodes can log debugging information on startup.
+
+|Property|Description|Default|
+|--------|-----------|-------|
+|`druid.startup.logging.logProperties`|Log all properties on startup (from common.runtime.properties, runtime.properties, and the JVM command line).|false|
+
+Note that some sensitive information may be logged if these settings are enabled.
+
 ### Request Logging
 
 All nodes that can serve queries can also log the query requests they see.

docs/content/ingestion/tasks.md

@@ -124,7 +124,7 @@ The indexSpec is optional and default parameters will be used if not specified.
 |property|description|possible values|default|required?|
 |--------|-----------|---------------|-------|---------|
 |bitmap|type of bitmap compression to use for inverted indices.|`"concise"`, `"roaring"`|`"concise"`|no|
-|dimensionCompression|compression format for dimension columns (currently only affects single-value dimensions, multi-value dimensions are always uncompressed)|`"uncompressed"`, `"lz4"`, `"lzf"`|`"lz4"`|no|
+|dimensionCompression|compression format for dimension columns|`"uncompressed"`, `"lz4"`, `"lzf"`|`"lz4"`|no|
 |metricCompression|compression format for metric columns, defaults to LZ4|`"lz4"`, `"lzf"`|`"lz4"`|no|
 
 Segment Merging Tasks

docs/content/querying/searchquery.md

@@ -34,6 +34,7 @@ There are several main parts to a search query:
 |dataSource|A String or Object defining the data source to query, very similar to a table in a relational database. See [DataSource](../querying/datasource.html) for more information.|yes|
 |granularity|Defines the granularity of the query. See [Granularities](../querying/granularities.html).|yes|
 |filter|See [Filters](../querying/filters.html).|no|
+|limit| Defines the maximum number per historical node (parsed as int) of search results to return. |no (default to 1000)|
 |intervals|A JSON Object representing ISO-8601 Intervals. This defines the time ranges to run the query over.|yes|
 |searchDimensions|The dimensions to run the search over. Excluding this means the search is run over all dimensions.|no|
 |query|See [SearchQuerySpec](../querying/searchqueryspec.html).|yes|

examples/conf-quickstart/druid/_common/common.runtime.properties

@@ -30,6 +30,13 @@ druid.extensions.loadList=[]
 # and uncomment the line below to point to your directory.
 #druid.extensions.hadoopDependenciesDir=/my/dir/hadoop-dependencies
 
+#
+# Logging
+#
+
+# Log all runtime properties on startup. Disable to avoid logging properties on startup:
+druid.startup.logging.logProperties=true
+
 #
 # Zookeeper
 #
@@ -41,6 +48,7 @@ druid.zk.paths.base=/druid
 # Metadata storage
 #
 
+# For Derby server on your Druid Coordinator (only viable in a cluster with a single Coordinator, no fail-over):
 druid.metadata.storage.type=derby
 druid.metadata.storage.connector.connectURI=jdbc:derby://localhost:1527/var/druid/metadata.db;create=true
 druid.metadata.storage.connector.host=localhost
@@ -62,6 +70,7 @@ druid.metadata.storage.connector.port=1527
 # Deep storage
 #
 
+# For local disk (only viable in a cluster if this is a network mount):
 druid.storage.type=local
 druid.storage.storageDirectory=var/druid/segments
 
@@ -80,6 +89,7 @@ druid.storage.storageDirectory=var/druid/segments
 # Indexing service logs
 #
 
+# For local disk (only viable in a cluster if this is a network mount):
 druid.indexer.logs.type=file
 druid.indexer.logs.directory=var/druid/indexing-logs
 
@@ -105,4 +115,4 @@ druid.selectors.coordinator.serviceName=druid/coordinator
 
 druid.monitoring.monitors=["com.metamx.metrics.JvmMonitor"]
 druid.emitter=logging
-druid.emitter.logging.logLevel=debug
+druid.emitter.logging.logLevel=info

examples/conf/druid/_common/common.runtime.properties

@@ -29,6 +29,13 @@ druid.extensions.loadList=["druid-kafka-eight", "druid-s3-extensions", "druid-hi
 # and uncomment the line below to point to your directory.
 #druid.extensions.hadoopDependenciesDir=/my/dir/hadoop-dependencies
 
+#
+# Logging
+#
+
+# Log all runtime properties on startup. Disable to avoid logging properties on startup:
+druid.startup.logging.logProperties=true
+
 #
 # Zookeeper
 #
@@ -40,6 +47,7 @@ druid.zk.paths.base=/druid
 # Metadata storage
 #
 
+# For Derby server on your Druid Coordinator (only viable in a cluster with a single Coordinator, no fail-over):
 druid.metadata.storage.type=derby
 druid.metadata.storage.connector.connectURI=jdbc:derby://metadata.store.ip:1527/var/druid/metadata.db;create=true
 druid.metadata.storage.connector.host=metadata.store.ip
@@ -61,6 +69,7 @@ druid.metadata.storage.connector.port=1527
 # Deep storage
 #
 
+# For local disk (only viable in a cluster if this is a network mount):
 druid.storage.type=local
 druid.storage.storageDirectory=var/druid/segments
 
@@ -79,12 +88,13 @@ druid.storage.storageDirectory=var/druid/segments
 # Indexing service logs
 #
 
+# For local disk (only viable in a cluster if this is a network mount):
 druid.indexer.logs.type=file
 druid.indexer.logs.directory=var/druid/indexing-logs
 
 # For HDFS (make sure to include the HDFS extension and that your Hadoop config files in the cp):
 #druid.indexer.logs.type=hdfs
-#druid.indexer.logs.directory=hdfs://namenode.example.com:9000/druid/indexing-logs
+#druid.indexer.logs.directory=/druid/indexing-logs
 
 # For S3:
 #druid.indexer.logs.type=s3

indexing-service/src/main/java/io/druid/indexing/overlord/setup/EqualDistributionWorkerSelectStrategy.java

@@ -47,7 +47,17 @@ public int compare(
               ImmutableZkWorker zkWorker, ImmutableZkWorker zkWorker2
           )
           {
-            return -Ints.compare(zkWorker2.getCurrCapacityUsed(), zkWorker.getCurrCapacityUsed());
+            int retVal = -Ints.compare(zkWorker2.getCurrCapacityUsed(), zkWorker.getCurrCapacityUsed());
+            // the version sorting is needed because if the workers have the same currCapacityUsed only one of them is
+            // returned. Exists the possibility that this worker is disabled and doesn't have valid version so can't
+            // run new tasks, so in this case the workers are sorted using version to ensure that if exists enable
+            // workers the comparator return one of them.
+
+            if(retVal == 0) {
+              retVal = zkWorker2.getWorker().getVersion().compareTo(zkWorker.getWorker().getVersion());
+            }
+
+            return retVal;
           }
         }
     );

indexing-service/src/main/java/io/druid/indexing/overlord/setup/FillCapacityWorkerSelectStrategy.java

@@ -50,8 +50,13 @@ public int compare(
           )
           {
             int retVal = Ints.compare(zkWorker2.getCurrCapacityUsed(), zkWorker.getCurrCapacityUsed());
+            // the version sorting is needed because if the workers have the same currCapacityUsed only one of them is
+            // returned. Exists the possibility that this worker is disabled and doesn't have valid version so can't
+            // run new tasks, so in this case the workers are sorted using version to ensure that if exists enable
+            // workers the comparator return one of them.
+
             if (retVal == 0) {
-              retVal = zkWorker.getWorker().getHost().compareTo(zkWorker2.getWorker().getHost());
+              retVal = zkWorker.getWorker().getVersion().compareTo(zkWorker2.getWorker().getVersion());
             }
 
             return retVal;

indexing-service/src/test/java/io/druid/indexing/overlord/setup/EqualDistributionWorkerSelectStrategyTest.java

@@ -63,4 +63,70 @@ public String getDataSource()
     ImmutableZkWorker worker = optional.get();
     Assert.assertEquals("lhost", worker.getWorker().getHost());
   }
+
+  @Test
+  public void testOneDisableWorkerDifferentUsedCapacity() throws Exception
+  {
+    String DISABLED_VERSION = "";
+    final EqualDistributionWorkerSelectStrategy strategy = new EqualDistributionWorkerSelectStrategy();
+
+    Optional<ImmutableZkWorker> optional = strategy.findWorkerForTask(
+        new RemoteTaskRunnerConfig(),
+        ImmutableMap.of(
+                      "lhost",
+                      new ImmutableZkWorker(
+                              new Worker("disableHost", "disableHost", 10, DISABLED_VERSION), 2,
+                              Sets.<String>newHashSet()
+                      ),
+                      "localhost",
+                      new ImmutableZkWorker(
+                              new Worker("enableHost", "enableHost", 10, "v1"), 5,
+                              Sets.<String>newHashSet()
+                      )
+        ),
+        new NoopTask(null, 1, 0, null, null, null)
+        {
+          @Override
+          public String getDataSource()
+                  {
+                      return "foo";
+                  }
+        }
+    );
+    ImmutableZkWorker worker = optional.get();
+    Assert.assertEquals("enableHost", worker.getWorker().getHost());
+  }
+
+  @Test
+  public void testOneDisableWorkerSameUsedCapacity() throws Exception
+  {
+    String DISABLED_VERSION = "";
+    final EqualDistributionWorkerSelectStrategy strategy = new EqualDistributionWorkerSelectStrategy();
+
+    Optional<ImmutableZkWorker> optional = strategy.findWorkerForTask(
+            new RemoteTaskRunnerConfig(),
+            ImmutableMap.of(
+                    "lhost",
+                    new ImmutableZkWorker(
+                            new Worker("disableHost", "disableHost", 10, DISABLED_VERSION), 5,
+                            Sets.<String>newHashSet()
+                    ),
+                    "localhost",
+                    new ImmutableZkWorker(
+                            new Worker("enableHost", "enableHost", 10, "v1"), 5,
+                            Sets.<String>newHashSet()
+                    )
+            ),
+            new NoopTask(null, 1, 0, null, null, null)
+            {
+                @Override
+                public String getDataSource()
+                {
+                    return "foo";
+                }
+            }
+    );
+    ImmutableZkWorker worker = optional.get();
+    Assert.assertEquals("enableHost", worker.getWorker().getHost());
+  }
 }

integration-tests/README.md

@@ -3,7 +3,7 @@ Integration Testing
 
 ## Installing Docker
 
-Please refer to instructions at [https://github.com/druid-io/docker-druid/blob/master/docker-install.md]()
+Please refer to instructions at [https://github.com/druid-io/docker-druid/blob/master/docker-install.md](https://github.com/druid-io/docker-druid/blob/master/docker-install.md).
 
 ## Creating the Docker VM
 

integration-tests/src/test/resources/queries/twitterstream_queries.json

@@ -597,6 +597,8 @@
                         "hasMultipleValues": false,
                         "size": 7773438,
                         "cardinality": 2,
+                        "minValue":"No",
+                        "maxValue":"Yes",
                         "errorMessage": null
                     }
                 },
@@ -613,6 +615,8 @@
                         "hasMultipleValues": false,
                         "size": 7901000,
                         "cardinality": 2,
+                        "minValue":"No",
+                        "maxValue":"Yes",
                         "errorMessage": null
                     }
                 },
@@ -629,6 +633,8 @@
                         "hasMultipleValues": false,
                         "size": 7405654,
                         "cardinality": 2,
+                        "minValue":"No",
+                        "maxValue":"Yes",
                         "errorMessage": null
                     }
                 },

integration-tests/src/test/resources/queries/wikipedia_editstream_queries.json

@@ -1048,13 +1048,17 @@
                         "hasMultipleValues": false,
                         "size": 41922148,
                         "cardinality": 208,
+                        "minValue":"",
+                        "maxValue":"mmx._unknown",
                         "errorMessage": null
                     },
                     "language": {
                         "type": "STRING",
                         "hasMultipleValues": false,
                         "size": 8924222,
                         "cardinality": 36,
+                        "minValue":"ar",
+                        "maxValue":"zh",
                         "errorMessage": null
                     }
                 },

processing/src/main/java/io/druid/guice/PropertiesModule.java

@@ -21,6 +21,7 @@
 
 import com.google.common.base.Charsets;
 import com.google.common.base.Throwables;
+import com.google.common.collect.Ordering;
 import com.google.inject.Binder;
 import com.google.inject.Module;
 import com.metamx.common.guava.CloseQuietly;

processing/src/main/java/io/druid/query/aggregation/hyperloglog/HyperLogLogCollector.java

@@ -681,21 +681,6 @@ private static short mergeAndStoreByteRegister(
   @Override
   public int compareTo(HyperLogLogCollector other)
   {
-    final int lhsOffset = (int) this.getRegisterOffset() & 0xffff;
-    final int rhsOffset = (int) other.getRegisterOffset() & 0xffff;
-
-    if (lhsOffset == rhsOffset) {
-      final int lhsNumNonZero = (int) this.getNumNonZeroRegisters() & 0xff;
-      final int rhsNumNonZero = (int) this.getNumNonZeroRegisters() & 0xff;
-      int retVal = Double.compare(lhsNumNonZero, rhsNumNonZero);
-
-      if (retVal == 0) {
-        retVal = Double.compare(this.estimateCardinality(), other.estimateCardinality());
-      }
-
-      return retVal;
-    } else {
-      return Double.compare(lhsOffset, rhsOffset);
-    }
+    return Double.compare(this.estimateCardinality(), other.estimateCardinality());
   }
 }

processing/src/test/java/io/druid/query/aggregation/hyperloglog/HyperLogLogCollectorTest.java

@@ -776,6 +776,45 @@ public void testCompare2() throws Exception
     }
   }
 
+  @Test
+  public void testCompareToShouldBehaveConsistentlyWithEstimatedCardinalitiesEvenInToughCases() throws Exception {
+    // given
+    Random rand = new Random(0);
+    HyperUniquesAggregatorFactory factory = new HyperUniquesAggregatorFactory("foo", "bar");
+    Comparator comparator = factory.getComparator();
+
+    for (int i = 0; i < 1000; ++i) {
+      // given
+      HyperLogLogCollector leftCollector = HyperLogLogCollector.makeLatestCollector();
+      int j = rand.nextInt(9000) + 5000;
+      for (int l = 0; l < j; ++l) {
+        leftCollector.add(fn.hashLong(rand.nextLong()).asBytes());
+      }
+
+      HyperLogLogCollector rightCollector = HyperLogLogCollector.makeLatestCollector();
+      int k = rand.nextInt(9000) + 5000;
+      for (int l = 0; l < k; ++l) {
+        rightCollector.add(fn.hashLong(rand.nextLong()).asBytes());
+      }
+
+      // when
+      final int orderedByCardinality = Double.compare(leftCollector.estimateCardinality(),
+              rightCollector.estimateCardinality());
+      final int orderedByComparator = comparator.compare(leftCollector, rightCollector);
+
+      // then, assert hyperloglog comparator behaves consistently with estimated cardinalities
+      Assert.assertEquals(
+              String.format("orderedByComparator=%d, orderedByCardinality=%d,\n" +
+                      "Left={cardinality=%f, hll=%s},\n" +
+                      "Right={cardinality=%f, hll=%s},\n", orderedByComparator, orderedByCardinality,
+                      leftCollector.estimateCardinality(), leftCollector,
+                      rightCollector.estimateCardinality(), rightCollector),
+              orderedByCardinality,
+              orderedByComparator
+      );
+    }
+  }
+
   @Test
   public void testMaxOverflow() {
     HyperLogLogCollector collector = HyperLogLogCollector.makeLatestCollector();

server/src/main/java/io/druid/guice/StartupLoggingModule.java

@@ -0,0 +1,33 @@
+/*
+ * Licensed to Metamarkets Group Inc. (Metamarkets) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. Metamarkets licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package io.druid.guice;
+
+import com.google.inject.Binder;
+import com.google.inject.Module;
+import io.druid.server.log.StartupLoggingConfig;
+
+public class StartupLoggingModule implements Module
+{
+  @Override
+  public void configure(Binder binder)
+  {
+    JsonConfigProvider.bind(binder, "druid.startup.logging", StartupLoggingConfig.class);
+  }
+}