revert ColumnAnalysis type, add typeSignature and use it for DruidSchema

docs/querying/segmentmetadataquery.md

@@ -87,9 +87,11 @@ The format of the result is:
 } ]
 ```
 
-Dimension columns will have type `STRING`, `FLOAT`, `DOUBLE`, or `LONG`.
-Metric columns will have type `FLOAT`, `DOUBLE`, or `LONG`, or the name of the underlying complex type such as `hyperUnique` in case of COMPLEX metric.
-Timestamp column will have type `LONG`.
+All columns contain a `typeSignature` that Druid uses to represent the column type information internally. The `typeSignature` is typically the same value used to identify the JSON type information at query or ingest time. One of: `STRING`, `FLOAT`, `DOUBLE`, `LONG`, or `COMPLEX<typeName>`, e.g. `COMPLEX<hyperUnique>`.
+
+Columns also have a legacy `type` name. For some column types, the value may match the `typeSignature`  (`STRING`, `FLOAT`, `DOUBLE`, or `LONG`). For `COMPLEX` columns, the `type` only contains the name of the underlying complex type such as `hyperUnique`.
+
+New applications should use `typeSignature`, not `type`.
 
 If the `errorMessage` field is non-null, you should not trust the other fields in the response. Their contents are
 undefined.

integration-tests/src/test/resources/hadoop/batch_hadoop_queries.json

@@ -15,6 +15,7 @@
         "id": "merged",
         "columns": {
           "location": {
+            "typeSignature": "STRING",
             "type": "STRING",
             "size": 0,
             "hasMultipleValues": false,
@@ -25,6 +26,7 @@
             "errorMessage": null
           },
           "user_id_sketch": {
+            "typeSignature": "COMPLEX<thetaSketch>",
             "type": "thetaSketch",
             "size": 0,
             "hasMultipleValues": false,
@@ -35,6 +37,7 @@
             "errorMessage": null
           },
           "other_metric": {
+            "typeSignature": "COMPLEX<thetaSketch>",
             "type": "thetaSketch",
             "size": 0,
             "hasMultipleValues": false,
@@ -45,6 +48,7 @@
             "errorMessage": null
           },
           "__time": {
+            "typeSignature": "LONG",
             "type": "LONG",
             "size": 0,
             "hasMultipleValues": false,
@@ -55,6 +59,7 @@
             "errorMessage": null
           },
           "product": {
+            "typeSignature": "STRING",
             "type": "STRING",
             "size": 0,
             "hasMultipleValues": false,

integration-tests/src/test/resources/queries/twitterstream_queries.json

@@ -596,6 +596,7 @@
                 "intervals":["2013-01-01T00:00:00.000Z/2013-01-02T00:00:00.000Z"],
                 "columns":{
                     "has_links":{
+                        "typeSignature": "STRING",
                         "type":"STRING",
                         "hasMultipleValues":false,
                         "size":0,
@@ -618,6 +619,7 @@
                 "intervals":["2013-01-02T00:00:00.000Z/2013-01-03T00:00:00.000Z"],
                 "columns":{
                     "has_links":{
+                        "typeSignature": "STRING",
                         "type":"STRING",
                         "hasMultipleValues":false,
                         "size":0,
@@ -640,6 +642,7 @@
                 "intervals":["2013-01-03T00:00:00.000Z/2013-01-04T00:00:00.000Z"],
                 "columns":{
                     "has_links":{
+                        "typeSignature": "STRING",
                         "type":"STRING",
                         "hasMultipleValues":false,
                         "size":0,

integration-tests/src/test/resources/queries/wikipedia_editstream_queries.json

@@ -1402,6 +1402,7 @@
                 "intervals":["2012-12-29T00:00:00.000Z/2013-01-10T08:00:00.000Z"],
                 "columns":{
                     "country_name":{
+                        "typeSignature": "STRING",
                         "type":"STRING",
                         "hasMultipleValues":false,
                         "size":0,
@@ -1412,6 +1413,7 @@
                         "hasNulls":true
                     },
                     "language":{
+                        "typeSignature": "STRING",
                         "type":"STRING",
                         "hasMultipleValues":false,
                         "size":0,

processing/src/main/java/org/apache/druid/query/metadata/SegmentAnalyzer.java

@@ -43,6 +43,7 @@
 import org.apache.druid.segment.column.ColumnCapabilitiesImpl;
 import org.apache.druid.segment.column.ColumnHolder;
 import org.apache.druid.segment.column.ColumnType;
+import org.apache.druid.segment.column.ColumnTypeFactory;
 import org.apache.druid.segment.column.ComplexColumn;
 import org.apache.druid.segment.column.DictionaryEncodedColumn;
 import org.apache.druid.segment.column.TypeSignature;
@@ -194,7 +195,8 @@ private ColumnAnalysis analyzeNumericColumn(
     }
 
     return new ColumnAnalysis(
-        capabilities.asTypeString(),
+        capabilities.toColumnType(),
+        capabilities.getType().name(),
         capabilities.hasMultipleValues().isTrue(),
         capabilities.hasNulls().isMaybeTrue(), // if we don't know for sure, then we should plan to check for nulls
         size,
@@ -250,7 +252,8 @@ private ColumnAnalysis analyzeStringColumn(
     }
 
     return new ColumnAnalysis(
-        capabilities.asTypeString(),
+        capabilities.toColumnType(),
+        capabilities.getType().name(),
         capabilities.hasMultipleValues().isTrue(),
         capabilities.hasNulls().isMaybeTrue(), // if we don't know for sure, then we should plan to check for nulls
         size,
@@ -328,7 +331,8 @@ public Long accumulate(Long accumulated, Cursor cursor)
     }
 
     return new ColumnAnalysis(
-        capabilities.asTypeString(),
+        capabilities.toColumnType(),
+        capabilities.getType().name(),
         capabilities.hasMultipleValues().isTrue(),
         capabilities.hasNulls().isMaybeTrue(), // if we don't know for sure, then we should plan to check for nulls
         size,
@@ -347,8 +351,6 @@ private ColumnAnalysis analyzeComplexColumn(
     final TypeSignature<ValueType> typeSignature = capabilities == null ? ColumnType.UNKNOWN_COMPLEX : capabilities;
     final String typeName = typeSignature.getComplexTypeName();
 
-    // serialize using asTypeString (which is also used for JSON so can easily round-trip complex type info back into ColumnType)
-    final String serdeTypeName = typeSignature.asTypeString();
     try (final ComplexColumn complexColumn = columnHolder != null ? (ComplexColumn) columnHolder.getColumn() : null) {
       final boolean hasMultipleValues = capabilities != null && capabilities.hasMultipleValues().isTrue();
       final boolean hasNulls = capabilities != null && capabilities.hasNulls().isMaybeTrue();
@@ -363,7 +365,8 @@ private ColumnAnalysis analyzeComplexColumn(
         final Function<Object, Long> inputSizeFn = serde.inputSizeFn();
         if (inputSizeFn == null) {
           return new ColumnAnalysis(
-              serdeTypeName,
+              ColumnTypeFactory.ofType(typeSignature),
+              typeName,
               hasMultipleValues,
               hasNulls,
               0,
@@ -381,7 +384,8 @@ private ColumnAnalysis analyzeComplexColumn(
       }
 
       return new ColumnAnalysis(
-          serdeTypeName,
+          ColumnTypeFactory.ofType(typeSignature),
+          typeName,
           hasMultipleValues,
           hasNulls,
           size,

processing/src/main/java/org/apache/druid/query/metadata/SegmentMetadataQueryQueryToolChest.java

@@ -23,7 +23,6 @@
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Function;
 import com.google.common.base.Functions;
-import com.google.common.base.Predicate;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.Iterables;
 import com.google.common.collect.Lists;
@@ -58,8 +57,6 @@
 import org.joda.time.DateTime;
 import org.joda.time.Interval;
 
-import javax.annotation.Nullable;
-import java.nio.ByteBuffer;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Comparator;
@@ -75,16 +72,10 @@ public class SegmentMetadataQueryQueryToolChest extends QueryToolChest<SegmentAn
   private static final TypeReference<SegmentAnalysis> TYPE_REFERENCE = new TypeReference<SegmentAnalysis>()
   {
   };
-  private static final byte[] SEGMENT_METADATA_CACHE_PREFIX = new byte[]{0x4};
+  private static final byte SEGMENT_METADATA_CACHE_PREFIX = 0x4;
   private static final byte SEGMENT_METADATA_QUERY = 0x16;
-  private static final Function<SegmentAnalysis, SegmentAnalysis> MERGE_TRANSFORM_FN = new Function<SegmentAnalysis, SegmentAnalysis>()
-  {
-    @Override
-    public SegmentAnalysis apply(SegmentAnalysis analysis)
-    {
-      return finalizeAnalysis(analysis);
-    }
-  };
+  private static final Function<SegmentAnalysis, SegmentAnalysis> MERGE_TRANSFORM_FN =
+      SegmentMetadataQueryQueryToolChest::finalizeAnalysis;
 
   private final SegmentMetadataQueryConfig config;
   private final GenericQueryMetricsFactory queryMetricsFactory;
@@ -195,13 +186,9 @@ public boolean isCacheable(SegmentMetadataQuery query, boolean willMergeRunners)
       public byte[] computeCacheKey(SegmentMetadataQuery query)
       {
         SegmentMetadataQuery updatedQuery = query.withFinalizedAnalysisTypes(config);
-        byte[] includerBytes = updatedQuery.getToInclude().getCacheKey();
-        byte[] analysisTypesBytes = updatedQuery.getAnalysisTypesCacheKey();
-        return ByteBuffer.allocate(1 + includerBytes.length + analysisTypesBytes.length)
-                         .put(SEGMENT_METADATA_CACHE_PREFIX)
-                         .put(includerBytes)
-                         .put(analysisTypesBytes)
-                         .array();
+        return new CacheKeyBuilder(SEGMENT_METADATA_CACHE_PREFIX).appendCacheable(updatedQuery.getToInclude())
+                                                                 .appendCacheables(updatedQuery.getAnalysisTypes())
+                                                                 .build();
       }
 
       @Override
@@ -223,27 +210,13 @@ public TypeReference<SegmentAnalysis> getCacheObjectClazz()
       @Override
       public Function<SegmentAnalysis, SegmentAnalysis> prepareForCache(boolean isResultLevelCache)
       {
-        return new Function<SegmentAnalysis, SegmentAnalysis>()
-        {
-          @Override
-          public SegmentAnalysis apply(@Nullable SegmentAnalysis input)
-          {
-            return input;
-          }
-        };
+        return input -> input;
       }
 
       @Override
       public Function<SegmentAnalysis, SegmentAnalysis> pullFromCache(boolean isResultLevelCache)
       {
-        return new Function<SegmentAnalysis, SegmentAnalysis>()
-        {
-          @Override
-          public SegmentAnalysis apply(@Nullable SegmentAnalysis input)
-          {
-            return input;
-          }
-        };
+        return input -> input;
       }
     };
   }
@@ -266,14 +239,7 @@ public <T extends LogicalSegment> List<T> filterSegments(SegmentMetadataQuery qu
     return Lists.newArrayList(
         Iterables.filter(
             segments,
-            new Predicate<T>()
-            {
-              @Override
-              public boolean apply(T input)
-              {
-                return (input.getInterval().overlaps(targetInterval));
-              }
-            }
+            input -> (input.getInterval().overlaps(targetInterval))
         )
     );
   }

processing/src/main/java/org/apache/druid/query/metadata/metadata/ColumnAnalysis.java

@@ -22,6 +22,8 @@
 import com.fasterxml.jackson.annotation.JsonCreator;
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.fasterxml.jackson.annotation.JsonTypeInfo;
+import org.apache.druid.java.util.common.StringUtils;
+import org.apache.druid.segment.column.ColumnType;
 
 import java.util.Objects;
 
@@ -33,10 +35,11 @@ public class ColumnAnalysis
 
   public static ColumnAnalysis error(String reason)
   {
-    return new ColumnAnalysis("STRING", false, false, -1, null, null, null, ERROR_PREFIX + reason);
+    return new ColumnAnalysis(ColumnType.STRING, "STRING", false, false, -1, null, null, null, ERROR_PREFIX + reason);
   }
 
   private final String type;
+  private final ColumnType typeSignature;
   private final boolean hasMultipleValues;
   private final boolean hasNulls;
   private final long size;
@@ -47,6 +50,7 @@ public static ColumnAnalysis error(String reason)
 
   @JsonCreator
   public ColumnAnalysis(
+      @JsonProperty("typeSignature") ColumnType typeSignature,
       @JsonProperty("type") String type,
       @JsonProperty("hasMultipleValues") boolean hasMultipleValues,
       @JsonProperty("hasNulls") boolean hasNulls,
@@ -57,6 +61,7 @@ public ColumnAnalysis(
       @JsonProperty("errorMessage") String errorMessage
   )
   {
+    this.typeSignature = typeSignature;
     this.type = type;
     this.hasMultipleValues = hasMultipleValues;
     this.hasNulls = hasNulls;
@@ -68,6 +73,13 @@ public ColumnAnalysis(
   }
 
   @JsonProperty
+  public ColumnType getTypeSignature()
+  {
+    return typeSignature;
+  }
+
+  @JsonProperty
+  @Deprecated
   public String getType()
   {
     return type;
@@ -135,8 +147,20 @@ public ColumnAnalysis fold(ColumnAnalysis rhs)
       return rhs;
     }
 
-    if (!type.equals(rhs.getType())) {
-      return ColumnAnalysis.error("cannot_merge_diff_types");
+    if (!Objects.equals(type, rhs.getType())) {
+      return ColumnAnalysis.error(
+          StringUtils.format("cannot_merge_diff_types: [%s] and [%s]", type, rhs.getType())
+      );
+    }
+
+    if (!Objects.equals(typeSignature, rhs.getTypeSignature())) {
+      return ColumnAnalysis.error(
+          StringUtils.format(
+              "cannot_merge_diff_types: [%s] and [%s]",
+              typeSignature.asTypeString(),
+              rhs.getTypeSignature().asTypeString()
+          )
+      );
     }
 
     Integer cardinality = getCardinality();
@@ -153,6 +177,7 @@ public ColumnAnalysis fold(ColumnAnalysis rhs)
     Comparable newMax = choose(maxValue, rhs.maxValue, true);
 
     return new ColumnAnalysis(
+        typeSignature,
         type,
         multipleValues,
         hasNulls || rhs.hasNulls,
@@ -180,7 +205,8 @@ private <T extends Comparable> T choose(T obj1, T obj2, boolean max)
   public String toString()
   {
     return "ColumnAnalysis{" +
-           "type='" + type + '\'' +
+           "typeSignature='" + typeSignature + '\'' +
+           ", type=" + type +
            ", hasMultipleValues=" + hasMultipleValues +
            ", hasNulls=" + hasNulls +
            ", size=" + size +
@@ -204,6 +230,7 @@ public boolean equals(Object o)
     return hasMultipleValues == that.hasMultipleValues &&
            hasNulls == that.hasNulls &&
            size == that.size &&
+           Objects.equals(typeSignature, that.typeSignature) &&
            Objects.equals(type, that.type) &&
            Objects.equals(cardinality, that.cardinality) &&
            Objects.equals(minValue, that.minValue) &&
@@ -214,6 +241,16 @@ public boolean equals(Object o)
   @Override
   public int hashCode()
   {
-    return Objects.hash(type, hasMultipleValues, hasNulls, size, cardinality, minValue, maxValue, errorMessage);
+    return Objects.hash(
+        typeSignature,
+        type,
+        hasMultipleValues,
+        hasNulls,
+        size,
+        cardinality,
+        minValue,
+        maxValue,
+        errorMessage
+    );
   }
 }

processing/src/main/java/org/apache/druid/query/metadata/metadata/ColumnIncluderator.java

@@ -21,6 +21,7 @@
 
 import com.fasterxml.jackson.annotation.JsonSubTypes;
 import com.fasterxml.jackson.annotation.JsonTypeInfo;
+import org.apache.druid.java.util.common.Cacheable;
 
 /**
  */
@@ -30,8 +31,7 @@
     @JsonSubTypes.Type(name = "all", value = AllColumnIncluderator.class),
     @JsonSubTypes.Type(name = "list", value = ListColumnIncluderator.class)
 })
-public interface ColumnIncluderator
+public interface ColumnIncluderator extends Cacheable
 {
   boolean include(String columnName);
-  byte[] getCacheKey();
 }