Expanding the VirtualColumn concept.

common/src/main/java/io/druid/math/expr/Evals.java

@@ -20,7 +20,6 @@
 package io.druid.math.expr;
 
 import com.google.common.base.Strings;
-import io.druid.common.guava.GuavaUtils;
 import io.druid.java.util.common.logger.Logger;
 
 import java.util.Arrays;
@@ -32,27 +31,6 @@ public class Evals
 {
   private static final Logger log = new Logger(Evals.class);
 
-  public static Number toNumber(Object value)
-  {
-    if (value == null) {
-      return 0L;
-    }
-    if (value instanceof Number) {
-      return (Number) value;
-    }
-    String stringValue = String.valueOf(value);
-    Long longValue = GuavaUtils.tryParseLong(stringValue);
-    if (longValue == null) {
-      return Double.valueOf(stringValue);
-    }
-    return longValue;
-  }
-
-  public static boolean isConstant(Expr expr)
-  {
-    return expr instanceof ConstantExpr;
-  }
-
   public static boolean isAllConstants(Expr... exprs)
   {
     return isAllConstants(Arrays.asList(exprs));

extensions-contrib/virtual-columns/src/main/java/io/druid/segment/MapVirtualColumn.java

@@ -22,21 +22,24 @@
 import com.fasterxml.jackson.annotation.JsonCreator;
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
 import com.google.common.collect.Maps;
 import com.metamx.common.StringUtils;
 import io.druid.query.dimension.DefaultDimensionSpec;
+import io.druid.query.dimension.DimensionSpec;
 import io.druid.query.filter.DimFilterUtils;
+import io.druid.segment.column.ValueType;
 import io.druid.segment.data.IndexedInts;
+import io.druid.segment.virtual.VirtualColumnCacheHelper;
 
 import java.nio.ByteBuffer;
+import java.util.List;
 import java.util.Map;
 
 /**
  */
 public class MapVirtualColumn implements VirtualColumn
 {
-  private static final byte VC_TYPE_ID = 0x00;
-
   private final String outputName;
   private final String keyDimension;
   private final String valueDimension;
@@ -58,13 +61,14 @@ public MapVirtualColumn(
   }
 
   @Override
-  public ObjectColumnSelector init(String dimension, ColumnSelectorFactory factory)
+  public ObjectColumnSelector makeObjectColumnSelector(String dimension, ColumnSelectorFactory factory)
   {
     final DimensionSelector keySelector = factory.makeDimensionSelector(DefaultDimensionSpec.of(keyDimension));
     final DimensionSelector valueSelector = factory.makeDimensionSelector(DefaultDimensionSpec.of(valueDimension));
 
-    int index = dimension.indexOf('.');
-    if (index < 0) {
+    final String subColumnName = VirtualColumns.splitColumnName(dimension).rhs;
+
+    if (subColumnName == null) {
       return new ObjectColumnSelector<Map>()
       {
         @Override
@@ -94,7 +98,7 @@ public Map get()
       };
     }
 
-    final int keyId = keySelector.lookupId(dimension.substring(index + 1));
+    final int keyId = keySelector.lookupId(subColumnName);
 
     return new ObjectColumnSelector<String>()
     {
@@ -123,6 +127,37 @@ public String get()
     };
   }
 
+  @Override
+  public DimensionSelector makeDimensionSelector(DimensionSpec dimensionSpec, ColumnSelectorFactory factory)
+  {
+    // Could probably do something useful here if the column name is dot-style. But for now just return nothing.
+    return null;
+  }
+
+  @Override
+  public FloatColumnSelector makeFloatColumnSelector(String columnName, ColumnSelectorFactory factory)
+  {
+    return null;
+  }
+
+  @Override
+  public LongColumnSelector makeLongColumnSelector(String columnName, ColumnSelectorFactory factory)
+  {
+    return null;
+  }
+
+  @Override
+  public ValueType nativeType(String columnName)
+  {
+    return columnName.indexOf('.') < 0 ? ValueType.COMPLEX : ValueType.STRING;
+  }
+
+  @Override
+  public List<String> requiredColumns()
+  {
+    return ImmutableList.of(keyDimension, valueDimension);
+  }
+
   @Override
   public boolean usesDotNotation()
   {
@@ -137,7 +172,7 @@ public byte[] getCacheKey()
     byte[] output = StringUtils.toUtf8(outputName);
 
     return ByteBuffer.allocate(3 + key.length + value.length + output.length)
-                     .put(VC_TYPE_ID)
+                     .put(VirtualColumnCacheHelper.CACHE_TYPE_ID_MAP)
                      .put(key).put(DimFilterUtils.STRING_SEPARATOR)
                      .put(value).put(DimFilterUtils.STRING_SEPARATOR)
                      .put(output)

indexing-hadoop/src/main/java/io/druid/indexer/InputRowSerde.java

@@ -36,6 +36,7 @@
 import io.druid.segment.incremental.IncrementalIndex;
 import io.druid.segment.serde.ComplexMetricSerde;
 import io.druid.segment.serde.ComplexMetrics;
+import io.druid.segment.VirtualColumns;
 import org.apache.hadoop.io.ArrayWritable;
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.io.WritableUtils;
@@ -87,10 +88,10 @@ public InputRow get()
 
         Aggregator agg = aggFactory.factorize(
             IncrementalIndex.makeColumnSelectorFactory(
+                VirtualColumns.EMPTY,
                 aggFactory,
                 supplier,
-                true,
-                null
+                true
             )
         );
         try {

processing/src/main/java/io/druid/query/Druids.java

@@ -55,6 +55,7 @@
 import io.druid.query.timeboundary.TimeBoundaryResultValue;
 import io.druid.query.timeseries.TimeseriesQuery;
 import io.druid.segment.VirtualColumn;
+import io.druid.segment.VirtualColumns;
 import org.joda.time.DateTime;
 import org.joda.time.Interval;
 
@@ -330,18 +331,20 @@ public static class TimeseriesQueryBuilder
   {
     private DataSource dataSource;
     private QuerySegmentSpec querySegmentSpec;
+    private boolean descending;
+    private VirtualColumns virtualColumns;
     private DimFilter dimFilter;
     private QueryGranularity granularity;
     private List<AggregatorFactory> aggregatorSpecs;
     private List<PostAggregator> postAggregatorSpecs;
     private Map<String, Object> context;
 
-    private boolean descending;
-
     private TimeseriesQueryBuilder()
     {
       dataSource = null;
       querySegmentSpec = null;
+      descending = false;
+      virtualColumns = null;
       dimFilter = null;
       granularity = QueryGranularities.ALL;
       aggregatorSpecs = Lists.newArrayList();
@@ -355,6 +358,7 @@ public TimeseriesQuery build()
           dataSource,
           querySegmentSpec,
           descending,
+          virtualColumns,
           dimFilter,
           granularity,
           aggregatorSpecs,
@@ -459,6 +463,22 @@ public TimeseriesQueryBuilder intervals(List<Interval> l)
       return this;
     }
 
+    public TimeseriesQueryBuilder virtualColumns(VirtualColumns virtualColumns)
+    {
+      this.virtualColumns = virtualColumns;
+      return this;
+    }
+
+    public TimeseriesQueryBuilder virtualColumns(List<VirtualColumn> virtualColumns)
+    {
+      return virtualColumns(VirtualColumns.create(virtualColumns));
+    }
+
+    public TimeseriesQueryBuilder virtualColumns(VirtualColumn... virtualColumns)
+    {
+      return virtualColumns(VirtualColumns.create(Arrays.asList(virtualColumns)));
+    }
+
     public TimeseriesQueryBuilder filters(String dimensionName, String value)
     {
       dimFilter = new SelectorDimFilter(dimensionName, value, null);
@@ -1104,7 +1124,7 @@ public static class SelectQueryBuilder
     private QueryGranularity granularity;
     private List<DimensionSpec> dimensions;
     private List<String> metrics;
-    private List<VirtualColumn> virtualColumns;
+    private VirtualColumns virtualColumns;
     private PagingSpec pagingSpec;
 
     public SelectQueryBuilder()
@@ -1233,12 +1253,22 @@ public SelectQueryBuilder metrics(List<String> m)
       return this;
     }
 
-    public SelectQueryBuilder virtualColumns(List<VirtualColumn> vcs)
+    public SelectQueryBuilder virtualColumns(VirtualColumns vcs)
     {
       virtualColumns = vcs;
       return this;
     }
 
+    public SelectQueryBuilder virtualColumns(List<VirtualColumn> vcs)
+    {
+      return virtualColumns(VirtualColumns.create(vcs));
+    }
+
+    public SelectQueryBuilder virtualColumns(VirtualColumn... vcs)
+    {
+      return virtualColumns(VirtualColumns.create(Arrays.asList(vcs)));
+    }
+
     public SelectQueryBuilder pagingSpec(PagingSpec p)
     {
       pagingSpec = p;

processing/src/main/java/io/druid/query/aggregation/AggregatorUtil.java

@@ -20,10 +20,6 @@
 package io.druid.query.aggregation;
 
 import com.google.common.collect.Lists;
-import io.druid.segment.ColumnSelectorFactory;
-import io.druid.segment.FloatColumnSelector;
-import io.druid.segment.LongColumnSelector;
-import io.druid.segment.NumericColumnSelector;
 import io.druid.java.util.common.Pair;
 
 import java.util.HashSet;
@@ -88,54 +84,4 @@ public static Pair<List<AggregatorFactory>, List<PostAggregator>> condensedAggre
     }
     return new Pair(condensedAggs, condensedPostAggs);
   }
-
-  public static FloatColumnSelector getFloatColumnSelector(
-      final ColumnSelectorFactory metricFactory,
-      final String fieldName,
-      final String fieldExpression,
-      final float nullValue
-  )
-  {
-    if (fieldName != null && fieldExpression == null) {
-      return metricFactory.makeFloatColumnSelector(fieldName);
-    }
-    if (fieldName == null && fieldExpression != null) {
-      final NumericColumnSelector numeric = metricFactory.makeMathExpressionSelector(fieldExpression);
-      return new FloatColumnSelector()
-      {
-        @Override
-        public float get()
-        {
-          final Number number = numeric.get();
-          return number == null ? nullValue : number.floatValue();
-        }
-      };
-    }
-    throw new IllegalArgumentException("Must have a valid, non-null fieldName or expression");
-  }
-
-  public static LongColumnSelector getLongColumnSelector(
-      final ColumnSelectorFactory metricFactory,
-      final String fieldName,
-      final String fieldExpression,
-      final long nullValue
-  )
-  {
-    if (fieldName != null && fieldExpression == null) {
-      return metricFactory.makeLongColumnSelector(fieldName);
-    }
-    if (fieldName == null && fieldExpression != null) {
-      final NumericColumnSelector numeric = metricFactory.makeMathExpressionSelector(fieldExpression);
-      return new LongColumnSelector()
-      {
-        @Override
-        public long get()
-        {
-          final Number number = numeric.get();
-          return number == null ? nullValue : number.longValue();
-        }
-      };
-    }
-    throw new IllegalArgumentException("Must have a valid, non-null fieldName or expression");
-  }
 }