From 7402a28fc345b60f79b60edc818ce755dc845c4d Mon Sep 17 00:00:00 2001 From: Gian Merlino Date: Tue, 6 Dec 2016 16:28:26 -0800 Subject: [PATCH 1/4] Expanding the VirtualColumn concept. After this patch, virtual columns work well for aggregating, but not for grouping (because the topN and groupBy engines don't deal well with DimensionSelectors that have no dictionaries) or filtering (because the CursorFactories don't do anything useful with them). Virtual columns: - Move VirtualColumn and related classes to their own package. - Add additional methods to VirtualColumn to support more kinds of selectors and to support cycle detection. - Add ExpressionVirtualColumn in core, offering math expressions as a virtual column. Queries: - Add VirtualColumns to timeseries, topN, and groupBy. Aggregators: - Remove the "expression" parameter (replaced by "expression" virtual columns). ColumnSelectorFactory: - Change getColumnCapabilities to getNativeType, since that's all anyone was using it for, and it's simpler. - Remove "makeMathExpressionSelector" (replaced by "expression" virtual columns). Storage adapters: - Add virtual column hooks before checking base columns. I think this ordering makes more sense than checking base columns first, since it prevents surprises when a new base column is added that happens to have the same name as a virtual column. --- .../benchmark/FilterPartitionBenchmark.java | 2 +- .../IncrementalIndexReadBenchmark.java | 2 +- .../main/java/io/druid/math/expr/Evals.java | 22 - .../io/druid/segment/MapVirtualColumn.java | 51 ++- .../druid/segment/MapVirtualColumnTest.java | 1 + .../java/io/druid/indexer/InputRowSerde.java | 5 +- .../src/main/java/io/druid/query/Druids.java | 40 +- .../io/druid/query/QueryRunnerHelper.java | 2 +- .../query/aggregation/AggregatorUtil.java | 54 --- .../DoubleMaxAggregatorFactory.java | 49 +-- .../DoubleMinAggregatorFactory.java | 49 +-- .../DoubleSumAggregatorFactory.java | 49 +-- .../FilteredAggregatorFactory.java | 18 +- .../aggregation/LongMaxAggregatorFactory.java | 49 +-- .../aggregation/LongMinAggregatorFactory.java | 49 +-- .../aggregation/LongSumAggregatorFactory.java | 49 +-- .../io/druid/query/groupby/GroupByQuery.java | 94 +++-- .../query/groupby/GroupByQueryEngine.java | 3 +- .../groupby/GroupByQueryQueryToolChest.java | 3 + .../epinephelinae/GroupByQueryEngineV2.java | 3 +- .../epinephelinae/RowBasedGrouperHelper.java | 56 +-- .../groupby/strategy/GroupByStrategyV1.java | 1 + .../groupby/strategy/GroupByStrategyV2.java | 1 + .../druid/query/metadata/SegmentAnalyzer.java | 2 +- .../druid/query/search/SearchQueryRunner.java | 4 +- .../io/druid/query/select/SelectQuery.java | 10 +- .../druid/query/select/SelectQueryEngine.java | 3 +- .../select/SelectQueryQueryToolChest.java | 22 +- .../TimeBoundaryQueryRunnerFactory.java | 2 +- .../query/timeseries/TimeseriesQuery.java | 51 ++- .../timeseries/TimeseriesQueryEngine.java | 3 +- .../TimeseriesQueryQueryToolChest.java | 10 +- .../AggregateTopNMetricFirstAlgorithm.java | 8 +- .../java/io/druid/query/topn/TopNQuery.java | 71 ++-- .../io/druid/query/topn/TopNQueryBuilder.java | 63 ++- .../io/druid/query/topn/TopNQueryEngine.java | 9 +- .../query/topn/TopNQueryQueryToolChest.java | 7 +- .../druid/segment/ColumnSelectorFactory.java | 22 +- .../java/io/druid/segment/CursorFactory.java | 1 + .../druid/segment/NullDimensionSelector.java | 22 +- .../segment/QueryableIndexStorageAdapter.java | 141 ++----- .../java/io/druid/segment/VirtualColumn.java | 66 --- .../java/io/druid/segment/VirtualColumns.java | 80 ---- .../segment/ZeroFloatColumnSelector.java | 41 ++ .../druid/segment/ZeroLongColumnSelector.java | 41 ++ .../io/druid/segment/column/ValueType.java | 14 +- .../segment/incremental/IncrementalIndex.java | 81 ++-- .../incremental/IncrementalIndexSchema.java | 19 +- .../IncrementalIndexStorageAdapter.java | 117 ++---- .../incremental/OffheapIncrementalIndex.java | 30 +- .../incremental/OnheapIncrementalIndex.java | 16 +- .../ExpressionObjectColumnSelector.java | 141 +++++++ .../virtual/ExpressionVirtualColumn.java | 199 ++++++++++ .../druid/segment/virtual/VirtualColumn.java | 140 +++++++ .../virtual/VirtualColumnCacheHelper.java | 34 ++ .../druid/segment/virtual/VirtualColumns.java | 275 +++++++++++++ .../VirtualizedColumnSelectorFactory.java | 91 +++++ .../io/druid/query/SchemaEvolutionTest.java | 8 +- .../aggregation/FilteredAggregatorTest.java | 23 +- .../aggregation/JavaScriptAggregatorTest.java | 11 +- .../query/groupby/GroupByQueryRunnerTest.java | 90 ++++- .../TestColumnSelectorFactory.java | 11 +- .../query/select/SelectQuerySpecTest.java | 2 +- .../TimeseriesQueryQueryToolChestTest.java | 2 + .../topn/TopNQueryQueryToolChestTest.java | 2 + .../druid/query/topn/TopNQueryRunnerTest.java | 16 +- .../segment/NullDimensionSelectorTest.java | 2 +- .../test/java/io/druid/segment/TestIndex.java | 11 +- .../druid/segment/filter/BaseFilterTest.java | 2 +- .../IncrementalIndexMultiValueSpecTest.java | 2 + .../IncrementalIndexStorageAdapterTest.java | 4 +- .../OnheapIncrementalIndexBenchmark.java | 4 +- .../segment/virtual/VirtualColumnsTest.java | 375 ++++++++++++++++++ .../firehose/IngestSegmentFirehose.java | 2 +- .../main/java/io/druid/cli/DumpSegment.java | 2 +- 75 files changed, 2004 insertions(+), 1053 deletions(-) delete mode 100644 processing/src/main/java/io/druid/segment/VirtualColumn.java delete mode 100644 processing/src/main/java/io/druid/segment/VirtualColumns.java create mode 100644 processing/src/main/java/io/druid/segment/ZeroFloatColumnSelector.java create mode 100644 processing/src/main/java/io/druid/segment/ZeroLongColumnSelector.java create mode 100644 processing/src/main/java/io/druid/segment/virtual/ExpressionObjectColumnSelector.java create mode 100644 processing/src/main/java/io/druid/segment/virtual/ExpressionVirtualColumn.java create mode 100644 processing/src/main/java/io/druid/segment/virtual/VirtualColumn.java create mode 100644 processing/src/main/java/io/druid/segment/virtual/VirtualColumnCacheHelper.java create mode 100644 processing/src/main/java/io/druid/segment/virtual/VirtualColumns.java create mode 100644 processing/src/main/java/io/druid/segment/virtual/VirtualizedColumnSelectorFactory.java create mode 100644 processing/src/test/java/io/druid/segment/virtual/VirtualColumnsTest.java diff --git a/benchmarks/src/main/java/io/druid/benchmark/FilterPartitionBenchmark.java b/benchmarks/src/main/java/io/druid/benchmark/FilterPartitionBenchmark.java index 825940f221c2..b5ff728555d1 100644 --- a/benchmarks/src/main/java/io/druid/benchmark/FilterPartitionBenchmark.java +++ b/benchmarks/src/main/java/io/druid/benchmark/FilterPartitionBenchmark.java @@ -61,7 +61,7 @@ import io.druid.segment.QueryableIndex; import io.druid.segment.QueryableIndexStorageAdapter; import io.druid.segment.StorageAdapter; -import io.druid.segment.VirtualColumns; +import io.druid.segment.virtual.VirtualColumns; import io.druid.segment.column.Column; import io.druid.segment.column.ColumnConfig; import io.druid.segment.data.IndexedInts; diff --git a/benchmarks/src/main/java/io/druid/benchmark/indexing/IncrementalIndexReadBenchmark.java b/benchmarks/src/main/java/io/druid/benchmark/indexing/IncrementalIndexReadBenchmark.java index 6cafa63ec464..7a6a768f1ed2 100644 --- a/benchmarks/src/main/java/io/druid/benchmark/indexing/IncrementalIndexReadBenchmark.java +++ b/benchmarks/src/main/java/io/druid/benchmark/indexing/IncrementalIndexReadBenchmark.java @@ -45,7 +45,7 @@ import io.druid.query.search.search.ContainsSearchQuerySpec; import io.druid.segment.Cursor; import io.druid.segment.DimensionSelector; -import io.druid.segment.VirtualColumns; +import io.druid.segment.virtual.VirtualColumns; import io.druid.segment.data.IndexedInts; import io.druid.segment.incremental.IncrementalIndex; import io.druid.segment.incremental.IncrementalIndexSchema; diff --git a/common/src/main/java/io/druid/math/expr/Evals.java b/common/src/main/java/io/druid/math/expr/Evals.java index 702037309a91..1ff14385738b 100644 --- a/common/src/main/java/io/druid/math/expr/Evals.java +++ b/common/src/main/java/io/druid/math/expr/Evals.java @@ -20,7 +20,6 @@ package io.druid.math.expr; import com.google.common.base.Strings; -import io.druid.common.guava.GuavaUtils; import io.druid.java.util.common.logger.Logger; import java.util.Arrays; @@ -32,27 +31,6 @@ public class Evals { private static final Logger log = new Logger(Evals.class); - public static Number toNumber(Object value) - { - if (value == null) { - return 0L; - } - if (value instanceof Number) { - return (Number) value; - } - String stringValue = String.valueOf(value); - Long longValue = GuavaUtils.tryParseLong(stringValue); - if (longValue == null) { - return Double.valueOf(stringValue); - } - return longValue; - } - - public static boolean isConstant(Expr expr) - { - return expr instanceof ConstantExpr; - } - public static boolean isAllConstants(Expr... exprs) { return isAllConstants(Arrays.asList(exprs)); diff --git a/extensions-contrib/virtual-columns/src/main/java/io/druid/segment/MapVirtualColumn.java b/extensions-contrib/virtual-columns/src/main/java/io/druid/segment/MapVirtualColumn.java index 75fe13ad6c82..659912f24004 100644 --- a/extensions-contrib/virtual-columns/src/main/java/io/druid/segment/MapVirtualColumn.java +++ b/extensions-contrib/virtual-columns/src/main/java/io/druid/segment/MapVirtualColumn.java @@ -22,21 +22,26 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; import com.google.common.collect.Maps; import com.metamx.common.StringUtils; import io.druid.query.dimension.DefaultDimensionSpec; +import io.druid.query.dimension.DimensionSpec; import io.druid.query.filter.DimFilterUtils; +import io.druid.segment.column.ValueType; import io.druid.segment.data.IndexedInts; +import io.druid.segment.virtual.VirtualColumn; +import io.druid.segment.virtual.VirtualColumnCacheHelper; +import io.druid.segment.virtual.VirtualColumns; import java.nio.ByteBuffer; +import java.util.List; import java.util.Map; /** */ public class MapVirtualColumn implements VirtualColumn { - private static final byte VC_TYPE_ID = 0x00; - private final String outputName; private final String keyDimension; private final String valueDimension; @@ -58,13 +63,14 @@ public MapVirtualColumn( } @Override - public ObjectColumnSelector init(String dimension, ColumnSelectorFactory factory) + public ObjectColumnSelector makeObjectColumnSelector(String dimension, ColumnSelectorFactory factory) { final DimensionSelector keySelector = factory.makeDimensionSelector(DefaultDimensionSpec.of(keyDimension)); final DimensionSelector valueSelector = factory.makeDimensionSelector(DefaultDimensionSpec.of(valueDimension)); - int index = dimension.indexOf('.'); - if (index < 0) { + final String subColumnName = VirtualColumns.splitColumnName(dimension).rhs; + + if (subColumnName == null) { return new ObjectColumnSelector() { @Override @@ -94,7 +100,7 @@ public Map get() }; } - final int keyId = keySelector.lookupId(dimension.substring(index + 1)); + final int keyId = keySelector.lookupId(subColumnName); return new ObjectColumnSelector() { @@ -123,6 +129,37 @@ public String get() }; } + @Override + public DimensionSelector makeDimensionSelector(DimensionSpec dimensionSpec, ColumnSelectorFactory factory) + { + // Could probably do something useful here if the column name is dot-style. But for now just return nothing. + return null; + } + + @Override + public FloatColumnSelector makeFloatColumnSelector(String columnName, ColumnSelectorFactory factory) + { + return null; + } + + @Override + public LongColumnSelector makeLongColumnSelector(String columnName, ColumnSelectorFactory factory) + { + return null; + } + + @Override + public ValueType nativeType(String columnName) + { + return columnName.indexOf('.') < 0 ? ValueType.COMPLEX : ValueType.STRING; + } + + @Override + public List requiredColumns() + { + return ImmutableList.of(keyDimension, valueDimension); + } + @Override public boolean usesDotNotation() { @@ -137,7 +174,7 @@ public byte[] getCacheKey() byte[] output = StringUtils.toUtf8(outputName); return ByteBuffer.allocate(3 + key.length + value.length + output.length) - .put(VC_TYPE_ID) + .put(VirtualColumnCacheHelper.CACHE_TYPE_ID_MAP) .put(key).put(DimFilterUtils.STRING_SEPARATOR) .put(value).put(DimFilterUtils.STRING_SEPARATOR) .put(output) diff --git a/extensions-contrib/virtual-columns/src/test/java/io/druid/segment/MapVirtualColumnTest.java b/extensions-contrib/virtual-columns/src/test/java/io/druid/segment/MapVirtualColumnTest.java index 2f3ff107883e..c7ca1a93e068 100644 --- a/extensions-contrib/virtual-columns/src/test/java/io/druid/segment/MapVirtualColumnTest.java +++ b/extensions-contrib/virtual-columns/src/test/java/io/druid/segment/MapVirtualColumnTest.java @@ -44,6 +44,7 @@ import io.druid.segment.incremental.IncrementalIndex; import io.druid.segment.incremental.IncrementalIndexSchema; import io.druid.segment.incremental.OnheapIncrementalIndex; +import io.druid.segment.virtual.VirtualColumn; import org.joda.time.DateTime; import org.junit.Assert; import org.junit.Test; diff --git a/indexing-hadoop/src/main/java/io/druid/indexer/InputRowSerde.java b/indexing-hadoop/src/main/java/io/druid/indexer/InputRowSerde.java index 4ef717dacd4f..9831a1ecb997 100644 --- a/indexing-hadoop/src/main/java/io/druid/indexer/InputRowSerde.java +++ b/indexing-hadoop/src/main/java/io/druid/indexer/InputRowSerde.java @@ -36,6 +36,7 @@ import io.druid.segment.incremental.IncrementalIndex; import io.druid.segment.serde.ComplexMetricSerde; import io.druid.segment.serde.ComplexMetrics; +import io.druid.segment.virtual.VirtualColumns; import org.apache.hadoop.io.ArrayWritable; import org.apache.hadoop.io.Text; import org.apache.hadoop.io.WritableUtils; @@ -87,10 +88,10 @@ public InputRow get() Aggregator agg = aggFactory.factorize( IncrementalIndex.makeColumnSelectorFactory( + VirtualColumns.EMPTY, aggFactory, supplier, - true, - null + true ) ); try { diff --git a/processing/src/main/java/io/druid/query/Druids.java b/processing/src/main/java/io/druid/query/Druids.java index dfa7d238fb50..80f5d647951d 100644 --- a/processing/src/main/java/io/druid/query/Druids.java +++ b/processing/src/main/java/io/druid/query/Druids.java @@ -54,7 +54,8 @@ import io.druid.query.timeboundary.TimeBoundaryQuery; import io.druid.query.timeboundary.TimeBoundaryResultValue; import io.druid.query.timeseries.TimeseriesQuery; -import io.druid.segment.VirtualColumn; +import io.druid.segment.virtual.VirtualColumn; +import io.druid.segment.virtual.VirtualColumns; import org.joda.time.DateTime; import org.joda.time.Interval; @@ -330,18 +331,20 @@ public static class TimeseriesQueryBuilder { private DataSource dataSource; private QuerySegmentSpec querySegmentSpec; + private boolean descending; + private VirtualColumns virtualColumns; private DimFilter dimFilter; private QueryGranularity granularity; private List aggregatorSpecs; private List postAggregatorSpecs; private Map context; - private boolean descending; - private TimeseriesQueryBuilder() { dataSource = null; querySegmentSpec = null; + descending = false; + virtualColumns = null; dimFilter = null; granularity = QueryGranularities.ALL; aggregatorSpecs = Lists.newArrayList(); @@ -355,6 +358,7 @@ public TimeseriesQuery build() dataSource, querySegmentSpec, descending, + virtualColumns, dimFilter, granularity, aggregatorSpecs, @@ -459,6 +463,22 @@ public TimeseriesQueryBuilder intervals(List l) return this; } + public TimeseriesQueryBuilder virtualColumns(VirtualColumns virtualColumns) + { + this.virtualColumns = virtualColumns; + return this; + } + + public TimeseriesQueryBuilder virtualColumns(List virtualColumns) + { + return virtualColumns(VirtualColumns.create(virtualColumns)); + } + + public TimeseriesQueryBuilder virtualColumns(VirtualColumn... virtualColumns) + { + return virtualColumns(VirtualColumns.create(Arrays.asList(virtualColumns))); + } + public TimeseriesQueryBuilder filters(String dimensionName, String value) { dimFilter = new SelectorDimFilter(dimensionName, value, null); @@ -1104,7 +1124,7 @@ public static class SelectQueryBuilder private QueryGranularity granularity; private List dimensions; private List metrics; - private List virtualColumns; + private VirtualColumns virtualColumns; private PagingSpec pagingSpec; public SelectQueryBuilder() @@ -1233,12 +1253,22 @@ public SelectQueryBuilder metrics(List m) return this; } - public SelectQueryBuilder virtualColumns(List vcs) + public SelectQueryBuilder virtualColumns(VirtualColumns vcs) { virtualColumns = vcs; return this; } + public SelectQueryBuilder virtualColumns(List vcs) + { + return virtualColumns(VirtualColumns.create(vcs)); + } + + public SelectQueryBuilder virtualColumns(VirtualColumn... vcs) + { + return virtualColumns(VirtualColumns.create(Arrays.asList(vcs))); + } + public SelectQueryBuilder pagingSpec(PagingSpec p) { pagingSpec = p; diff --git a/processing/src/main/java/io/druid/query/QueryRunnerHelper.java b/processing/src/main/java/io/druid/query/QueryRunnerHelper.java index 0d1cd979d9c9..2b15b012ef55 100644 --- a/processing/src/main/java/io/druid/query/QueryRunnerHelper.java +++ b/processing/src/main/java/io/druid/query/QueryRunnerHelper.java @@ -30,7 +30,7 @@ import io.druid.query.filter.Filter; import io.druid.segment.Cursor; import io.druid.segment.StorageAdapter; -import io.druid.segment.VirtualColumns; +import io.druid.segment.virtual.VirtualColumns; import org.joda.time.Interval; import java.io.Closeable; diff --git a/processing/src/main/java/io/druid/query/aggregation/AggregatorUtil.java b/processing/src/main/java/io/druid/query/aggregation/AggregatorUtil.java index e106ef828c59..f16ddcce38ac 100644 --- a/processing/src/main/java/io/druid/query/aggregation/AggregatorUtil.java +++ b/processing/src/main/java/io/druid/query/aggregation/AggregatorUtil.java @@ -20,10 +20,6 @@ package io.druid.query.aggregation; import com.google.common.collect.Lists; -import io.druid.segment.ColumnSelectorFactory; -import io.druid.segment.FloatColumnSelector; -import io.druid.segment.LongColumnSelector; -import io.druid.segment.NumericColumnSelector; import io.druid.java.util.common.Pair; import java.util.HashSet; @@ -88,54 +84,4 @@ public static Pair, List> condensedAggre } return new Pair(condensedAggs, condensedPostAggs); } - - public static FloatColumnSelector getFloatColumnSelector( - final ColumnSelectorFactory metricFactory, - final String fieldName, - final String fieldExpression, - final float nullValue - ) - { - if (fieldName != null && fieldExpression == null) { - return metricFactory.makeFloatColumnSelector(fieldName); - } - if (fieldName == null && fieldExpression != null) { - final NumericColumnSelector numeric = metricFactory.makeMathExpressionSelector(fieldExpression); - return new FloatColumnSelector() - { - @Override - public float get() - { - final Number number = numeric.get(); - return number == null ? nullValue : number.floatValue(); - } - }; - } - throw new IllegalArgumentException("Must have a valid, non-null fieldName or expression"); - } - - public static LongColumnSelector getLongColumnSelector( - final ColumnSelectorFactory metricFactory, - final String fieldName, - final String fieldExpression, - final long nullValue - ) - { - if (fieldName != null && fieldExpression == null) { - return metricFactory.makeLongColumnSelector(fieldName); - } - if (fieldName == null && fieldExpression != null) { - final NumericColumnSelector numeric = metricFactory.makeMathExpressionSelector(fieldExpression); - return new LongColumnSelector() - { - @Override - public long get() - { - final Number number = numeric.get(); - return number == null ? nullValue : number.longValue(); - } - }; - } - throw new IllegalArgumentException("Must have a valid, non-null fieldName or expression"); - } } diff --git a/processing/src/main/java/io/druid/query/aggregation/DoubleMaxAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/DoubleMaxAggregatorFactory.java index 36c3073e2a1c..871f0fe8a16e 100644 --- a/processing/src/main/java/io/druid/query/aggregation/DoubleMaxAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/DoubleMaxAggregatorFactory.java @@ -22,11 +22,10 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; import com.google.common.primitives.Doubles; import io.druid.common.utils.StringUtils; -import io.druid.math.expr.Parser; import io.druid.segment.ColumnSelectorFactory; -import io.druid.segment.FloatColumnSelector; import java.nio.ByteBuffer; import java.util.Arrays; @@ -42,46 +41,30 @@ public class DoubleMaxAggregatorFactory extends AggregatorFactory private final String name; private final String fieldName; - private final String expression; @JsonCreator public DoubleMaxAggregatorFactory( @JsonProperty("name") String name, - @JsonProperty("fieldName") final String fieldName, - @JsonProperty("expression") String expression + @JsonProperty("fieldName") final String fieldName ) { Preconditions.checkNotNull(name, "Must have a valid, non-null aggregator name"); - Preconditions.checkArgument( - fieldName == null ^ expression == null, - "Must have a valid, non-null fieldName or expression" - ); + Preconditions.checkNotNull(fieldName, "Must have a valid, non-null fieldName"); this.name = name; this.fieldName = fieldName; - this.expression = expression; - } - - public DoubleMaxAggregatorFactory(String name, String fieldName) - { - this(name, fieldName, null); } @Override public Aggregator factorize(ColumnSelectorFactory metricFactory) { - return new DoubleMaxAggregator(getFloatColumnSelector(metricFactory)); + return new DoubleMaxAggregator(metricFactory.makeFloatColumnSelector(fieldName)); } @Override public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) { - return new DoubleMaxBufferAggregator(getFloatColumnSelector(metricFactory)); - } - - private FloatColumnSelector getFloatColumnSelector(ColumnSelectorFactory metricFactory) - { - return AggregatorUtil.getFloatColumnSelector(metricFactory, fieldName, expression, Float.MIN_VALUE); + return new DoubleMaxBufferAggregator(metricFactory.makeFloatColumnSelector(fieldName)); } @Override @@ -99,7 +82,7 @@ public Object combine(Object lhs, Object rhs) @Override public AggregatorFactory getCombiningFactory() { - return new DoubleMaxAggregatorFactory(name, name, null); + return new DoubleMaxAggregatorFactory(name, name); } @Override @@ -115,7 +98,7 @@ public AggregatorFactory getMergingFactory(AggregatorFactory other) throws Aggre @Override public List getRequiredColumns() { - return Arrays.asList(new DoubleMaxAggregatorFactory(fieldName, fieldName, expression)); + return Arrays.asList(new DoubleMaxAggregatorFactory(fieldName, fieldName)); } @Override @@ -140,12 +123,6 @@ public String getFieldName() return fieldName; } - @JsonProperty - public String getExpression() - { - return expression; - } - @Override @JsonProperty public String getName() @@ -156,20 +133,17 @@ public String getName() @Override public List requiredFields() { - return fieldName != null ? Arrays.asList(fieldName) : Parser.findRequiredBindings(expression); + return ImmutableList.of(fieldName); } @Override public byte[] getCacheKey() { byte[] fieldNameBytes = StringUtils.toUtf8WithNullToEmpty(fieldName); - byte[] expressionBytes = StringUtils.toUtf8WithNullToEmpty(expression); - return ByteBuffer.allocate(2 + fieldNameBytes.length + expressionBytes.length) + return ByteBuffer.allocate(1 + fieldNameBytes.length) .put(CACHE_TYPE_ID) .put(fieldNameBytes) - .put(AggregatorUtil.STRING_SEPARATOR) - .put(expressionBytes) .array(); } @@ -190,7 +164,6 @@ public String toString() { return "DoubleMaxAggregatorFactory{" + "fieldName='" + fieldName + '\'' + - ", expression='" + expression + '\'' + ", name='" + name + '\'' + '}'; } @@ -210,9 +183,6 @@ public boolean equals(Object o) if (!Objects.equals(fieldName, that.fieldName)) { return false; } - if (!Objects.equals(expression, that.expression)) { - return false; - } if (!Objects.equals(name, that.name)) { return false; } @@ -224,7 +194,6 @@ public boolean equals(Object o) public int hashCode() { int result = fieldName != null ? fieldName.hashCode() : 0; - result = 31 * result + (expression != null ? expression.hashCode() : 0); result = 31 * result + (name != null ? name.hashCode() : 0); return result; } diff --git a/processing/src/main/java/io/druid/query/aggregation/DoubleMinAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/DoubleMinAggregatorFactory.java index bec58003a329..6fddde2d3a98 100644 --- a/processing/src/main/java/io/druid/query/aggregation/DoubleMinAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/DoubleMinAggregatorFactory.java @@ -22,11 +22,10 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; import com.google.common.primitives.Doubles; import io.druid.common.utils.StringUtils; -import io.druid.math.expr.Parser; import io.druid.segment.ColumnSelectorFactory; -import io.druid.segment.FloatColumnSelector; import java.nio.ByteBuffer; import java.util.Arrays; @@ -42,46 +41,30 @@ public class DoubleMinAggregatorFactory extends AggregatorFactory private final String name; private final String fieldName; - private final String expression; @JsonCreator public DoubleMinAggregatorFactory( @JsonProperty("name") String name, - @JsonProperty("fieldName") final String fieldName, - @JsonProperty("expression") String expression + @JsonProperty("fieldName") final String fieldName ) { Preconditions.checkNotNull(name, "Must have a valid, non-null aggregator name"); - Preconditions.checkArgument( - fieldName == null ^ expression == null, - "Must have a valid, non-null fieldName or expression" - ); + Preconditions.checkNotNull(fieldName, "Must have a valid, non-null fieldName"); this.name = name; this.fieldName = fieldName; - this.expression = expression; - } - - public DoubleMinAggregatorFactory(String name, String fieldName) - { - this(name, fieldName, null); } @Override public Aggregator factorize(ColumnSelectorFactory metricFactory) { - return new DoubleMinAggregator(getFloatColumnSelector(metricFactory)); + return new DoubleMinAggregator(metricFactory.makeFloatColumnSelector(fieldName)); } @Override public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) { - return new DoubleMinBufferAggregator(getFloatColumnSelector(metricFactory)); - } - - private FloatColumnSelector getFloatColumnSelector(ColumnSelectorFactory metricFactory) - { - return AggregatorUtil.getFloatColumnSelector(metricFactory, fieldName, expression, Float.MAX_VALUE); + return new DoubleMinBufferAggregator(metricFactory.makeFloatColumnSelector(fieldName)); } @Override @@ -99,7 +82,7 @@ public Object combine(Object lhs, Object rhs) @Override public AggregatorFactory getCombiningFactory() { - return new DoubleMinAggregatorFactory(name, name, null); + return new DoubleMinAggregatorFactory(name, name); } @Override @@ -115,7 +98,7 @@ public AggregatorFactory getMergingFactory(AggregatorFactory other) throws Aggre @Override public List getRequiredColumns() { - return Arrays.asList(new DoubleMinAggregatorFactory(fieldName, fieldName, expression)); + return Arrays.asList(new DoubleMinAggregatorFactory(fieldName, fieldName)); } @Override @@ -140,12 +123,6 @@ public String getFieldName() return fieldName; } - @JsonProperty - public String getExpression() - { - return expression; - } - @Override @JsonProperty public String getName() @@ -156,20 +133,17 @@ public String getName() @Override public List requiredFields() { - return fieldName != null ? Arrays.asList(fieldName) : Parser.findRequiredBindings(expression); + return ImmutableList.of(fieldName); } @Override public byte[] getCacheKey() { byte[] fieldNameBytes = StringUtils.toUtf8WithNullToEmpty(fieldName); - byte[] expressionBytes = StringUtils.toUtf8WithNullToEmpty(expression); - return ByteBuffer.allocate(2 + fieldNameBytes.length + expressionBytes.length) + return ByteBuffer.allocate(1 + fieldNameBytes.length) .put(CACHE_TYPE_ID) .put(fieldNameBytes) - .put(AggregatorUtil.STRING_SEPARATOR) - .put(expressionBytes) .array(); } @@ -190,7 +164,6 @@ public String toString() { return "DoubleMinAggregatorFactory{" + "fieldName='" + fieldName + '\'' + - ", expression='" + expression + '\'' + ", name='" + name + '\'' + '}'; } @@ -210,9 +183,6 @@ public boolean equals(Object o) if (!Objects.equals(fieldName, that.fieldName)) { return false; } - if (!Objects.equals(expression, that.expression)) { - return false; - } if (!Objects.equals(name, that.name)) { return false; } @@ -224,7 +194,6 @@ public boolean equals(Object o) public int hashCode() { int result = fieldName != null ? fieldName.hashCode() : 0; - result = 31 * result + (expression != null ? expression.hashCode() : 0); result = 31 * result + (name != null ? name.hashCode() : 0); return result; } diff --git a/processing/src/main/java/io/druid/query/aggregation/DoubleSumAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/DoubleSumAggregatorFactory.java index fa2b172fea83..538d128aa0aa 100644 --- a/processing/src/main/java/io/druid/query/aggregation/DoubleSumAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/DoubleSumAggregatorFactory.java @@ -22,11 +22,10 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; import com.google.common.primitives.Doubles; import io.druid.common.utils.StringUtils; -import io.druid.math.expr.Parser; import io.druid.segment.ColumnSelectorFactory; -import io.druid.segment.FloatColumnSelector; import java.nio.ByteBuffer; import java.util.Arrays; @@ -42,46 +41,30 @@ public class DoubleSumAggregatorFactory extends AggregatorFactory private final String name; private final String fieldName; - private final String expression; @JsonCreator public DoubleSumAggregatorFactory( @JsonProperty("name") String name, - @JsonProperty("fieldName") String fieldName, - @JsonProperty("expression") String expression + @JsonProperty("fieldName") String fieldName ) { Preconditions.checkNotNull(name, "Must have a valid, non-null aggregator name"); - Preconditions.checkArgument( - fieldName == null ^ expression == null, - "Must have a valid, non-null fieldName or expression" - ); + Preconditions.checkNotNull(fieldName, "Must have a valid, non-null fieldName"); this.name = name; this.fieldName = fieldName; - this.expression = expression; - } - - public DoubleSumAggregatorFactory(String name, String fieldName) - { - this(name, fieldName, null); } @Override public Aggregator factorize(ColumnSelectorFactory metricFactory) { - return new DoubleSumAggregator(getFloatColumnSelector(metricFactory)); + return new DoubleSumAggregator(metricFactory.makeFloatColumnSelector(fieldName)); } @Override public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) { - return new DoubleSumBufferAggregator(getFloatColumnSelector(metricFactory)); - } - - private FloatColumnSelector getFloatColumnSelector(ColumnSelectorFactory metricFactory) - { - return AggregatorUtil.getFloatColumnSelector(metricFactory, fieldName, expression, 0f); + return new DoubleSumBufferAggregator(metricFactory.makeFloatColumnSelector(fieldName)); } @Override @@ -99,7 +82,7 @@ public Object combine(Object lhs, Object rhs) @Override public AggregatorFactory getCombiningFactory() { - return new DoubleSumAggregatorFactory(name, name, null); + return new DoubleSumAggregatorFactory(name, name); } @Override @@ -115,7 +98,7 @@ public AggregatorFactory getMergingFactory(AggregatorFactory other) throws Aggre @Override public List getRequiredColumns() { - return Arrays.asList(new DoubleSumAggregatorFactory(fieldName, fieldName, expression)); + return Arrays.asList(new DoubleSumAggregatorFactory(fieldName, fieldName)); } @Override @@ -140,12 +123,6 @@ public String getFieldName() return fieldName; } - @JsonProperty - public String getExpression() - { - return expression; - } - @Override @JsonProperty public String getName() @@ -156,20 +133,17 @@ public String getName() @Override public List requiredFields() { - return fieldName != null ? Arrays.asList(fieldName) : Parser.findRequiredBindings(expression); + return ImmutableList.of(fieldName); } @Override public byte[] getCacheKey() { byte[] fieldNameBytes = StringUtils.toUtf8WithNullToEmpty(fieldName); - byte[] expressionBytes = StringUtils.toUtf8WithNullToEmpty(expression); - return ByteBuffer.allocate(2 + fieldNameBytes.length + expressionBytes.length) + return ByteBuffer.allocate(1 + fieldNameBytes.length) .put(CACHE_TYPE_ID) .put(fieldNameBytes) - .put(AggregatorUtil.STRING_SEPARATOR) - .put(expressionBytes) .array(); } @@ -190,7 +164,6 @@ public String toString() { return "DoubleSumAggregatorFactory{" + "fieldName='" + fieldName + '\'' + - ", expression='" + expression + '\'' + ", name='" + name + '\'' + '}'; } @@ -210,9 +183,6 @@ public boolean equals(Object o) if (!Objects.equals(fieldName, that.fieldName)) { return false; } - if (!Objects.equals(expression, that.expression)) { - return false; - } if (!Objects.equals(name, that.name)) { return false; } @@ -224,7 +194,6 @@ public boolean equals(Object o) public int hashCode() { int result = fieldName != null ? fieldName.hashCode() : 0; - result = 31 * result + (expression != null ? expression.hashCode() : 0); result = 31 * result + (name != null ? name.hashCode() : 0); return result; } diff --git a/processing/src/main/java/io/druid/query/aggregation/FilteredAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/FilteredAggregatorFactory.java index 9e830f71df8d..2186b354a7bc 100644 --- a/processing/src/main/java/io/druid/query/aggregation/FilteredAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/FilteredAggregatorFactory.java @@ -31,8 +31,6 @@ import io.druid.query.filter.ValueMatcherFactory; import io.druid.segment.ColumnSelectorFactory; import io.druid.segment.DimensionSelector; -import io.druid.segment.column.Column; -import io.druid.segment.column.ColumnCapabilities; import io.druid.segment.column.ValueType; import io.druid.segment.data.IndexedInts; import io.druid.segment.filter.BooleanValueMatcher; @@ -221,7 +219,7 @@ public FilteredAggregatorValueMatcherFactory(ColumnSelectorFactory columnSelecto @Override public ValueMatcher makeValueMatcher(final String dimension, final String value) { - if (getTypeForDimension(dimension) == ValueType.LONG) { + if (getNativeType(dimension) == ValueType.LONG) { return Filters.getLongValueMatcher( columnSelectorFactory.makeLongColumnSelector(dimension), value @@ -293,7 +291,7 @@ public boolean matches() public ValueMatcher makeValueMatcher(final String dimension, final DruidPredicateFactory predicateFactory) { - ValueType type = getTypeForDimension(dimension); + final ValueType type = getNativeType(dimension); switch (type) { case LONG: return makeLongValueMatcher(dimension, predicateFactory.makeLongPredicate()); @@ -380,16 +378,10 @@ private ValueMatcher makeLongValueMatcher(String dimension, DruidLongPredicate p ); } - private ValueType getTypeForDimension(String dimension) + private ValueType getNativeType(String columnName) { - // FilteredAggregatorFactory is sometimes created from a ColumnSelectorFactory that - // has no knowledge of column capabilities/types. - // Default to LONG for __time, STRING for everything else. - if (dimension.equals(Column.TIME_COLUMN_NAME)) { - return ValueType.LONG; - } - ColumnCapabilities capabilities = columnSelectorFactory.getColumnCapabilities(dimension); - return capabilities == null ? ValueType.STRING : capabilities.getType(); + final ValueType nativeType = columnSelectorFactory.getNativeType(columnName); + return nativeType == null ? ValueType.STRING : nativeType; } } } diff --git a/processing/src/main/java/io/druid/query/aggregation/LongMaxAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/LongMaxAggregatorFactory.java index e777de612617..6ebddb397a44 100644 --- a/processing/src/main/java/io/druid/query/aggregation/LongMaxAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/LongMaxAggregatorFactory.java @@ -22,11 +22,10 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; import com.google.common.primitives.Longs; import io.druid.common.utils.StringUtils; -import io.druid.math.expr.Parser; import io.druid.segment.ColumnSelectorFactory; -import io.druid.segment.LongColumnSelector; import java.nio.ByteBuffer; import java.util.Arrays; @@ -42,46 +41,30 @@ public class LongMaxAggregatorFactory extends AggregatorFactory private final String name; private final String fieldName; - private final String expression; @JsonCreator public LongMaxAggregatorFactory( @JsonProperty("name") String name, - @JsonProperty("fieldName") final String fieldName, - @JsonProperty("expression") String expression + @JsonProperty("fieldName") final String fieldName ) { Preconditions.checkNotNull(name, "Must have a valid, non-null aggregator name"); - Preconditions.checkArgument( - fieldName == null ^ expression == null, - "Must have a valid, non-null fieldName or expression" - ); + Preconditions.checkNotNull(fieldName, "Must have a valid, non-null fieldName"); this.name = name; this.fieldName = fieldName; - this.expression = expression; - } - - public LongMaxAggregatorFactory(String name, String fieldName) - { - this(name, fieldName, null); } @Override public Aggregator factorize(ColumnSelectorFactory metricFactory) { - return new LongMaxAggregator(getLongColumnSelector(metricFactory)); + return new LongMaxAggregator(metricFactory.makeLongColumnSelector(fieldName)); } @Override public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) { - return new LongMaxBufferAggregator(getLongColumnSelector(metricFactory)); - } - - private LongColumnSelector getLongColumnSelector(ColumnSelectorFactory metricFactory) - { - return AggregatorUtil.getLongColumnSelector(metricFactory, fieldName, expression, Long.MIN_VALUE); + return new LongMaxBufferAggregator(metricFactory.makeLongColumnSelector(fieldName)); } @Override @@ -99,7 +82,7 @@ public Object combine(Object lhs, Object rhs) @Override public AggregatorFactory getCombiningFactory() { - return new LongMaxAggregatorFactory(name, name, null); + return new LongMaxAggregatorFactory(name, name); } @Override @@ -115,7 +98,7 @@ public AggregatorFactory getMergingFactory(AggregatorFactory other) throws Aggre @Override public List getRequiredColumns() { - return Arrays.asList(new LongMaxAggregatorFactory(fieldName, fieldName, expression)); + return Arrays.asList(new LongMaxAggregatorFactory(fieldName, fieldName)); } @Override @@ -136,12 +119,6 @@ public String getFieldName() return fieldName; } - @JsonProperty - public String getExpression() - { - return expression; - } - @Override @JsonProperty public String getName() @@ -152,20 +129,17 @@ public String getName() @Override public List requiredFields() { - return fieldName != null ? Arrays.asList(fieldName) : Parser.findRequiredBindings(expression); + return ImmutableList.of(fieldName); } @Override public byte[] getCacheKey() { byte[] fieldNameBytes = StringUtils.toUtf8WithNullToEmpty(fieldName); - byte[] expressionBytes = StringUtils.toUtf8WithNullToEmpty(expression); - return ByteBuffer.allocate(2 + fieldNameBytes.length + expressionBytes.length) + return ByteBuffer.allocate(1 + fieldNameBytes.length) .put(CACHE_TYPE_ID) .put(fieldNameBytes) - .put(AggregatorUtil.STRING_SEPARATOR) - .put(expressionBytes) .array(); } @@ -186,7 +160,6 @@ public String toString() { return "LongMaxAggregatorFactory{" + "fieldName='" + fieldName + '\'' + - ", expression='" + expression + '\'' + ", name='" + name + '\'' + '}'; } @@ -206,9 +179,6 @@ public boolean equals(Object o) if (!Objects.equals(fieldName, that.fieldName)) { return false; } - if (!Objects.equals(expression, that.expression)) { - return false; - } if (!Objects.equals(name, that.name)) { return false; } @@ -220,7 +190,6 @@ public boolean equals(Object o) public int hashCode() { int result = fieldName != null ? fieldName.hashCode() : 0; - result = 31 * result + (expression != null ? expression.hashCode() : 0); result = 31 * result + (name != null ? name.hashCode() : 0); return result; } diff --git a/processing/src/main/java/io/druid/query/aggregation/LongMinAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/LongMinAggregatorFactory.java index 52317f4bd2f3..f50c6e3c45ad 100644 --- a/processing/src/main/java/io/druid/query/aggregation/LongMinAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/LongMinAggregatorFactory.java @@ -22,11 +22,10 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; import com.google.common.primitives.Longs; import io.druid.common.utils.StringUtils; -import io.druid.math.expr.Parser; import io.druid.segment.ColumnSelectorFactory; -import io.druid.segment.LongColumnSelector; import java.nio.ByteBuffer; import java.util.Arrays; @@ -42,46 +41,30 @@ public class LongMinAggregatorFactory extends AggregatorFactory private final String name; private final String fieldName; - private final String expression; @JsonCreator public LongMinAggregatorFactory( @JsonProperty("name") String name, - @JsonProperty("fieldName") final String fieldName, - @JsonProperty("expression") String expression + @JsonProperty("fieldName") final String fieldName ) { Preconditions.checkNotNull(name, "Must have a valid, non-null aggregator name"); - Preconditions.checkArgument( - fieldName == null ^ expression == null, - "Must have a valid, non-null fieldName or expression" - ); + Preconditions.checkNotNull(fieldName, "Must have a valid, non-null fieldName"); this.name = name; this.fieldName = fieldName; - this.expression = expression; - } - - public LongMinAggregatorFactory(String name, String fieldName) - { - this(name, fieldName, null); } @Override public Aggregator factorize(ColumnSelectorFactory metricFactory) { - return new LongMinAggregator(getLongColumnSelector(metricFactory)); + return new LongMinAggregator(metricFactory.makeLongColumnSelector(fieldName)); } @Override public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) { - return new LongMinBufferAggregator(getLongColumnSelector(metricFactory)); - } - - private LongColumnSelector getLongColumnSelector(ColumnSelectorFactory metricFactory) - { - return AggregatorUtil.getLongColumnSelector(metricFactory, fieldName, expression, Long.MAX_VALUE); + return new LongMinBufferAggregator(metricFactory.makeLongColumnSelector(fieldName)); } @Override @@ -99,7 +82,7 @@ public Object combine(Object lhs, Object rhs) @Override public AggregatorFactory getCombiningFactory() { - return new LongMinAggregatorFactory(name, name, null); + return new LongMinAggregatorFactory(name, name); } @Override @@ -115,7 +98,7 @@ public AggregatorFactory getMergingFactory(AggregatorFactory other) throws Aggre @Override public List getRequiredColumns() { - return Arrays.asList(new LongMinAggregatorFactory(fieldName, fieldName, expression)); + return Arrays.asList(new LongMinAggregatorFactory(fieldName, fieldName)); } @Override @@ -136,12 +119,6 @@ public String getFieldName() return fieldName; } - @JsonProperty - public String getExpression() - { - return expression; - } - @Override @JsonProperty public String getName() @@ -152,20 +129,17 @@ public String getName() @Override public List requiredFields() { - return fieldName != null ? Arrays.asList(fieldName) : Parser.findRequiredBindings(expression); + return ImmutableList.of(fieldName); } @Override public byte[] getCacheKey() { byte[] fieldNameBytes = StringUtils.toUtf8WithNullToEmpty(fieldName); - byte[] expressionBytes = StringUtils.toUtf8WithNullToEmpty(expression); - return ByteBuffer.allocate(2 + fieldNameBytes.length + expressionBytes.length) + return ByteBuffer.allocate(1 + fieldNameBytes.length) .put(CACHE_TYPE_ID) .put(fieldNameBytes) - .put(AggregatorUtil.STRING_SEPARATOR) - .put(expressionBytes) .array(); } @@ -186,7 +160,6 @@ public String toString() { return "LongMinAggregatorFactory{" + "fieldName='" + fieldName + '\'' + - ", expression='" + expression + '\'' + ", name='" + name + '\'' + '}'; } @@ -206,9 +179,6 @@ public boolean equals(Object o) if (!Objects.equals(fieldName, that.fieldName)) { return false; } - if (!Objects.equals(expression, that.expression)) { - return false; - } if (!Objects.equals(name, that.name)) { return false; } @@ -220,7 +190,6 @@ public boolean equals(Object o) public int hashCode() { int result = fieldName != null ? fieldName.hashCode() : 0; - result = 31 * result + (expression != null ? expression.hashCode() : 0); result = 31 * result + (name != null ? name.hashCode() : 0); return result; } diff --git a/processing/src/main/java/io/druid/query/aggregation/LongSumAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/LongSumAggregatorFactory.java index 6c5679af332b..05acd7c1d3b9 100644 --- a/processing/src/main/java/io/druid/query/aggregation/LongSumAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/LongSumAggregatorFactory.java @@ -22,11 +22,10 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; import com.google.common.primitives.Longs; import io.druid.common.utils.StringUtils; -import io.druid.math.expr.Parser; import io.druid.segment.ColumnSelectorFactory; -import io.druid.segment.LongColumnSelector; import java.nio.ByteBuffer; import java.util.Arrays; @@ -42,46 +41,30 @@ public class LongSumAggregatorFactory extends AggregatorFactory private final String name; private final String fieldName; - private final String expression; @JsonCreator public LongSumAggregatorFactory( @JsonProperty("name") String name, - @JsonProperty("fieldName") String fieldName, - @JsonProperty("expression") String expression + @JsonProperty("fieldName") String fieldName ) { Preconditions.checkNotNull(name, "Must have a valid, non-null aggregator name"); - Preconditions.checkArgument( - fieldName == null ^ expression == null, - "Must have a valid, non-null fieldName or expression" - ); + Preconditions.checkNotNull(fieldName, "Must have a valid, non-null fieldName"); this.name = name; this.fieldName = fieldName; - this.expression = expression; - } - - public LongSumAggregatorFactory(String name, String fieldName) - { - this(name, fieldName, null); } @Override public Aggregator factorize(ColumnSelectorFactory metricFactory) { - return new LongSumAggregator(getLongColumnSelector(metricFactory)); + return new LongSumAggregator(metricFactory.makeLongColumnSelector(fieldName)); } @Override public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) { - return new LongSumBufferAggregator(getLongColumnSelector(metricFactory)); - } - - private LongColumnSelector getLongColumnSelector(ColumnSelectorFactory metricFactory) - { - return AggregatorUtil.getLongColumnSelector(metricFactory, fieldName, expression, 0L); + return new LongSumBufferAggregator(metricFactory.makeLongColumnSelector(fieldName)); } @Override @@ -99,7 +82,7 @@ public Object combine(Object lhs, Object rhs) @Override public AggregatorFactory getCombiningFactory() { - return new LongSumAggregatorFactory(name, name, null); + return new LongSumAggregatorFactory(name, name); } @Override @@ -115,7 +98,7 @@ public AggregatorFactory getMergingFactory(AggregatorFactory other) throws Aggre @Override public List getRequiredColumns() { - return Arrays.asList(new LongSumAggregatorFactory(fieldName, fieldName, expression)); + return Arrays.asList(new LongSumAggregatorFactory(fieldName, fieldName)); } @Override @@ -136,12 +119,6 @@ public String getFieldName() return fieldName; } - @JsonProperty - public String getExpression() - { - return expression; - } - @Override @JsonProperty public String getName() @@ -152,20 +129,17 @@ public String getName() @Override public List requiredFields() { - return fieldName != null ? Arrays.asList(fieldName) : Parser.findRequiredBindings(expression); + return ImmutableList.of(fieldName); } @Override public byte[] getCacheKey() { byte[] fieldNameBytes = StringUtils.toUtf8WithNullToEmpty(fieldName); - byte[] expressionBytes = StringUtils.toUtf8WithNullToEmpty(expression); - return ByteBuffer.allocate(2 + fieldNameBytes.length + expressionBytes.length) + return ByteBuffer.allocate(1 + fieldNameBytes.length) .put(CACHE_TYPE_ID) .put(fieldNameBytes) - .put(AggregatorUtil.STRING_SEPARATOR) - .put(expressionBytes) .array(); } @@ -186,7 +160,6 @@ public String toString() { return "LongSumAggregatorFactory{" + "fieldName='" + fieldName + '\'' + - ", expression='" + expression + '\'' + ", name='" + name + '\'' + '}'; } @@ -206,9 +179,6 @@ public boolean equals(Object o) if (!Objects.equals(fieldName, that.fieldName)) { return false; } - if (!Objects.equals(expression, that.expression)) { - return false; - } if (!Objects.equals(name, that.name)) { return false; } @@ -220,7 +190,6 @@ public boolean equals(Object o) public int hashCode() { int result = fieldName != null ? fieldName.hashCode() : 0; - result = 31 * result + (expression != null ? expression.hashCode() : 0); result = 31 * result + (name != null ? name.hashCode() : 0); return result; } diff --git a/processing/src/main/java/io/druid/query/groupby/GroupByQuery.java b/processing/src/main/java/io/druid/query/groupby/GroupByQuery.java index 7df97a4c56e5..5c1af08da634 100644 --- a/processing/src/main/java/io/druid/query/groupby/GroupByQuery.java +++ b/processing/src/main/java/io/druid/query/groupby/GroupByQuery.java @@ -56,8 +56,11 @@ import io.druid.query.groupby.orderby.OrderByColumnSpec; import io.druid.query.spec.LegacySegmentSpec; import io.druid.query.spec.QuerySegmentSpec; +import io.druid.segment.virtual.VirtualColumn; +import io.druid.segment.virtual.VirtualColumns; import org.joda.time.Interval; +import java.util.Arrays; import java.util.Comparator; import java.util.List; import java.util.Map; @@ -88,6 +91,7 @@ public static Builder builder() return new Builder(); } + private final VirtualColumns virtualColumns; private final LimitSpec limitSpec; private final HavingSpec havingSpec; private final DimFilter dimFilter; @@ -102,6 +106,7 @@ public static Builder builder() public GroupByQuery( @JsonProperty("dataSource") DataSource dataSource, @JsonProperty("intervals") QuerySegmentSpec querySegmentSpec, + @JsonProperty("virtualColumns") VirtualColumns virtualColumns, @JsonProperty("filter") DimFilter dimFilter, @JsonProperty("granularity") QueryGranularity granularity, @JsonProperty("dimensions") List dimensions, @@ -113,6 +118,7 @@ public GroupByQuery( ) { super(dataSource, querySegmentSpec, false, context); + this.virtualColumns = virtualColumns == null ? VirtualColumns.EMPTY : virtualColumns; this.dimFilter = dimFilter; this.granularity = granularity; this.dimensions = dimensions == null ? ImmutableList.of() : dimensions; @@ -169,6 +175,7 @@ public boolean apply(Row input) private GroupByQuery( DataSource dataSource, QuerySegmentSpec querySegmentSpec, + VirtualColumns virtualColumns, DimFilter dimFilter, QueryGranularity granularity, List dimensions, @@ -182,6 +189,7 @@ private GroupByQuery( { super(dataSource, querySegmentSpec, false, context); + this.virtualColumns = virtualColumns; this.dimFilter = dimFilter; this.granularity = granularity; this.dimensions = dimensions; @@ -192,6 +200,12 @@ private GroupByQuery( this.limitFn = limitFn; } + @JsonProperty + public VirtualColumns getVirtualColumns() + { + return virtualColumns; + } + @JsonProperty("filter") public DimFilter getDimFilter() { @@ -380,6 +394,7 @@ public GroupByQuery withOverriddenContext(Map contextOverride) return new GroupByQuery( getDataSource(), getQuerySegmentSpec(), + virtualColumns, dimFilter, granularity, dimensions, @@ -398,6 +413,7 @@ public GroupByQuery withQuerySegmentSpec(QuerySegmentSpec spec) return new GroupByQuery( getDataSource(), spec, + virtualColumns, dimFilter, granularity, dimensions, @@ -415,6 +431,7 @@ public GroupByQuery withDimFilter(final DimFilter dimFilter) return new GroupByQuery( getDataSource(), getQuerySegmentSpec(), + virtualColumns, dimFilter, getGranularity(), getDimensions(), @@ -433,6 +450,7 @@ public Query withDataSource(DataSource dataSource) return new GroupByQuery( dataSource, getQuerySegmentSpec(), + virtualColumns, dimFilter, granularity, dimensions, @@ -450,6 +468,7 @@ public GroupByQuery withDimensionSpecs(final List dimensionSpecs) return new GroupByQuery( getDataSource(), getQuerySegmentSpec(), + virtualColumns, getDimFilter(), getGranularity(), dimensionSpecs, @@ -467,6 +486,7 @@ public GroupByQuery withLimitSpec(final LimitSpec limitSpec) return new GroupByQuery( getDataSource(), getQuerySegmentSpec(), + virtualColumns, getDimFilter(), getGranularity(), getDimensions(), @@ -483,6 +503,7 @@ public GroupByQuery withAggregatorSpecs(final List aggregator return new GroupByQuery( getDataSource(), getQuerySegmentSpec(), + virtualColumns, getDimFilter(), getGranularity(), getDimensions(), @@ -500,6 +521,7 @@ public GroupByQuery withPostAggregatorSpecs(final List postAggre return new GroupByQuery( getDataSource(), getQuerySegmentSpec(), + virtualColumns, getDimFilter(), getGranularity(), getDimensions(), @@ -542,6 +564,7 @@ public static class Builder { private DataSource dataSource; private QuerySegmentSpec querySegmentSpec; + private VirtualColumns virtualColumns; private DimFilter dimFilter; private QueryGranularity granularity; private List dimensions; @@ -563,6 +586,7 @@ public Builder(GroupByQuery query) { dataSource = query.getDataSource(); querySegmentSpec = query.getQuerySegmentSpec(); + virtualColumns = query.getVirtualColumns(); limitSpec = query.getLimitSpec(); dimFilter = query.getDimFilter(); granularity = query.getGranularity(); @@ -577,6 +601,7 @@ public Builder(Builder builder) { dataSource = builder.dataSource; querySegmentSpec = builder.querySegmentSpec; + virtualColumns = builder.virtualColumns; limitSpec = builder.limitSpec; dimFilter = builder.dimFilter; granularity = builder.granularity; @@ -627,6 +652,15 @@ public Builder setInterval(String interval) return setQuerySegmentSpec(new LegacySegmentSpec(interval)); } + public Builder setVirtualColumns(List virtualColumns) { + this.virtualColumns = VirtualColumns.create(virtualColumns); + return this; + } + + public Builder setVirtualColumns(VirtualColumn... virtualColumns) { + return setVirtualColumns(Arrays.asList(virtualColumns)); + } + public Builder limit(int limit) { ensureExplicitLimitNotSet(); @@ -789,6 +823,7 @@ public GroupByQuery build() return new GroupByQuery( dataSource, querySegmentSpec, + virtualColumns, dimFilter, granularity, dimensions, @@ -801,22 +836,6 @@ public GroupByQuery build() } } - @Override - public String toString() - { - return "GroupByQuery{" + - "dataSource='" + getDataSource() + '\'' + - ", querySegmentSpec=" + getQuerySegmentSpec() + - ", limitSpec=" + limitSpec + - ", dimFilter=" + dimFilter + - ", granularity=" + granularity + - ", dimensions=" + dimensions + - ", aggregatorSpecs=" + aggregatorSpecs + - ", postAggregatorSpecs=" + postAggregatorSpecs + - ", havingSpec=" + havingSpec + - '}'; - } - @Override public boolean equals(Object o) { @@ -832,44 +851,57 @@ public boolean equals(Object o) GroupByQuery that = (GroupByQuery) o; - if (aggregatorSpecs != null ? !aggregatorSpecs.equals(that.aggregatorSpecs) : that.aggregatorSpecs != null) { + if (!virtualColumns.equals(that.virtualColumns)) { return false; } - if (dimFilter != null ? !dimFilter.equals(that.dimFilter) : that.dimFilter != null) { + if (!limitSpec.equals(that.limitSpec)) { return false; } - if (dimensions != null ? !dimensions.equals(that.dimensions) : that.dimensions != null) { + if (havingSpec != null ? !havingSpec.equals(that.havingSpec) : that.havingSpec != null) { return false; } - if (granularity != null ? !granularity.equals(that.granularity) : that.granularity != null) { + if (dimFilter != null ? !dimFilter.equals(that.dimFilter) : that.dimFilter != null) { return false; } - if (havingSpec != null ? !havingSpec.equals(that.havingSpec) : that.havingSpec != null) { + if (granularity != null ? !granularity.equals(that.granularity) : that.granularity != null) { return false; } - if (limitSpec != null ? !limitSpec.equals(that.limitSpec) : that.limitSpec != null) { + if (!dimensions.equals(that.dimensions)) { return false; } - if (postAggregatorSpecs != null - ? !postAggregatorSpecs.equals(that.postAggregatorSpecs) - : that.postAggregatorSpecs != null) { + if (!aggregatorSpecs.equals(that.aggregatorSpecs)) { return false; } - - return true; + return postAggregatorSpecs.equals(that.postAggregatorSpecs); } @Override public int hashCode() { int result = super.hashCode(); - result = 31 * result + (limitSpec != null ? limitSpec.hashCode() : 0); + result = 31 * result + virtualColumns.hashCode(); + result = 31 * result + limitSpec.hashCode(); result = 31 * result + (havingSpec != null ? havingSpec.hashCode() : 0); result = 31 * result + (dimFilter != null ? dimFilter.hashCode() : 0); result = 31 * result + (granularity != null ? granularity.hashCode() : 0); - result = 31 * result + (dimensions != null ? dimensions.hashCode() : 0); - result = 31 * result + (aggregatorSpecs != null ? aggregatorSpecs.hashCode() : 0); - result = 31 * result + (postAggregatorSpecs != null ? postAggregatorSpecs.hashCode() : 0); + result = 31 * result + dimensions.hashCode(); + result = 31 * result + aggregatorSpecs.hashCode(); + result = 31 * result + postAggregatorSpecs.hashCode(); return result; } + + @Override + public String toString() + { + return "GroupByQuery{" + + "virtualColumns=" + virtualColumns + + ", limitSpec=" + limitSpec + + ", havingSpec=" + havingSpec + + ", dimFilter=" + dimFilter + + ", granularity=" + granularity + + ", dimensions=" + dimensions + + ", aggregations=" + aggregatorSpecs + + ", postAggregations=" + postAggregatorSpecs + + '}'; + } } diff --git a/processing/src/main/java/io/druid/query/groupby/GroupByQueryEngine.java b/processing/src/main/java/io/druid/query/groupby/GroupByQueryEngine.java index d78f64bb61f1..1b3fd3e5e04e 100644 --- a/processing/src/main/java/io/druid/query/groupby/GroupByQueryEngine.java +++ b/processing/src/main/java/io/druid/query/groupby/GroupByQueryEngine.java @@ -48,7 +48,6 @@ import io.druid.segment.Cursor; import io.druid.segment.DimensionSelector; import io.druid.segment.StorageAdapter; -import io.druid.segment.VirtualColumns; import io.druid.segment.data.IndexedInts; import io.druid.segment.filter.Filters; import org.joda.time.DateTime; @@ -101,7 +100,7 @@ public Sequence process(final GroupByQuery query, final StorageAdapter stor final Sequence cursors = storageAdapter.makeCursors( filter, intervals.get(0), - VirtualColumns.EMPTY, + query.getVirtualColumns(), query.getGranularity(), false ); diff --git a/processing/src/main/java/io/druid/query/groupby/GroupByQueryQueryToolChest.java b/processing/src/main/java/io/druid/query/groupby/GroupByQueryQueryToolChest.java index 436e9d31ce5a..21aaf8e1eadf 100644 --- a/processing/src/main/java/io/druid/query/groupby/GroupByQueryQueryToolChest.java +++ b/processing/src/main/java/io/druid/query/groupby/GroupByQueryQueryToolChest.java @@ -355,6 +355,7 @@ public byte[] computeCacheKey(GroupByQuery query) } final byte[] havingBytes = query.getHavingSpec() == null ? new byte[]{} : query.getHavingSpec().getCacheKey(); final byte[] limitBytes = query.getLimitSpec().getCacheKey(); + final byte[] virtualColumnsBytes = query.getVirtualColumns().getCacheKey(); ByteBuffer buffer = ByteBuffer .allocate( @@ -365,6 +366,7 @@ public byte[] computeCacheKey(GroupByQuery query) + dimensionsBytesSize + havingBytes.length + limitBytes.length + + virtualColumnsBytes.length ) .put(GROUPBY_QUERY) .put(CACHE_STRATEGY_VERSION) @@ -379,6 +381,7 @@ public byte[] computeCacheKey(GroupByQuery query) return buffer .put(havingBytes) .put(limitBytes) + .put(virtualColumnsBytes) .array(); } diff --git a/processing/src/main/java/io/druid/query/groupby/epinephelinae/GroupByQueryEngineV2.java b/processing/src/main/java/io/druid/query/groupby/epinephelinae/GroupByQueryEngineV2.java index 7fdae22a87d4..08e0879544de 100644 --- a/processing/src/main/java/io/druid/query/groupby/epinephelinae/GroupByQueryEngineV2.java +++ b/processing/src/main/java/io/druid/query/groupby/epinephelinae/GroupByQueryEngineV2.java @@ -41,7 +41,6 @@ import io.druid.segment.Cursor; import io.druid.segment.DimensionSelector; import io.druid.segment.StorageAdapter; -import io.druid.segment.VirtualColumns; import io.druid.segment.data.EmptyIndexedInts; import io.druid.segment.data.IndexedInts; import io.druid.segment.filter.Filters; @@ -84,7 +83,7 @@ public static Sequence process( final Sequence cursors = storageAdapter.makeCursors( Filters.toFilter(query.getDimFilter()), intervals.get(0), - VirtualColumns.EMPTY, + query.getVirtualColumns(), query.getGranularity(), false ); diff --git a/processing/src/main/java/io/druid/query/groupby/epinephelinae/RowBasedGrouperHelper.java b/processing/src/main/java/io/druid/query/groupby/epinephelinae/RowBasedGrouperHelper.java index 33f014662fdb..c6c9cd2f74f3 100644 --- a/processing/src/main/java/io/druid/query/groupby/epinephelinae/RowBasedGrouperHelper.java +++ b/processing/src/main/java/io/druid/query/groupby/epinephelinae/RowBasedGrouperHelper.java @@ -25,7 +25,6 @@ import com.google.common.base.Function; import com.google.common.base.Preconditions; import com.google.common.base.Strings; -import com.google.common.base.Supplier; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.primitives.Chars; @@ -36,9 +35,6 @@ import io.druid.granularity.AllGranularity; import io.druid.java.util.common.Pair; import io.druid.java.util.common.guava.Accumulator; -import io.druid.math.expr.Evals; -import io.druid.math.expr.Expr; -import io.druid.math.expr.Parser; import io.druid.query.QueryInterruptedException; import io.druid.query.aggregation.AggregatorFactory; import io.druid.query.dimension.DimensionSpec; @@ -50,10 +46,9 @@ import io.druid.segment.DimensionSelector; import io.druid.segment.FloatColumnSelector; import io.druid.segment.LongColumnSelector; -import io.druid.segment.NumericColumnSelector; import io.druid.segment.ObjectColumnSelector; import io.druid.segment.column.Column; -import io.druid.segment.column.ColumnCapabilities; +import io.druid.segment.column.ValueType; import io.druid.segment.data.IndexedInts; import it.unimi.dsi.fastutil.ints.IntIterator; import it.unimi.dsi.fastutil.ints.IntIterators; @@ -94,12 +89,14 @@ public static Pair, Accumulator, Row>> querySpecificConfig.getMaxMergingDictionarySize() / (concurrencyHint == -1 ? 1 : concurrencyHint) ); final RowBasedColumnSelectorFactory columnSelectorFactory = new RowBasedColumnSelectorFactory(); + final ColumnSelectorFactory virtualizedColumnSelectorFactory = query.getVirtualColumns() + .wrap(columnSelectorFactory); final Grouper grouper; if (concurrencyHint == -1) { grouper = new SpillingGrouper<>( buffer, keySerdeFactory, - columnSelectorFactory, + virtualizedColumnSelectorFactory, aggregatorFactories, querySpecificConfig.getBufferGrouperMaxSize(), querySpecificConfig.getBufferGrouperMaxLoadFactor(), @@ -112,7 +109,7 @@ public static Pair, Accumulator, Row>> grouper = new ConcurrentGrouper<>( buffer, keySerdeFactory, - columnSelectorFactory, + virtualizedColumnSelectorFactory, aggregatorFactories, querySpecificConfig.getBufferGrouperMaxSize(), querySpecificConfig.getBufferGrouperMaxLoadFactor(), @@ -127,7 +124,7 @@ public static Pair, Accumulator, Row>> if (isInputRaw) { dimensionSelectors = new DimensionSelector[query.getDimensions().size()]; for (int i = 0; i < dimensionSelectors.length; i++) { - dimensionSelectors[i] = columnSelectorFactory.makeDimensionSelector(query.getDimensions().get(i)); + dimensionSelectors[i] = virtualizedColumnSelectorFactory.makeDimensionSelector(query.getDimensions().get(i)); } } else { dimensionSelectors = null; @@ -777,42 +774,17 @@ public Object get() } @Override - public NumericColumnSelector makeMathExpressionSelector(String expression) + public ValueType getNativeType(String columnName) { - final Expr parsed = Parser.parse(expression); + // This ColumnSelectorFactory implementation has no innate knowledge of column types. All it knows + // is that TIME_COLUMN_NAME is a long. - final List required = Parser.findRequiredBindings(parsed); - final Map> values = Maps.newHashMapWithExpectedSize(required.size()); - - for (final String columnName : required) { - values.put( - columnName, new Supplier() - { - @Override - public Number get() - { - return Evals.toNumber(row.get().getRaw(columnName)); - } - } - ); + if (Column.TIME_COLUMN_NAME.equals(columnName)) { + return ValueType.LONG; + } else { + // Return null, caller will assume default types in this case. + return null; } - final Expr.ObjectBinding binding = Parser.withSuppliers(values); - - return new NumericColumnSelector() - { - @Override - public Number get() - { - return parsed.eval(binding).numericValue(); - } - }; - } - - @Override - public ColumnCapabilities getColumnCapabilities(String columnName) - { - // We don't have any information on the column value type, returning null defaults type to string - return null; } } diff --git a/processing/src/main/java/io/druid/query/groupby/strategy/GroupByStrategyV1.java b/processing/src/main/java/io/druid/query/groupby/strategy/GroupByStrategyV1.java index 9660a8c2bb7a..a292ff9a8743 100644 --- a/processing/src/main/java/io/druid/query/groupby/strategy/GroupByStrategyV1.java +++ b/processing/src/main/java/io/druid/query/groupby/strategy/GroupByStrategyV1.java @@ -93,6 +93,7 @@ public Sequence mergeResults( new GroupByQuery( query.getDataSource(), query.getQuerySegmentSpec(), + query.getVirtualColumns(), query.getDimFilter(), query.getGranularity(), query.getDimensions(), diff --git a/processing/src/main/java/io/druid/query/groupby/strategy/GroupByStrategyV2.java b/processing/src/main/java/io/druid/query/groupby/strategy/GroupByStrategyV2.java index 7e1ab35f6b9e..27401ebe0c05 100644 --- a/processing/src/main/java/io/druid/query/groupby/strategy/GroupByStrategyV2.java +++ b/processing/src/main/java/io/druid/query/groupby/strategy/GroupByStrategyV2.java @@ -145,6 +145,7 @@ protected BinaryFn createMergeFn(Query queryParam) new GroupByQuery( query.getDataSource(), query.getQuerySegmentSpec(), + query.getVirtualColumns(), query.getDimFilter(), query.getGranularity(), query.getDimensions(), diff --git a/processing/src/main/java/io/druid/query/metadata/SegmentAnalyzer.java b/processing/src/main/java/io/druid/query/metadata/SegmentAnalyzer.java index 603f8c8e99f9..de7f0bf1ad3b 100644 --- a/processing/src/main/java/io/druid/query/metadata/SegmentAnalyzer.java +++ b/processing/src/main/java/io/druid/query/metadata/SegmentAnalyzer.java @@ -39,7 +39,6 @@ import io.druid.segment.QueryableIndex; import io.druid.segment.Segment; import io.druid.segment.StorageAdapter; -import io.druid.segment.VirtualColumns; import io.druid.segment.column.BitmapIndex; import io.druid.segment.column.Column; import io.druid.segment.column.ColumnCapabilities; @@ -49,6 +48,7 @@ import io.druid.segment.data.IndexedInts; import io.druid.segment.serde.ComplexMetricSerde; import io.druid.segment.serde.ComplexMetrics; +import io.druid.segment.virtual.VirtualColumns; import org.joda.time.Interval; import javax.annotation.Nullable; diff --git a/processing/src/main/java/io/druid/query/search/SearchQueryRunner.java b/processing/src/main/java/io/druid/query/search/SearchQueryRunner.java index 9e6b7e85920d..f00c0b922115 100644 --- a/processing/src/main/java/io/druid/query/search/SearchQueryRunner.java +++ b/processing/src/main/java/io/druid/query/search/SearchQueryRunner.java @@ -25,10 +25,10 @@ import com.google.common.collect.Iterables; import com.google.common.collect.Lists; import com.google.common.collect.Maps; +import com.metamx.emitter.EmittingLogger; import io.druid.collections.bitmap.BitmapFactory; import io.druid.collections.bitmap.ImmutableBitmap; import io.druid.collections.bitmap.MutableBitmap; -import com.metamx.emitter.EmittingLogger; import io.druid.java.util.common.IAE; import io.druid.java.util.common.ISE; import io.druid.java.util.common.guava.Accumulator; @@ -52,12 +52,12 @@ import io.druid.segment.QueryableIndex; import io.druid.segment.Segment; import io.druid.segment.StorageAdapter; -import io.druid.segment.VirtualColumns; import io.druid.segment.column.BitmapIndex; import io.druid.segment.column.Column; import io.druid.segment.column.GenericColumn; import io.druid.segment.data.IndexedInts; import io.druid.segment.filter.Filters; +import io.druid.segment.virtual.VirtualColumns; import org.apache.commons.lang.mutable.MutableInt; import org.joda.time.Interval; diff --git a/processing/src/main/java/io/druid/query/select/SelectQuery.java b/processing/src/main/java/io/druid/query/select/SelectQuery.java index 4dd27a593083..bbc2cf04cbd6 100644 --- a/processing/src/main/java/io/druid/query/select/SelectQuery.java +++ b/processing/src/main/java/io/druid/query/select/SelectQuery.java @@ -31,7 +31,7 @@ import io.druid.query.dimension.DimensionSpec; import io.druid.query.filter.DimFilter; import io.druid.query.spec.QuerySegmentSpec; -import io.druid.segment.VirtualColumn; +import io.druid.segment.virtual.VirtualColumns; import java.util.List; import java.util.Map; @@ -46,7 +46,7 @@ public class SelectQuery extends BaseQuery> private final QueryGranularity granularity; private final List dimensions; private final List metrics; - private final List virtualColumns; + private final VirtualColumns virtualColumns; private final PagingSpec pagingSpec; @JsonCreator @@ -58,7 +58,7 @@ public SelectQuery( @JsonProperty("granularity") QueryGranularity granularity, @JsonProperty("dimensions") List dimensions, @JsonProperty("metrics") List metrics, - @JsonProperty("virtualColumns") List virtualColumns, + @JsonProperty("virtualColumns") VirtualColumns virtualColumns, @JsonProperty("pagingSpec") PagingSpec pagingSpec, @JsonProperty("context") Map context ) @@ -67,7 +67,7 @@ public SelectQuery( this.dimFilter = dimFilter; this.granularity = granularity; this.dimensions = dimensions; - this.virtualColumns = virtualColumns; + this.virtualColumns = virtualColumns == null ? VirtualColumns.EMPTY : virtualColumns; this.metrics = metrics; this.pagingSpec = pagingSpec; @@ -134,7 +134,7 @@ public List getMetrics() } @JsonProperty - public List getVirtualColumns() + public VirtualColumns getVirtualColumns() { return virtualColumns; } diff --git a/processing/src/main/java/io/druid/query/select/SelectQueryEngine.java b/processing/src/main/java/io/druid/query/select/SelectQueryEngine.java index eb6786f9edbc..d47a71663db1 100644 --- a/processing/src/main/java/io/druid/query/select/SelectQueryEngine.java +++ b/processing/src/main/java/io/druid/query/select/SelectQueryEngine.java @@ -37,7 +37,6 @@ import io.druid.segment.ObjectColumnSelector; import io.druid.segment.Segment; import io.druid.segment.StorageAdapter; -import io.druid.segment.VirtualColumns; import io.druid.segment.column.Column; import io.druid.segment.data.IndexedInts; import io.druid.segment.filter.Filters; @@ -90,7 +89,7 @@ public Sequence> process(final SelectQuery query, fina adapter, query.getQuerySegmentSpec().getIntervals(), filter, - VirtualColumns.valueOf(query.getVirtualColumns()), + query.getVirtualColumns(), query.isDescending(), query.getGranularity(), new Function>() diff --git a/processing/src/main/java/io/druid/query/select/SelectQueryQueryToolChest.java b/processing/src/main/java/io/druid/query/select/SelectQueryQueryToolChest.java index 99ff72251fd4..2bb009835e2d 100644 --- a/processing/src/main/java/io/druid/query/select/SelectQueryQueryToolChest.java +++ b/processing/src/main/java/io/druid/query/select/SelectQueryQueryToolChest.java @@ -49,7 +49,6 @@ import io.druid.query.dimension.DimensionSpec; import io.druid.query.filter.DimFilter; import io.druid.timeline.DataSegmentUtils; -import io.druid.segment.VirtualColumn; import io.druid.timeline.LogicalSegment; import org.joda.time.DateTime; import org.joda.time.Interval; @@ -192,20 +191,7 @@ public byte[] computeCacheKey(SelectQuery query) ++index; } - List virtualColumns = query.getVirtualColumns(); - if (virtualColumns == null) { - virtualColumns = Collections.emptyList(); - } - - final byte[][] virtualColumnsBytes = new byte[virtualColumns.size()][]; - int virtualColumnsBytesSize = 0; - index = 0; - for (VirtualColumn vc : virtualColumns) { - virtualColumnsBytes[index] = vc.getCacheKey(); - virtualColumnsBytesSize += virtualColumnsBytes[index].length; - ++index; - } - + final byte[] virtualColumnsCacheKey = query.getVirtualColumns().getCacheKey(); final ByteBuffer queryCacheKey = ByteBuffer .allocate( 1 @@ -214,7 +200,7 @@ public byte[] computeCacheKey(SelectQuery query) + query.getPagingSpec().getCacheKey().length + dimensionsBytesSize + metricBytesSize - + virtualColumnsBytesSize + + virtualColumnsCacheKey.length ) .put(SELECT_QUERY) .put(granularityBytes) @@ -229,9 +215,7 @@ public byte[] computeCacheKey(SelectQuery query) queryCacheKey.put(metricByte); } - for (byte[] vcByte : virtualColumnsBytes) { - queryCacheKey.put(vcByte); - } + queryCacheKey.put(virtualColumnsCacheKey); return queryCacheKey.array(); } diff --git a/processing/src/main/java/io/druid/query/timeboundary/TimeBoundaryQueryRunnerFactory.java b/processing/src/main/java/io/druid/query/timeboundary/TimeBoundaryQueryRunnerFactory.java index 921e1c3a909b..c77b5f8d3e52 100644 --- a/processing/src/main/java/io/druid/query/timeboundary/TimeBoundaryQueryRunnerFactory.java +++ b/processing/src/main/java/io/druid/query/timeboundary/TimeBoundaryQueryRunnerFactory.java @@ -39,9 +39,9 @@ import io.druid.segment.LongColumnSelector; import io.druid.segment.Segment; import io.druid.segment.StorageAdapter; -import io.druid.segment.VirtualColumns; import io.druid.segment.column.Column; import io.druid.segment.filter.Filters; +import io.druid.segment.virtual.VirtualColumns; import org.joda.time.DateTime; import java.util.Iterator; diff --git a/processing/src/main/java/io/druid/query/timeseries/TimeseriesQuery.java b/processing/src/main/java/io/druid/query/timeseries/TimeseriesQuery.java index 964bf38fb9d1..0a90f17e2426 100644 --- a/processing/src/main/java/io/druid/query/timeseries/TimeseriesQuery.java +++ b/processing/src/main/java/io/druid/query/timeseries/TimeseriesQuery.java @@ -33,6 +33,7 @@ import io.druid.query.aggregation.PostAggregator; import io.druid.query.filter.DimFilter; import io.druid.query.spec.QuerySegmentSpec; +import io.druid.segment.virtual.VirtualColumns; import java.util.List; import java.util.Map; @@ -42,6 +43,7 @@ @JsonTypeName("timeseries") public class TimeseriesQuery extends BaseQuery> { + private final VirtualColumns virtualColumns; private final DimFilter dimFilter; private final QueryGranularity granularity; private final List aggregatorSpecs; @@ -52,6 +54,7 @@ public TimeseriesQuery( @JsonProperty("dataSource") DataSource dataSource, @JsonProperty("intervals") QuerySegmentSpec querySegmentSpec, @JsonProperty("descending") boolean descending, + @JsonProperty("virtualColumns") VirtualColumns virtualColumns, @JsonProperty("filter") DimFilter dimFilter, @JsonProperty("granularity") QueryGranularity granularity, @JsonProperty("aggregations") List aggregatorSpecs, @@ -60,6 +63,7 @@ public TimeseriesQuery( ) { super(dataSource, querySegmentSpec, descending, context); + this.virtualColumns = virtualColumns == null ? VirtualColumns.EMPTY : virtualColumns; this.dimFilter = dimFilter; this.granularity = granularity; this.aggregatorSpecs = aggregatorSpecs == null ? ImmutableList.of() : aggregatorSpecs; @@ -86,6 +90,12 @@ public String getType() return Query.TIMESERIES; } + @JsonProperty + public VirtualColumns getVirtualColumns() + { + return virtualColumns; + } + @JsonProperty("filter") public DimFilter getDimensionsFilter() { @@ -121,6 +131,7 @@ public TimeseriesQuery withQuerySegmentSpec(QuerySegmentSpec querySegmentSpec) getDataSource(), querySegmentSpec, isDescending(), + virtualColumns, dimFilter, granularity, aggregatorSpecs, @@ -136,6 +147,7 @@ public Query> withDataSource(DataSource dataSource dataSource, getQuerySegmentSpec(), isDescending(), + virtualColumns, dimFilter, granularity, aggregatorSpecs, @@ -150,6 +162,7 @@ public TimeseriesQuery withOverriddenContext(Map contextOverride getDataSource(), getQuerySegmentSpec(), isDescending(), + virtualColumns, dimFilter, granularity, aggregatorSpecs, @@ -164,6 +177,7 @@ public TimeseriesQuery withDimFilter(DimFilter dimFilter) getDataSource(), getQuerySegmentSpec(), isDescending(), + virtualColumns, dimFilter, granularity, aggregatorSpecs, @@ -172,21 +186,6 @@ public TimeseriesQuery withDimFilter(DimFilter dimFilter) ); } - @Override - public String toString() - { - return "TimeseriesQuery{" + - "dataSource='" + getDataSource() + '\'' + - ", querySegmentSpec=" + getQuerySegmentSpec() + - ", descending=" + isDescending() + - ", dimFilter=" + dimFilter + - ", granularity='" + granularity + '\'' + - ", aggregatorSpecs=" + aggregatorSpecs + - ", postAggregatorSpecs=" + postAggregatorSpecs + - ", context=" + getContext() + - '}'; - } - @Override public boolean equals(Object o) { @@ -202,7 +201,7 @@ public boolean equals(Object o) TimeseriesQuery that = (TimeseriesQuery) o; - if (aggregatorSpecs != null ? !aggregatorSpecs.equals(that.aggregatorSpecs) : that.aggregatorSpecs != null) { + if (!virtualColumns.equals(that.virtualColumns)) { return false; } if (dimFilter != null ? !dimFilter.equals(that.dimFilter) : that.dimFilter != null) { @@ -211,21 +210,35 @@ public boolean equals(Object o) if (granularity != null ? !granularity.equals(that.granularity) : that.granularity != null) { return false; } - if (postAggregatorSpecs != null ? !postAggregatorSpecs.equals(that.postAggregatorSpecs) : that.postAggregatorSpecs != null) { + if (aggregatorSpecs != null ? !aggregatorSpecs.equals(that.aggregatorSpecs) : that.aggregatorSpecs != null) { return false; } - - return true; + return postAggregatorSpecs != null + ? postAggregatorSpecs.equals(that.postAggregatorSpecs) + : that.postAggregatorSpecs == null; } @Override public int hashCode() { int result = super.hashCode(); + result = 31 * result + virtualColumns.hashCode(); result = 31 * result + (dimFilter != null ? dimFilter.hashCode() : 0); result = 31 * result + (granularity != null ? granularity.hashCode() : 0); result = 31 * result + (aggregatorSpecs != null ? aggregatorSpecs.hashCode() : 0); result = 31 * result + (postAggregatorSpecs != null ? postAggregatorSpecs.hashCode() : 0); return result; } + + @Override + public String toString() + { + return "TimeseriesQuery{" + + "virtualColumns=" + virtualColumns + + ", dimFilter=" + dimFilter + + ", granularity=" + granularity + + ", aggregatorSpecs=" + aggregatorSpecs + + ", postAggregatorSpecs=" + postAggregatorSpecs + + '}'; + } } diff --git a/processing/src/main/java/io/druid/query/timeseries/TimeseriesQueryEngine.java b/processing/src/main/java/io/druid/query/timeseries/TimeseriesQueryEngine.java index 5ee286519cc5..bbe57f5df3d9 100644 --- a/processing/src/main/java/io/druid/query/timeseries/TimeseriesQueryEngine.java +++ b/processing/src/main/java/io/druid/query/timeseries/TimeseriesQueryEngine.java @@ -29,7 +29,6 @@ import io.druid.segment.Cursor; import io.druid.segment.SegmentMissingException; import io.druid.segment.StorageAdapter; -import io.druid.segment.VirtualColumns; import io.druid.segment.filter.Filters; import java.util.List; @@ -52,7 +51,7 @@ public Sequence> process(final TimeseriesQuery que adapter, query.getQuerySegmentSpec().getIntervals(), filter, - VirtualColumns.EMPTY, + query.getVirtualColumns(), query.isDescending(), query.getGranularity(), new Function>() diff --git a/processing/src/main/java/io/druid/query/timeseries/TimeseriesQueryQueryToolChest.java b/processing/src/main/java/io/druid/query/timeseries/TimeseriesQueryQueryToolChest.java index 090220e11e9e..3aa737821890 100644 --- a/processing/src/main/java/io/druid/query/timeseries/TimeseriesQueryQueryToolChest.java +++ b/processing/src/main/java/io/druid/query/timeseries/TimeseriesQueryQueryToolChest.java @@ -138,15 +138,23 @@ public byte[] computeCacheKey(TimeseriesQuery query) final byte[] granularityBytes = query.getGranularity().cacheKey(); final byte descending = query.isDescending() ? (byte) 1 : 0; final byte skipEmptyBuckets = query.isSkipEmptyBuckets() ? (byte) 1 : 0; + final byte[] virtualColumnsBytes = query.getVirtualColumns().getCacheKey(); return ByteBuffer - .allocate(3 + granularityBytes.length + filterBytes.length + aggregatorBytes.length) + .allocate( + 3 + + granularityBytes.length + + filterBytes.length + + aggregatorBytes.length + + virtualColumnsBytes.length + ) .put(TIMESERIES_QUERY) .put(descending) .put(skipEmptyBuckets) .put(granularityBytes) .put(filterBytes) .put(aggregatorBytes) + .put(virtualColumnsBytes) .array(); } diff --git a/processing/src/main/java/io/druid/query/topn/AggregateTopNMetricFirstAlgorithm.java b/processing/src/main/java/io/druid/query/topn/AggregateTopNMetricFirstAlgorithm.java index 3a7d16e51a63..644ddaf683a5 100644 --- a/processing/src/main/java/io/druid/query/topn/AggregateTopNMetricFirstAlgorithm.java +++ b/processing/src/main/java/io/druid/query/topn/AggregateTopNMetricFirstAlgorithm.java @@ -81,10 +81,10 @@ public void run( throw new ISE("WTF! Can't find the metric to do topN over?"); } // Run topN for only a single metric - TopNQuery singleMetricQuery = new TopNQueryBuilder().copy(query) - .aggregators(condensedAggPostAggPair.lhs) - .postAggregators(condensedAggPostAggPair.rhs) - .build(); + TopNQuery singleMetricQuery = new TopNQueryBuilder(query) + .aggregators(condensedAggPostAggPair.lhs) + .postAggregators(condensedAggPostAggPair.rhs) + .build(); final TopNResultBuilder singleMetricResultBuilder = BaseTopNAlgorithm.makeResultBuilder(params, singleMetricQuery); PooledTopNAlgorithm singleMetricAlgo = new PooledTopNAlgorithm(capabilities, singleMetricQuery, bufferPool); diff --git a/processing/src/main/java/io/druid/query/topn/TopNQuery.java b/processing/src/main/java/io/druid/query/topn/TopNQuery.java index 2221e65d4863..41915b122c6e 100644 --- a/processing/src/main/java/io/druid/query/topn/TopNQuery.java +++ b/processing/src/main/java/io/druid/query/topn/TopNQuery.java @@ -34,6 +34,7 @@ import io.druid.query.dimension.DimensionSpec; import io.druid.query.filter.DimFilter; import io.druid.query.spec.QuerySegmentSpec; +import io.druid.segment.virtual.VirtualColumns; import java.util.List; import java.util.Map; @@ -44,6 +45,7 @@ public class TopNQuery extends BaseQuery> { public static final String TOPN = "topN"; + private final VirtualColumns virtualColumns; private final DimensionSpec dimensionSpec; private final TopNMetricSpec topNMetricSpec; private final int threshold; @@ -55,6 +57,7 @@ public class TopNQuery extends BaseQuery> @JsonCreator public TopNQuery( @JsonProperty("dataSource") DataSource dataSource, + @JsonProperty("virtualColumns") VirtualColumns virtualColumns, @JsonProperty("dimension") DimensionSpec dimensionSpec, @JsonProperty("metric") TopNMetricSpec topNMetricSpec, @JsonProperty("threshold") int threshold, @@ -67,6 +70,7 @@ public TopNQuery( ) { super(dataSource, querySegmentSpec, false, context); + this.virtualColumns = virtualColumns == null ? VirtualColumns.EMPTY : virtualColumns; this.dimensionSpec = dimensionSpec; this.topNMetricSpec = topNMetricSpec; this.threshold = threshold; @@ -103,6 +107,12 @@ public String getType() return TOPN; } + @JsonProperty + public VirtualColumns getVirtualColumns() + { + return virtualColumns; + } + @JsonProperty("dimension") public DimensionSpec getDimensionSpec() { @@ -157,6 +167,7 @@ public TopNQuery withQuerySegmentSpec(QuerySegmentSpec querySegmentSpec) { return new TopNQuery( getDataSource(), + virtualColumns, dimensionSpec, topNMetricSpec, threshold, @@ -173,6 +184,7 @@ public TopNQuery withDimensionSpec(DimensionSpec spec) { return new TopNQuery( getDataSource(), + virtualColumns, spec, topNMetricSpec, threshold, @@ -189,6 +201,7 @@ public TopNQuery withAggregatorSpecs(List aggregatorSpecs) { return new TopNQuery( getDataSource(), + virtualColumns, getDimensionSpec(), topNMetricSpec, threshold, @@ -205,6 +218,7 @@ public TopNQuery withPostAggregatorSpecs(List postAggregatorSpec { return new TopNQuery( getDataSource(), + virtualColumns, getDimensionSpec(), topNMetricSpec, threshold, @@ -222,6 +236,7 @@ public Query> withDataSource(DataSource dataSource) { return new TopNQuery( dataSource, + virtualColumns, dimensionSpec, topNMetricSpec, threshold, @@ -238,6 +253,7 @@ public TopNQuery withThreshold(int threshold) { return new TopNQuery( getDataSource(), + virtualColumns, dimensionSpec, topNMetricSpec, threshold, @@ -254,6 +270,7 @@ public TopNQuery withOverriddenContext(Map contextOverrides) { return new TopNQuery( getDataSource(), + virtualColumns, dimensionSpec, topNMetricSpec, threshold, @@ -270,6 +287,7 @@ public TopNQuery withDimFilter(DimFilter dimFilter) { return new TopNQuery( getDataSource(), + virtualColumns, getDimensionSpec(), topNMetricSpec, threshold, @@ -282,22 +300,6 @@ public TopNQuery withDimFilter(DimFilter dimFilter) ); } - @Override - public String toString() - { - return "TopNQuery{" + - "dataSource='" + getDataSource() + '\'' + - ", dimensionSpec=" + dimensionSpec + - ", topNMetricSpec=" + topNMetricSpec + - ", threshold=" + threshold + - ", querySegmentSpec=" + getQuerySegmentSpec() + - ", dimFilter=" + dimFilter + - ", granularity='" + granularity + '\'' + - ", aggregatorSpecs=" + aggregatorSpecs + - ", postAggregatorSpecs=" + postAggregatorSpecs + - '}'; - } - @Override public boolean equals(Object o) { @@ -311,37 +313,39 @@ public boolean equals(Object o) return false; } - TopNQuery topNQuery = (TopNQuery) o; + TopNQuery query = (TopNQuery) o; - if (threshold != topNQuery.threshold) { + if (threshold != query.threshold) { return false; } - if (aggregatorSpecs != null ? !aggregatorSpecs.equals(topNQuery.aggregatorSpecs) : topNQuery.aggregatorSpecs != null) { + if (!virtualColumns.equals(query.virtualColumns)) { return false; } - if (dimFilter != null ? !dimFilter.equals(topNQuery.dimFilter) : topNQuery.dimFilter != null) { + if (dimensionSpec != null ? !dimensionSpec.equals(query.dimensionSpec) : query.dimensionSpec != null) { return false; } - if (dimensionSpec != null ? !dimensionSpec.equals(topNQuery.dimensionSpec) : topNQuery.dimensionSpec != null) { + if (topNMetricSpec != null ? !topNMetricSpec.equals(query.topNMetricSpec) : query.topNMetricSpec != null) { return false; } - if (granularity != null ? !granularity.equals(topNQuery.granularity) : topNQuery.granularity != null) { + if (dimFilter != null ? !dimFilter.equals(query.dimFilter) : query.dimFilter != null) { return false; } - if (postAggregatorSpecs != null ? !postAggregatorSpecs.equals(topNQuery.postAggregatorSpecs) : topNQuery.postAggregatorSpecs != null) { + if (granularity != null ? !granularity.equals(query.granularity) : query.granularity != null) { return false; } - if (topNMetricSpec != null ? !topNMetricSpec.equals(topNQuery.topNMetricSpec) : topNQuery.topNMetricSpec != null) { + if (aggregatorSpecs != null ? !aggregatorSpecs.equals(query.aggregatorSpecs) : query.aggregatorSpecs != null) { return false; } - - return true; + return postAggregatorSpecs != null + ? postAggregatorSpecs.equals(query.postAggregatorSpecs) + : query.postAggregatorSpecs == null; } @Override public int hashCode() { int result = super.hashCode(); + result = 31 * result + virtualColumns.hashCode(); result = 31 * result + (dimensionSpec != null ? dimensionSpec.hashCode() : 0); result = 31 * result + (topNMetricSpec != null ? topNMetricSpec.hashCode() : 0); result = 31 * result + threshold; @@ -351,4 +355,19 @@ public int hashCode() result = 31 * result + (postAggregatorSpecs != null ? postAggregatorSpecs.hashCode() : 0); return result; } + + @Override + public String toString() + { + return "TopNQuery{" + + "virtualColumns=" + virtualColumns + + ", dimensionSpec=" + dimensionSpec + + ", topNMetricSpec=" + topNMetricSpec + + ", threshold=" + threshold + + ", dimFilter=" + dimFilter + + ", granularity=" + granularity + + ", aggregatorSpecs=" + aggregatorSpecs + + ", postAggregatorSpecs=" + postAggregatorSpecs + + '}'; + } } diff --git a/processing/src/main/java/io/druid/query/topn/TopNQueryBuilder.java b/processing/src/main/java/io/druid/query/topn/TopNQueryBuilder.java index bdd09b95353d..2d0f077c6868 100644 --- a/processing/src/main/java/io/druid/query/topn/TopNQueryBuilder.java +++ b/processing/src/main/java/io/druid/query/topn/TopNQueryBuilder.java @@ -33,19 +33,22 @@ import io.druid.query.filter.SelectorDimFilter; import io.druid.query.spec.LegacySegmentSpec; import io.druid.query.spec.QuerySegmentSpec; +import io.druid.segment.virtual.VirtualColumn; +import io.druid.segment.virtual.VirtualColumns; import org.joda.time.Interval; +import java.util.Arrays; import java.util.List; import java.util.Map; /** * A Builder for TopNQuery. - * + * * Required: dataSource(), intervals(), metric() and threshold() must be called before build() * Additional requirement for numeric metric sorts: aggregators() must be called before build() - * + * * Optional: filters(), granularity(), postAggregators() and context() can be called before build() - * + * * Usage example: * 

  *   TopNQuery query = new TopNQueryBuilder()
@@ -62,6 +65,7 @@
 public class TopNQueryBuilder
 {
   private DataSource dataSource;
+  private VirtualColumns virtualColumns;
   private DimensionSpec dimensionSpec;
   private TopNMetricSpec topNMetricSpec;
   private int threshold;
@@ -75,6 +79,7 @@ public class TopNQueryBuilder
   public TopNQueryBuilder()
   {
     dataSource = null;
+    virtualColumns = null;
     dimensionSpec = null;
     topNMetricSpec = null;
     threshold = 0;
@@ -86,11 +91,31 @@ public TopNQueryBuilder()
     context = null;
   }
 
+  public TopNQueryBuilder(final TopNQuery query)
+  {
+      this.dataSource = query.getDataSource();
+      this.virtualColumns = query.getVirtualColumns();
+      this.dimensionSpec = query.getDimensionSpec();
+      this.topNMetricSpec = query.getTopNMetricSpec();
+      this.threshold = query.getThreshold();
+      this.querySegmentSpec = query.getQuerySegmentSpec();
+      this.dimFilter = query.getDimensionsFilter();
+      this.granularity = query.getGranularity();
+      this.aggregatorSpecs = query.getAggregatorSpecs();
+      this.postAggregatorSpecs = query.getPostAggregatorSpecs();
+      this.context = query.getContext();
+  }
+
   public DataSource getDataSource()
   {
     return dataSource;
   }
 
+  public VirtualColumns getVirtualColumns()
+  {
+    return virtualColumns;
+  }
+
   public DimensionSpec getDimensionSpec()
   {
     return dimensionSpec;
@@ -140,6 +165,7 @@ public TopNQuery build()
   {
     return new TopNQuery(
         dataSource,
+        virtualColumns,
         dimensionSpec,
         topNMetricSpec,
         threshold,
@@ -152,25 +178,18 @@ public TopNQuery build()
     );
   }
 
+  @Deprecated
   public TopNQueryBuilder copy(TopNQuery query)
   {
-    return new TopNQueryBuilder()
-        .dataSource(query.getDataSource().toString())
-        .dimension(query.getDimensionSpec())
-        .metric(query.getTopNMetricSpec())
-        .threshold(query.getThreshold())
-        .intervals(query.getIntervals())
-        .filters(query.getDimensionsFilter())
-        .granularity(query.getGranularity())
-        .aggregators(query.getAggregatorSpecs())
-        .postAggregators(query.getPostAggregatorSpecs())
-        .context(query.getContext());
+    return new TopNQueryBuilder(query);
   }
 
+  @Deprecated
   public TopNQueryBuilder copy(TopNQueryBuilder builder)
   {
     return new TopNQueryBuilder()
         .dataSource(builder.dataSource)
+        .virtualColumns(builder.virtualColumns)
         .dimension(builder.dimensionSpec)
         .metric(builder.topNMetricSpec)
         .threshold(builder.threshold)
@@ -188,6 +207,22 @@ public TopNQueryBuilder dataSource(String d)
     return this;
   }
 
+  public TopNQueryBuilder virtualColumns(VirtualColumns virtualColumns)
+  {
+    this.virtualColumns = virtualColumns;
+    return this;
+  }
+
+  public TopNQueryBuilder virtualColumns(List virtualColumns)
+  {
+    return virtualColumns(VirtualColumns.create(virtualColumns));
+  }
+
+  public TopNQueryBuilder virtualColumns(VirtualColumn... virtualColumns)
+  {
+    return virtualColumns(VirtualColumns.create(Arrays.asList(virtualColumns)));
+  }
+
   public TopNQueryBuilder dataSource(DataSource d)
   {
     dataSource = d;
diff --git a/processing/src/main/java/io/druid/query/topn/TopNQueryEngine.java b/processing/src/main/java/io/druid/query/topn/TopNQueryEngine.java
index bb4e121f6e61..e1467da10494 100644
--- a/processing/src/main/java/io/druid/query/topn/TopNQueryEngine.java
+++ b/processing/src/main/java/io/druid/query/topn/TopNQueryEngine.java
@@ -35,7 +35,6 @@
 import io.druid.segment.Cursor;
 import io.druid.segment.SegmentMissingException;
 import io.druid.segment.StorageAdapter;
-import io.druid.segment.VirtualColumns;
 import io.druid.segment.column.Column;
 import io.druid.segment.filter.Filters;
 import org.joda.time.Interval;
@@ -75,7 +74,13 @@ public Sequence> query(final TopNQuery query, final Stor
 
     return Sequences.filter(
         Sequences.map(
-            adapter.makeCursors(filter, queryIntervals.get(0), VirtualColumns.EMPTY, granularity, query.isDescending()),
+            adapter.makeCursors(
+                filter,
+                queryIntervals.get(0),
+                query.getVirtualColumns(),
+                granularity,
+                query.isDescending()
+            ),
             new Function>()
             {
               @Override
diff --git a/processing/src/main/java/io/druid/query/topn/TopNQueryQueryToolChest.java b/processing/src/main/java/io/druid/query/topn/TopNQueryQueryToolChest.java
index 052f14710d5c..8a1104470398 100644
--- a/processing/src/main/java/io/druid/query/topn/TopNQueryQueryToolChest.java
+++ b/processing/src/main/java/io/druid/query/topn/TopNQueryQueryToolChest.java
@@ -313,11 +313,13 @@ public byte[] computeCacheKey(TopNQuery query)
         final byte[] filterBytes = dimFilter == null ? new byte[]{} : dimFilter.getCacheKey();
         final byte[] aggregatorBytes = QueryCacheHelper.computeAggregatorBytes(query.getAggregatorSpecs());
         final byte[] granularityBytes = query.getGranularity().cacheKey();
+        final byte[] virtualColumnBytes = query.getVirtualColumns().getCacheKey();
 
         return ByteBuffer
             .allocate(
-                1 + dimensionSpecBytes.length + metricSpecBytes.length + 4 +
-                granularityBytes.length + filterBytes.length + aggregatorBytes.length
+                1 + dimensionSpecBytes.length + metricSpecBytes.length + 4
+                + granularityBytes.length + filterBytes.length + aggregatorBytes.length
+                + virtualColumnBytes.length
             )
             .put(TOPN_QUERY)
             .put(dimensionSpecBytes)
@@ -326,6 +328,7 @@ public byte[] computeCacheKey(TopNQuery query)
             .put(granularityBytes)
             .put(filterBytes)
             .put(aggregatorBytes)
+            .put(virtualColumnBytes)
             .array();
       }
 
diff --git a/processing/src/main/java/io/druid/segment/ColumnSelectorFactory.java b/processing/src/main/java/io/druid/segment/ColumnSelectorFactory.java
index afb38476c048..5dcc6a97c868 100644
--- a/processing/src/main/java/io/druid/segment/ColumnSelectorFactory.java
+++ b/processing/src/main/java/io/druid/segment/ColumnSelectorFactory.java
@@ -20,17 +20,25 @@
 package io.druid.segment;
 
 import io.druid.query.dimension.DimensionSpec;
-import io.druid.segment.column.ColumnCapabilities;
+import io.druid.segment.column.ValueType;
 
 /**
  * Factory class for MetricSelectors
  */
 public interface ColumnSelectorFactory
 {
-  public DimensionSelector makeDimensionSelector(DimensionSpec dimensionSpec);
-  public FloatColumnSelector makeFloatColumnSelector(String columnName);
-  public LongColumnSelector makeLongColumnSelector(String columnName);
-  public ObjectColumnSelector makeObjectColumnSelector(String columnName);
-  public NumericColumnSelector makeMathExpressionSelector(String expression);
-  public ColumnCapabilities getColumnCapabilities(String columnName);
+  DimensionSelector makeDimensionSelector(DimensionSpec dimensionSpec);
+  FloatColumnSelector makeFloatColumnSelector(String columnName);
+  LongColumnSelector makeLongColumnSelector(String columnName);
+  ObjectColumnSelector makeObjectColumnSelector(String columnName);
+
+  /**
+   * Returns the native type of the named column, which should match the type returned, which
+   * should correspond to the best performing selector. May be null if unknown.
+   *
+   * @param columnName name of the column
+   *
+   * @return native type, or null if unknown
+   */
+  ValueType getNativeType(String columnName);
 }
diff --git a/processing/src/main/java/io/druid/segment/CursorFactory.java b/processing/src/main/java/io/druid/segment/CursorFactory.java
index b9e409f05651..f8137d2fddca 100644
--- a/processing/src/main/java/io/druid/segment/CursorFactory.java
+++ b/processing/src/main/java/io/druid/segment/CursorFactory.java
@@ -22,6 +22,7 @@
 import io.druid.granularity.QueryGranularity;
 import io.druid.java.util.common.guava.Sequence;
 import io.druid.query.filter.Filter;
+import io.druid.segment.virtual.VirtualColumns;
 import org.joda.time.Interval;
 
 /**
diff --git a/processing/src/main/java/io/druid/segment/NullDimensionSelector.java b/processing/src/main/java/io/druid/segment/NullDimensionSelector.java
index f771afe408e1..620166f32eb1 100644
--- a/processing/src/main/java/io/druid/segment/NullDimensionSelector.java
+++ b/processing/src/main/java/io/druid/segment/NullDimensionSelector.java
@@ -28,20 +28,34 @@
 
 public class NullDimensionSelector implements DimensionSelector
 {
+  private static final NullDimensionSelector INSTANCE = new NullDimensionSelector();
 
-  private static final IndexedInts SINGLETON = new IndexedInts() {
+  private NullDimensionSelector()
+  {
+  }
+
+  public static final NullDimensionSelector instance()
+  {
+    return INSTANCE;
+  }
+
+  private static final IndexedInts SINGLETON = new IndexedInts()
+  {
     @Override
-    public int size() {
+    public int size()
+    {
       return 1;
     }
 
     @Override
-    public int get(int index) {
+    public int get(int index)
+    {
       return 0;
     }
 
     @Override
-    public IntIterator iterator() {
+    public IntIterator iterator()
+    {
       return IntIterators.singleton(0);
     }
 
diff --git a/processing/src/main/java/io/druid/segment/QueryableIndexStorageAdapter.java b/processing/src/main/java/io/druid/segment/QueryableIndexStorageAdapter.java
index 9e3a04a017cf..b7fcd6c78c44 100644
--- a/processing/src/main/java/io/druid/segment/QueryableIndexStorageAdapter.java
+++ b/processing/src/main/java/io/druid/segment/QueryableIndexStorageAdapter.java
@@ -23,19 +23,15 @@
 import com.google.common.base.Predicate;
 import com.google.common.base.Predicates;
 import com.google.common.base.Strings;
-import com.google.common.base.Supplier;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;
 import com.google.common.collect.Sets;
 import com.google.common.io.Closer;
-
 import io.druid.collections.bitmap.ImmutableBitmap;
 import io.druid.granularity.QueryGranularity;
 import io.druid.java.util.common.guava.Sequence;
 import io.druid.java.util.common.guava.Sequences;
-import io.druid.math.expr.Expr;
-import io.druid.math.expr.Parser;
 import io.druid.query.QueryInterruptedException;
 import io.druid.query.dimension.DefaultDimensionSpec;
 import io.druid.query.dimension.DimensionSpec;
@@ -50,7 +46,6 @@
 import io.druid.segment.column.BitmapIndex;
 import io.druid.segment.column.Column;
 import io.druid.segment.column.ColumnCapabilities;
-import io.druid.segment.column.ColumnCapabilitiesImpl;
 import io.druid.segment.column.ComplexColumn;
 import io.druid.segment.column.DictionaryEncodedColumn;
 import io.druid.segment.column.GenericColumn;
@@ -61,6 +56,7 @@
 import io.druid.segment.filter.AndFilter;
 import io.druid.segment.filter.BooleanValueMatcher;
 import io.druid.segment.filter.Filters;
+import io.druid.segment.virtual.VirtualColumns;
 import it.unimi.dsi.fastutil.ints.IntIterators;
 import org.joda.time.DateTime;
 import org.joda.time.Interval;
@@ -76,8 +72,6 @@
  */
 public class QueryableIndexStorageAdapter implements StorageAdapter
 {
-  private static final NullDimensionSelector NULL_DIMENSION_SELECTOR = new NullDimensionSelector();
-
   private final QueryableIndex index;
 
   public QueryableIndexStorageAdapter(
@@ -436,6 +430,10 @@ public DimensionSelector makeDimensionSelector(
                         DimensionSpec dimensionSpec
                     )
                     {
+                      if (virtualColumns.exists(dimensionSpec.getDimension())) {
+                        return virtualColumns.makeDimensionSelector(dimensionSpec, this);
+                      }
+
                       return dimensionSpec.decorate(makeDimensionSelectorUndecorated(dimensionSpec));
                     }
 
@@ -448,7 +446,7 @@ private DimensionSelector makeDimensionSelectorUndecorated(
 
                       final Column columnDesc = index.getColumn(dimension);
                       if (columnDesc == null) {
-                        return NULL_DIMENSION_SELECTOR;
+                        return NullDimensionSelector.instance();
                       }
 
                       if (dimension.equals(Column.TIME_COLUMN_NAME)) {
@@ -469,7 +467,7 @@ private DimensionSelector makeDimensionSelectorUndecorated(
                       final DictionaryEncodedColumn column = cachedColumn;
 
                       if (column == null) {
-                        return NULL_DIMENSION_SELECTOR;
+                        return NullDimensionSelector.instance();
                       } else if (columnDesc.getCapabilities().hasMultipleValues()) {
                         return new DimensionSelector()
                         {
@@ -577,6 +575,10 @@ public int lookupId(String name)
                     @Override
                     public FloatColumnSelector makeFloatColumnSelector(String columnName)
                     {
+                      if (virtualColumns.exists(columnName)) {
+                        return virtualColumns.makeFloatColumnSelector(columnName, this);
+                      }
+
                       GenericColumn cachedMetricVals = genericColumnCache.get(columnName);
 
                       if (cachedMetricVals == null) {
@@ -590,14 +592,7 @@ public FloatColumnSelector makeFloatColumnSelector(String columnName)
                       }
 
                       if (cachedMetricVals == null) {
-                        return new FloatColumnSelector()
-                        {
-                          @Override
-                          public float get()
-                          {
-                            return 0.0f;
-                          }
-                        };
+                        return ZeroFloatColumnSelector.instance();
                       }
 
                       final GenericColumn metricVals = cachedMetricVals;
@@ -614,6 +609,10 @@ public float get()
                     @Override
                     public LongColumnSelector makeLongColumnSelector(String columnName)
                     {
+                      if (virtualColumns.exists(columnName)) {
+                        return virtualColumns.makeLongColumnSelector(columnName, this);
+                      }
+
                       GenericColumn cachedMetricVals = genericColumnCache.get(columnName);
 
                       if (cachedMetricVals == null) {
@@ -627,14 +626,7 @@ public LongColumnSelector makeLongColumnSelector(String columnName)
                       }
 
                       if (cachedMetricVals == null) {
-                        return new LongColumnSelector()
-                        {
-                          @Override
-                          public long get()
-                          {
-                            return 0L;
-                          }
-                        };
+                        return ZeroLongColumnSelector.instance();
                       }
 
                       final GenericColumn metricVals = cachedMetricVals;
@@ -652,6 +644,10 @@ public long get()
                     @Override
                     public ObjectColumnSelector makeObjectColumnSelector(String column)
                     {
+                      if (virtualColumns.exists(column)) {
+                        return virtualColumns.makeObjectColumnSelector(column, this);
+                      }
+
                       Object cachedColumnVals = objectColumnCache.get(column);
 
                       if (cachedColumnVals == null) {
@@ -676,10 +672,6 @@ public ObjectColumnSelector makeObjectColumnSelector(String column)
                       }
 
                       if (cachedColumnVals == null) {
-                        VirtualColumn vc = virtualColumns.getVirtualColumn(column);
-                        if (vc != null) {
-                          return vc.init(column, this);
-                        }
                         return null;
                       }
 
@@ -807,72 +799,10 @@ public Object get()
                     }
 
                     @Override
-                    public NumericColumnSelector makeMathExpressionSelector(String expression)
-                    {
-                      final Expr parsed = Parser.parse(expression);
-                      final List required = Parser.findRequiredBindings(parsed);
-
-                      final Map> values = Maps.newHashMapWithExpectedSize(required.size());
-                      for (String columnName : index.getColumnNames()) {
-                        if (!required.contains(columnName)) {
-                          continue;
-                        }
-                        final GenericColumn column = index.getColumn(columnName).getGenericColumn();
-                        if (column == null) {
-                          continue;
-                        }
-                        closer.register(column);
-                        if (column.getType() == ValueType.FLOAT) {
-                          values.put(
-                              columnName, new Supplier()
-                              {
-                                @Override
-                                public Number get()
-                                {
-                                  return column.getFloatSingleValueRow(cursorOffset.getOffset());
-                                }
-                              }
-                          );
-                        } else if (column.getType() == ValueType.LONG) {
-                          values.put(
-                              columnName, new Supplier()
-                              {
-                                @Override
-                                public Number get()
-                                {
-                                  return column.getLongSingleValueRow(cursorOffset.getOffset());
-                                }
-                              }
-                          );
-                        } else {
-                          throw new UnsupportedOperationException(
-                              "Not supported type " + column.getType() + " for column " + columnName
-                          );
-                        }
-                      }
-                      final Expr.ObjectBinding binding = Parser.withSuppliers(values);
-                      return new NumericColumnSelector()
-                      {
-                        @Override
-                        public Number get()
-                        {
-                          return parsed.eval(binding).numericValue();
-                        }
-                      };
-                    }
-
-                    @Override
-                    public ColumnCapabilities getColumnCapabilities(String columnName)
+                    public ValueType getNativeType(String columnName)
                     {
-                      ColumnCapabilities capabilities = getColumnCapabilites(index, columnName);
-                      if (capabilities == null && !virtualColumns.isEmpty()) {
-                        VirtualColumn virtualColumn = virtualColumns.getVirtualColumn(columnName);
-                        if (virtualColumn != null) {
-                          Class clazz = virtualColumn.init(columnName, this).classOfObject();
-                          capabilities = new ColumnCapabilitiesImpl().setType(ValueType.typeFor(clazz));
-                        }
-                      }
-                      return capabilities;
+                      final ColumnCapabilities capabilities = getColumnCapabilites(index, columnName);
+                      return capabilities != null ? capabilities.getType() : virtualColumns.getNativeType(columnName);
                     }
                   }
 
@@ -1054,7 +984,7 @@ public CursorOffsetHolderValueMatcherFactory(
     @Override
     public ValueMatcher makeValueMatcher(String dimension, final String value)
     {
-      if (getTypeForDimension(dimension) == ValueType.LONG) {
+      if (cursor.getNativeType(dimension) == ValueType.LONG) {
         return Filters.getLongValueMatcher(
             cursor.makeLongColumnSelector(dimension),
             value
@@ -1095,14 +1025,13 @@ public boolean matches()
     @Override
     public ValueMatcher makeValueMatcher(String dimension, final DruidPredicateFactory predicateFactory)
     {
-      ValueType type = getTypeForDimension(dimension);
-      switch (type) {
-        case LONG:
-          return makeLongValueMatcher(dimension, predicateFactory.makeLongPredicate());
-        case STRING:
-          return makeStringValueMatcher(dimension, predicateFactory.makeStringPredicate());
-        default:
-          return new BooleanValueMatcher(predicateFactory.makeStringPredicate().apply(null));
+      final ValueType type = cursor.getNativeType(dimension);
+      if (type == ValueType.LONG) {
+        return makeLongValueMatcher(dimension, predicateFactory.makeLongPredicate());
+      } else if (type == ValueType.STRING) {
+        return makeStringValueMatcher(dimension, predicateFactory.makeStringPredicate());
+      } else {
+        return new BooleanValueMatcher(predicateFactory.makeStringPredicate().apply(null));
       }
     }
 
@@ -1140,12 +1069,6 @@ private ValueMatcher makeLongValueMatcher(String dimension, final DruidLongPredi
           predicate
       );
     }
-
-    private ValueType getTypeForDimension(String dimension)
-    {
-      ColumnCapabilities capabilities = getColumnCapabilites(index, dimension);
-      return capabilities == null ? ValueType.STRING : capabilities.getType();
-    }
   }
 
   private static class CursorOffsetHolderRowOffsetMatcherFactory implements RowOffsetMatcherFactory
diff --git a/processing/src/main/java/io/druid/segment/VirtualColumn.java b/processing/src/main/java/io/druid/segment/VirtualColumn.java
deleted file mode 100644
index 855affe8bf4f..000000000000
--- a/processing/src/main/java/io/druid/segment/VirtualColumn.java
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Licensed to Metamarkets Group Inc. (Metamarkets) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. Metamarkets licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package io.druid.segment;
-
-import com.fasterxml.jackson.annotation.JsonTypeInfo;
-
-/**
- */
-@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type")
-/**
- * Virtual columns are "views" created over a ColumnSelectorFactory. They can potentially draw from multiple
- * underlying columns, although they always present themselves as if they were a single column.
- */
-public interface VirtualColumn
-{
-  /**
-   * Output name of this column.
-   *
-   * @return name
-   */
-  String getOutputName();
-
-  /**
-   * Build a selector corresponding to this virtual column. Also provides the name that the
-   * virtual column was referenced with, which is useful if this column uses dot notation.
-   *
-   * @param columnName the name this virtual column was referenced with
-   * @param factory column selector factory
-   * @return the selector
-   */
-  ObjectColumnSelector init(String columnName, ColumnSelectorFactory factory);
-
-  /**
-   * Indicates that this virtual column can be referenced with dot notation. For example,
-   * a virtual column named "foo" could be referred to as "foo.bar" with the Cursor it is
-   * registered with. In that case, init will be called with columnName "foo.bar" rather
-   * than "foo".
-   *
-   * @return whether to use dot notation
-   */
-  boolean usesDotNotation();
-
-  /**
-   * Returns cache key
-   *
-   * @return cache key
-   */
-  byte[] getCacheKey();
-}
diff --git a/processing/src/main/java/io/druid/segment/VirtualColumns.java b/processing/src/main/java/io/druid/segment/VirtualColumns.java
deleted file mode 100644
index 799a0ca366e7..000000000000
--- a/processing/src/main/java/io/druid/segment/VirtualColumns.java
+++ /dev/null
@@ -1,80 +0,0 @@
-/*
- * Licensed to Metamarkets Group Inc. (Metamarkets) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. Metamarkets licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package io.druid.segment;
-
-import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.Maps;
-
-import java.util.List;
-import java.util.Map;
-
-/**
- */
-public class VirtualColumns
-{
-  public static final VirtualColumns EMPTY = new VirtualColumns(
-      ImmutableMap.of(), ImmutableMap.of()
-  );
-
-  public static VirtualColumns valueOf(List virtualColumns) {
-    if (virtualColumns == null || virtualColumns.isEmpty()) {
-      return EMPTY;
-    }
-    Map withDotSupport = Maps.newHashMap();
-    Map withoutDotSupport = Maps.newHashMap();
-    for (VirtualColumn vc : virtualColumns) {
-      if (vc.usesDotNotation()) {
-        withDotSupport.put(vc.getOutputName(), vc);
-      } else {
-        withoutDotSupport.put(vc.getOutputName(), vc);
-      }
-    }
-    return new VirtualColumns(withDotSupport, withoutDotSupport);
-  }
-
-  public VirtualColumns(Map withDotSupport, Map withoutDotSupport)
-  {
-    this.withDotSupport = withDotSupport;
-    this.withoutDotSupport = withoutDotSupport;
-  }
-
-  private final Map withDotSupport;
-  private final Map withoutDotSupport;
-
-  public VirtualColumn getVirtualColumn(String dimension)
-  {
-    VirtualColumn vc = withoutDotSupport.get(dimension);
-    if (vc != null) {
-      return vc;
-    }
-    for (int index = dimension.indexOf('.'); index >= 0; index = dimension.indexOf('.', index + 1)) {
-      vc = withDotSupport.get(dimension.substring(0, index));
-      if (vc != null) {
-        return vc;
-      }
-    }
-    return withDotSupport.get(dimension);
-  }
-
-  public boolean isEmpty()
-  {
-    return withDotSupport.isEmpty() && withoutDotSupport.isEmpty();
-  }
-}
diff --git a/processing/src/main/java/io/druid/segment/ZeroFloatColumnSelector.java b/processing/src/main/java/io/druid/segment/ZeroFloatColumnSelector.java
new file mode 100644
index 000000000000..8e8eb99a4f8b
--- /dev/null
+++ b/processing/src/main/java/io/druid/segment/ZeroFloatColumnSelector.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to Metamarkets Group Inc. (Metamarkets) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. Metamarkets licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package io.druid.segment;
+
+public class ZeroFloatColumnSelector implements FloatColumnSelector
+{
+  private static final ZeroFloatColumnSelector INSTANCE = new ZeroFloatColumnSelector();
+
+  private ZeroFloatColumnSelector()
+  {
+    // No instantiation.
+  }
+
+  public static ZeroFloatColumnSelector instance()
+  {
+    return INSTANCE;
+  }
+
+  @Override
+  public float get()
+  {
+    return 0.0f;
+  }
+}
diff --git a/processing/src/main/java/io/druid/segment/ZeroLongColumnSelector.java b/processing/src/main/java/io/druid/segment/ZeroLongColumnSelector.java
new file mode 100644
index 000000000000..a24f16111c9f
--- /dev/null
+++ b/processing/src/main/java/io/druid/segment/ZeroLongColumnSelector.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to Metamarkets Group Inc. (Metamarkets) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. Metamarkets licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package io.druid.segment;
+
+public class ZeroLongColumnSelector implements LongColumnSelector
+{
+  private static final ZeroLongColumnSelector INSTANCE = new ZeroLongColumnSelector();
+
+  private ZeroLongColumnSelector()
+  {
+    // No instantiation.
+  }
+
+  public static ZeroLongColumnSelector instance()
+  {
+    return INSTANCE;
+  }
+
+  @Override
+  public long get()
+  {
+    return 0;
+  }
+}
diff --git a/processing/src/main/java/io/druid/segment/column/ValueType.java b/processing/src/main/java/io/druid/segment/column/ValueType.java
index 1760fea8c725..aa9919a129b9 100644
--- a/processing/src/main/java/io/druid/segment/column/ValueType.java
+++ b/processing/src/main/java/io/druid/segment/column/ValueType.java
@@ -26,17 +26,5 @@ public enum ValueType
   FLOAT,
   LONG,
   STRING,
-  COMPLEX;
-
-  public static ValueType typeFor(Class clazz)
-  {
-    if (clazz == String.class) {
-      return STRING;
-    } else if (clazz == float.class || clazz == Float.TYPE) {
-      return FLOAT;
-    } else if (clazz == long.class || clazz == Long.TYPE) {
-      return LONG;
-    }
-    return COMPLEX;
-  }
+  COMPLEX
 }
diff --git a/processing/src/main/java/io/druid/segment/incremental/IncrementalIndex.java b/processing/src/main/java/io/druid/segment/incremental/IncrementalIndex.java
index 6ec6db6d5f9e..dfe82e605e9b 100644
--- a/processing/src/main/java/io/druid/segment/incremental/IncrementalIndex.java
+++ b/processing/src/main/java/io/druid/segment/incremental/IncrementalIndex.java
@@ -38,9 +38,6 @@
 import io.druid.data.input.impl.DimensionsSpec;
 import io.druid.data.input.impl.SpatialDimensionSchema;
 import io.druid.granularity.QueryGranularity;
-import io.druid.math.expr.Evals;
-import io.druid.math.expr.Expr;
-import io.druid.math.expr.Parser;
 import io.druid.java.util.common.IAE;
 import io.druid.java.util.common.ISE;
 import io.druid.query.aggregation.AggregatorFactory;
@@ -55,7 +52,6 @@
 import io.druid.segment.FloatColumnSelector;
 import io.druid.segment.LongColumnSelector;
 import io.druid.segment.Metadata;
-import io.druid.segment.NumericColumnSelector;
 import io.druid.segment.ObjectColumnSelector;
 import io.druid.segment.column.Column;
 import io.druid.segment.column.ColumnCapabilities;
@@ -65,6 +61,7 @@
 import io.druid.segment.serde.ComplexMetricExtractor;
 import io.druid.segment.serde.ComplexMetricSerde;
 import io.druid.segment.serde.ComplexMetrics;
+import io.druid.segment.virtual.VirtualColumns;
 import it.unimi.dsi.fastutil.ints.IntIterator;
 import it.unimi.dsi.fastutil.ints.IntIterators;
 import org.joda.time.DateTime;
@@ -108,14 +105,23 @@ public abstract class IncrementalIndex implements Iterable,
       .put(DimensionSchema.ValueType.STRING, ValueType.STRING)
       .build();
 
+  /**
+   * Column selector used at ingestion time for inputs to aggregators.
+   *
+   * @param agg                       the aggregator
+   * @param in                        ingestion-time input row supplier
+   * @param deserializeComplexMetrics whether complex objects should be deserialized by a {@link ComplexMetricExtractor}
+   *
+   * @return column selector factory
+   */
   public static ColumnSelectorFactory makeColumnSelectorFactory(
+      final VirtualColumns virtualColumns,
       final AggregatorFactory agg,
       final Supplier in,
-      final boolean deserializeComplexMetrics,
-      final Map columnCapabilities
+      final boolean deserializeComplexMetrics
   )
   {
-    return new ColumnSelectorFactory()
+    final ColumnSelectorFactory baseFactory = new ColumnSelectorFactory()
     {
       @Override
       public LongColumnSelector makeLongColumnSelector(final String columnName)
@@ -201,13 +207,17 @@ public Object get()
       }
 
       @Override
-      public ColumnCapabilities getColumnCapabilities(String columnName)
+      public ValueType getNativeType(String columnName)
       {
-        // This ColumnSelectorFactory implementation has no knowledge of column capabilities.
-        // However, this method may still be called by FilteredAggregatorFactory's ValueMatcherFactory
-        // to check column types.
-        // Just return null, the caller will assume default types in that case.
-        return null;
+        // This ColumnSelectorFactory implementation has no innate knowledge of column types. All it knows
+        // is that TIME_COLUMN_NAME is a long.
+
+        if (Column.TIME_COLUMN_NAME.equals(columnName)) {
+          return ValueType.LONG;
+        } else {
+          // Return null, caller will assume default types in this case.
+          return null;
+        }
       }
 
       @Override
@@ -293,45 +303,16 @@ public int lookupId(String name)
           }
         };
       }
-
-      @Override
-      public NumericColumnSelector makeMathExpressionSelector(String expression)
-      {
-        final Expr parsed = Parser.parse(expression);
-
-        final List required = Parser.findRequiredBindings(parsed);
-        final Map> values = Maps.newHashMapWithExpectedSize(required.size());
-
-        for (final String columnName : required) {
-          values.put(
-              columnName, new Supplier()
-              {
-                @Override
-                public Number get()
-                {
-                  return Evals.toNumber(in.get().getRaw(columnName));
-                }
-              }
-          );
-        }
-        final Expr.ObjectBinding binding = Parser.withSuppliers(values);
-
-        return new NumericColumnSelector()
-        {
-          @Override
-          public Number get()
-          {
-            return parsed.eval(binding).numericValue();
-          }
-        };
-      }
     };
+
+    return virtualColumns.wrap(baseFactory);
   }
 
   private final long minTimestamp;
   private final QueryGranularity gran;
   private final boolean rollup;
   private final List> rowTransformers;
+  private final VirtualColumns virtualColumns;
   private final AggregatorFactory[] metrics;
   private final AggregatorType[] aggs;
   private final boolean deserializeComplexMetrics;
@@ -375,6 +356,7 @@ public IncrementalIndex(
     this.minTimestamp = incrementalIndexSchema.getMinTimestamp();
     this.gran = incrementalIndexSchema.getGran();
     this.rollup = incrementalIndexSchema.isRollup();
+    this.virtualColumns = incrementalIndexSchema.getVirtualColumns();
     this.metrics = incrementalIndexSchema.getMetrics();
     this.rowTransformers = new CopyOnWriteArrayList<>();
     this.deserializeComplexMetrics = deserializeComplexMetrics;
@@ -1052,6 +1034,15 @@ public int hashCode()
     }
   }
 
+  protected ColumnSelectorFactory makeColumnSelectorFactory(
+      final AggregatorFactory agg,
+      final Supplier in,
+      final boolean deserializeComplexMetrics
+  )
+  {
+    return makeColumnSelectorFactory(virtualColumns, agg, in, deserializeComplexMetrics);
+  }
+
   protected final Comparator dimsComparator()
   {
     return new TimeAndDimsComp(dimensionDescsList);
diff --git a/processing/src/main/java/io/druid/segment/incremental/IncrementalIndexSchema.java b/processing/src/main/java/io/druid/segment/incremental/IncrementalIndexSchema.java
index b3bbfb187896..3750629abfc8 100644
--- a/processing/src/main/java/io/druid/segment/incremental/IncrementalIndexSchema.java
+++ b/processing/src/main/java/io/druid/segment/incremental/IncrementalIndexSchema.java
@@ -25,6 +25,7 @@
 import io.druid.granularity.QueryGranularities;
 import io.druid.granularity.QueryGranularity;
 import io.druid.query.aggregation.AggregatorFactory;
+import io.druid.segment.virtual.VirtualColumns;
 
 /**
  */
@@ -34,6 +35,7 @@ public class IncrementalIndexSchema
   private final long minTimestamp;
   private final TimestampSpec timestampSpec;
   private final QueryGranularity gran;
+  private final VirtualColumns virtualColumns;
   private final DimensionsSpec dimensionsSpec;
   private final AggregatorFactory[] metrics;
   private final boolean rollup;
@@ -42,6 +44,7 @@ public IncrementalIndexSchema(
       long minTimestamp,
       TimestampSpec timestampSpec,
       QueryGranularity gran,
+      VirtualColumns virtualColumns,
       DimensionsSpec dimensionsSpec,
       AggregatorFactory[] metrics,
       boolean rollup
@@ -50,6 +53,7 @@ public IncrementalIndexSchema(
     this.minTimestamp = minTimestamp;
     this.timestampSpec = timestampSpec;
     this.gran = gran;
+    this.virtualColumns = virtualColumns == null ? VirtualColumns.EMPTY : virtualColumns;
     this.dimensionsSpec = dimensionsSpec;
     this.metrics = metrics;
     this.rollup = rollup;
@@ -70,6 +74,11 @@ public QueryGranularity getGran()
     return gran;
   }
 
+  public VirtualColumns getVirtualColumns()
+  {
+    return virtualColumns;
+  }
+
   public DimensionsSpec getDimensionsSpec()
   {
     return dimensionsSpec;
@@ -90,6 +99,7 @@ public static class Builder
     private long minTimestamp;
     private TimestampSpec timestampSpec;
     private QueryGranularity gran;
+    private VirtualColumns virtualColumns;
     private DimensionsSpec dimensionsSpec;
     private AggregatorFactory[] metrics;
     private boolean rollup;
@@ -98,6 +108,7 @@ public Builder()
     {
       this.minTimestamp = 0L;
       this.gran = QueryGranularities.NONE;
+      this.virtualColumns = VirtualColumns.EMPTY;
       this.dimensionsSpec = new DimensionsSpec(null, null, null);
       this.metrics = new AggregatorFactory[]{};
       this.rollup = true;
@@ -133,6 +144,12 @@ public Builder withQueryGranularity(QueryGranularity gran)
       return this;
     }
 
+    public Builder withVirtualColumns(VirtualColumns virtualColumns)
+    {
+      this.virtualColumns = virtualColumns;
+      return this;
+    }
+
     public Builder withDimensionsSpec(DimensionsSpec dimensionsSpec)
     {
       this.dimensionsSpec = dimensionsSpec == null ? DimensionsSpec.ofEmpty() : dimensionsSpec;
@@ -167,7 +184,7 @@ public Builder withRollup(boolean rollup)
     public IncrementalIndexSchema build()
     {
       return new IncrementalIndexSchema(
-          minTimestamp, timestampSpec, gran, dimensionsSpec, metrics, rollup
+          minTimestamp, timestampSpec, gran, virtualColumns, dimensionsSpec, metrics, rollup
       );
     }
   }
diff --git a/processing/src/main/java/io/druid/segment/incremental/IncrementalIndexStorageAdapter.java b/processing/src/main/java/io/druid/segment/incremental/IncrementalIndexStorageAdapter.java
index 443abed638ae..c3f21d7bdcc8 100644
--- a/processing/src/main/java/io/druid/segment/incremental/IncrementalIndexStorageAdapter.java
+++ b/processing/src/main/java/io/druid/segment/incremental/IncrementalIndexStorageAdapter.java
@@ -21,15 +21,10 @@
 
 import com.google.common.base.Function;
 import com.google.common.base.Strings;
-import com.google.common.base.Supplier;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.Iterators;
 import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
-
 import io.druid.granularity.QueryGranularity;
-import io.druid.math.expr.Expr;
-import io.druid.math.expr.Parser;
 import io.druid.java.util.common.guava.Sequence;
 import io.druid.java.util.common.guava.Sequences;
 import io.druid.query.QueryInterruptedException;
@@ -50,34 +45,30 @@
 import io.druid.segment.LongColumnSelector;
 import io.druid.segment.Metadata;
 import io.druid.segment.NullDimensionSelector;
-import io.druid.segment.NumericColumnSelector;
 import io.druid.segment.ObjectColumnSelector;
 import io.druid.segment.SingleScanTimeDimSelector;
 import io.druid.segment.StorageAdapter;
-import io.druid.segment.VirtualColumn;
-import io.druid.segment.VirtualColumns;
+import io.druid.segment.ZeroFloatColumnSelector;
+import io.druid.segment.ZeroLongColumnSelector;
 import io.druid.segment.column.Column;
 import io.druid.segment.column.ColumnCapabilities;
-import io.druid.segment.column.ColumnCapabilitiesImpl;
 import io.druid.segment.column.ValueType;
 import io.druid.segment.data.Indexed;
 import io.druid.segment.data.ListIndexed;
 import io.druid.segment.filter.BooleanValueMatcher;
 import io.druid.segment.filter.Filters;
+import io.druid.segment.virtual.VirtualColumns;
 import org.joda.time.DateTime;
 import org.joda.time.Interval;
 
 import javax.annotation.Nullable;
 import java.util.Iterator;
-import java.util.List;
 import java.util.Map;
 
 /**
  */
 public class IncrementalIndexStorageAdapter implements StorageAdapter
 {
-  private static final NullDimensionSelector NULL_DIMENSION_SELECTOR = new NullDimensionSelector();
-
   private final IncrementalIndex index;
 
   public IncrementalIndexStorageAdapter(
@@ -352,6 +343,10 @@ public DimensionSelector makeDimensionSelector(
                   DimensionSpec dimensionSpec
               )
               {
+                if (virtualColumns.exists(dimensionSpec.getDimension())) {
+                  return virtualColumns.makeDimensionSelector(dimensionSpec, this);
+                }
+
                 final String dimension = dimensionSpec.getDimension();
                 final ExtractionFn extractionFn = dimensionSpec.getExtractionFn();
 
@@ -366,7 +361,7 @@ public DimensionSelector makeDimensionSelector(
 
                 final IncrementalIndex.DimensionDesc dimensionDesc = index.getDimension(dimensionSpec.getDimension());
                 if (dimensionDesc == null) {
-                  return dimensionSpec.decorate(NULL_DIMENSION_SELECTOR);
+                  return dimensionSpec.decorate(NullDimensionSelector.instance());
                 }
 
                 final DimensionIndexer indexer = dimensionDesc.getIndexer();
@@ -376,6 +371,10 @@ public DimensionSelector makeDimensionSelector(
               @Override
               public FloatColumnSelector makeFloatColumnSelector(String columnName)
               {
+                if (virtualColumns.exists(columnName)) {
+                  return virtualColumns.makeFloatColumnSelector(columnName, this);
+                }
+
                 final Integer dimIndex = index.getDimensionIndex(columnName);
                 if (dimIndex != null) {
                   final IncrementalIndex.DimensionDesc dimensionDesc = index.getDimension(columnName);
@@ -389,14 +388,7 @@ public FloatColumnSelector makeFloatColumnSelector(String columnName)
 
                 final Integer metricIndexInt = index.getMetricIndex(columnName);
                 if (metricIndexInt == null) {
-                  return new FloatColumnSelector()
-                  {
-                    @Override
-                    public float get()
-                    {
-                      return 0.0f;
-                    }
-                  };
+                  return ZeroFloatColumnSelector.instance();
                 }
 
                 final int metricIndex = metricIndexInt;
@@ -413,6 +405,10 @@ public float get()
               @Override
               public LongColumnSelector makeLongColumnSelector(String columnName)
               {
+                if (virtualColumns.exists(columnName)) {
+                  return virtualColumns.makeLongColumnSelector(columnName, this);
+                }
+
                 if (columnName.equals(Column.TIME_COLUMN_NAME)) {
                   return new LongColumnSelector()
                   {
@@ -437,14 +433,7 @@ public long get()
 
                 final Integer metricIndexInt = index.getMetricIndex(columnName);
                 if (metricIndexInt == null) {
-                  return new LongColumnSelector()
-                  {
-                    @Override
-                    public long get()
-                    {
-                      return 0L;
-                    }
-                  };
+                  return ZeroLongColumnSelector.instance();
                 }
 
                 final int metricIndex = metricIndexInt;
@@ -465,6 +454,10 @@ public long get()
               @Override
               public ObjectColumnSelector makeObjectColumnSelector(String column)
               {
+                if (virtualColumns.exists(column)) {
+                  return virtualColumns.makeObjectColumnSelector(column, this);
+                }
+
                 if (column.equals(Column.TIME_COLUMN_NAME)) {
                   return new ObjectColumnSelector()
                   {
@@ -508,10 +501,6 @@ public Object get()
                 IncrementalIndex.DimensionDesc dimensionDesc = index.getDimension(column);
 
                 if (dimensionDesc == null) {
-                  VirtualColumn virtualColumn = virtualColumns.getVirtualColumn(column);
-                  if (virtualColumn != null) {
-                    return virtualColumn.init(column, this);
-                  }
                   return null;
                 } else {
 
@@ -548,66 +537,10 @@ public Object get()
               }
 
               @Override
-              public ColumnCapabilities getColumnCapabilities(String columnName)
-              {
-                ColumnCapabilities capabilities = index.getCapabilities(columnName);
-                if (capabilities == null && !virtualColumns.isEmpty()) {
-                  VirtualColumn virtualColumn = virtualColumns.getVirtualColumn(columnName);
-                  if (virtualColumn != null) {
-                    Class clazz = virtualColumn.init(columnName, this).classOfObject();
-                    capabilities = new ColumnCapabilitiesImpl().setType(ValueType.typeFor(clazz));
-                  }
-                }
-                return capabilities;
-              }
-
-              @Override
-              public NumericColumnSelector makeMathExpressionSelector(String expression)
+              public ValueType getNativeType(String columnName)
               {
-                final Expr parsed = Parser.parse(expression);
-
-                final List required = Parser.findRequiredBindings(parsed);
-                final Map> values = Maps.newHashMapWithExpectedSize(required.size());
-
-                for (String columnName : index.getMetricNames()) {
-                  if (!required.contains(columnName)) {
-                    continue;
-                  }
-                  ValueType type = index.getCapabilities(columnName).getType();
-                  if (type == ValueType.FLOAT) {
-                    final int metricIndex = index.getMetricIndex(columnName);
-                    values.put(
-                        columnName, new Supplier()
-                        {
-                          @Override
-                          public Number get()
-                          {
-                            return index.getMetricFloatValue(currEntry.getValue(), metricIndex);
-                          }
-                        }
-                    );
-                  } else if (type == ValueType.LONG) {
-                    final int metricIndex = index.getMetricIndex(columnName);
-                    values.put(
-                        columnName, new Supplier()
-                        {
-                          @Override
-                          public Number get()
-                          {
-                            return index.getMetricLongValue(currEntry.getValue(), metricIndex);
-                          }
-                        }
-                    );
-                  }
-                }
-                final Expr.ObjectBinding binding = Parser.withSuppliers(values);
-                return new NumericColumnSelector() {
-                  @Override
-                  public Number get()
-                  {
-                    return parsed.eval(binding).numericValue();
-                  }
-                };
+                final ColumnCapabilities capabilities = index.getCapabilities(columnName);
+                return capabilities != null ? capabilities.getType() : virtualColumns.getNativeType(columnName);
               }
             };
           }
diff --git a/processing/src/main/java/io/druid/segment/incremental/OffheapIncrementalIndex.java b/processing/src/main/java/io/druid/segment/incremental/OffheapIncrementalIndex.java
index a0f654b40519..4655d7c27623 100644
--- a/processing/src/main/java/io/druid/segment/incremental/OffheapIncrementalIndex.java
+++ b/processing/src/main/java/io/druid/segment/incremental/OffheapIncrementalIndex.java
@@ -91,31 +91,6 @@ public OffheapIncrementalIndex(
     aggBuffers.add(bb);
   }
 
-  public OffheapIncrementalIndex(
-      long minTimestamp,
-      QueryGranularity gran,
-      final AggregatorFactory[] metrics,
-      boolean deserializeComplexMetrics,
-      boolean reportParseExceptions,
-      boolean sortFacts,
-      int maxRowCount,
-      StupidPool bufferPool
-  )
-  {
-    this(
-        new IncrementalIndexSchema.Builder().withMinTimestamp(minTimestamp)
-                                            .withQueryGranularity(gran)
-                                            .withMetrics(metrics)
-                                            .withRollup(IncrementalIndexSchema.DEFAULT_ROLLUP)
-                                            .build(),
-        deserializeComplexMetrics,
-        reportParseExceptions,
-        sortFacts,
-        maxRowCount,
-        bufferPool
-    );
-  }
-
   public OffheapIncrementalIndex(
       long minTimestamp,
       QueryGranularity gran,
@@ -177,8 +152,7 @@ protected BufferAggregator[] initAggs(
       ColumnSelectorFactory columnSelectorFactory = makeColumnSelectorFactory(
           agg,
           rowSupplier,
-          deserializeComplexMetrics,
-          getColumnCapabilities()
+          deserializeComplexMetrics
       );
 
       selectors.put(
@@ -229,7 +203,7 @@ protected Integer addToFacts(
           for (int i = 0; i < metrics.length; i++) {
             final AggregatorFactory agg = metrics[i];
             getAggs()[i] = agg.factorizeBuffered(
-                makeColumnSelectorFactory(agg, rowSupplier, deserializeComplexMetrics, getColumnCapabilities())
+                makeColumnSelectorFactory(agg, rowSupplier, deserializeComplexMetrics)
             );
           }
           rowContainer.set(null);
diff --git a/processing/src/main/java/io/druid/segment/incremental/OnheapIncrementalIndex.java b/processing/src/main/java/io/druid/segment/incremental/OnheapIncrementalIndex.java
index 4604b9ce8e04..f7c29a80b1ad 100644
--- a/processing/src/main/java/io/druid/segment/incremental/OnheapIncrementalIndex.java
+++ b/processing/src/main/java/io/druid/segment/incremental/OnheapIncrementalIndex.java
@@ -33,9 +33,8 @@
 import io.druid.segment.DimensionSelector;
 import io.druid.segment.FloatColumnSelector;
 import io.druid.segment.LongColumnSelector;
-import io.druid.segment.NumericColumnSelector;
 import io.druid.segment.ObjectColumnSelector;
-import io.druid.segment.column.ColumnCapabilities;
+import io.druid.segment.column.ValueType;
 
 import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
@@ -158,7 +157,7 @@ protected Aggregator[] initAggs(
     for (AggregatorFactory agg : metrics) {
       selectors.put(
           agg.getName(),
-          new ObjectCachingColumnSelectorFactory(makeColumnSelectorFactory(agg, rowSupplier, deserializeComplexMetrics, getColumnCapabilities()))
+          new ObjectCachingColumnSelectorFactory(makeColumnSelectorFactory(agg, rowSupplier, deserializeComplexMetrics))
       );
     }
 
@@ -404,16 +403,9 @@ public ObjectColumnSelector makeObjectColumnSelector(String columnName)
     }
 
     @Override
-    public ColumnCapabilities getColumnCapabilities(String columnName)
+    public ValueType getNativeType(String columnName)
     {
-      return delegate.getColumnCapabilities(columnName);
-    }
-
-    @Override
-    public NumericColumnSelector makeMathExpressionSelector(String expression)
-    {
-      return delegate.makeMathExpressionSelector(expression);
+      return delegate.getNativeType(columnName);
     }
   }
-
 }
diff --git a/processing/src/main/java/io/druid/segment/virtual/ExpressionObjectColumnSelector.java b/processing/src/main/java/io/druid/segment/virtual/ExpressionObjectColumnSelector.java
new file mode 100644
index 000000000000..9bd7b7e0f9ee
--- /dev/null
+++ b/processing/src/main/java/io/druid/segment/virtual/ExpressionObjectColumnSelector.java
@@ -0,0 +1,141 @@
+/*
+ * Licensed to Metamarkets Group Inc. (Metamarkets) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. Metamarkets licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package io.druid.segment.virtual;
+
+import com.google.common.base.Preconditions;
+import com.google.common.base.Supplier;
+import com.google.common.collect.Maps;
+import com.google.common.primitives.Doubles;
+import io.druid.common.guava.GuavaUtils;
+import io.druid.math.expr.Expr;
+import io.druid.math.expr.Parser;
+import io.druid.segment.ColumnSelectorFactory;
+import io.druid.segment.FloatColumnSelector;
+import io.druid.segment.LongColumnSelector;
+import io.druid.segment.ObjectColumnSelector;
+import io.druid.segment.column.ValueType;
+
+import java.util.Map;
+
+public class ExpressionObjectColumnSelector implements ObjectColumnSelector
+{
+  private final Expr expression;
+  private final Expr.ObjectBinding bindings;
+
+  ExpressionObjectColumnSelector(Expr expression, ColumnSelectorFactory columnSelectorFactory)
+  {
+    this.expression = Preconditions.checkNotNull(expression, "expression");
+    this.bindings = createBindings(expression, columnSelectorFactory);
+  }
+
+  private static Expr.ObjectBinding createBindings(Expr expression, ColumnSelectorFactory columnSelectorFactory)
+  {
+    final Map> suppliers = Maps.newHashMap();
+    for (String columnName : Parser.findRequiredBindings(expression)) {
+      final ValueType nativeType = columnSelectorFactory.getNativeType(columnName);
+      final Supplier supplier;
+
+      if (nativeType == ValueType.FLOAT) {
+        final FloatColumnSelector selector = columnSelectorFactory.makeFloatColumnSelector(columnName);
+        supplier = new Supplier()
+        {
+          @Override
+          public Number get()
+          {
+            return selector.get();
+          }
+        };
+      } else if (nativeType == ValueType.LONG) {
+        final LongColumnSelector selector = columnSelectorFactory.makeLongColumnSelector(columnName);
+        supplier = new Supplier()
+        {
+          @Override
+          public Number get()
+          {
+            return selector.get();
+          }
+        };
+      } else if (nativeType == null) {
+        // Unknown ValueType. Try making an Object selector and see if that gives us anything useful.
+        final ObjectColumnSelector selector = columnSelectorFactory.makeObjectColumnSelector(columnName);
+        final Class clazz = selector == null ? null : selector.classOfObject();
+        if (selector == null || (clazz != Object.class && Number.class.isAssignableFrom(clazz))) {
+          // We know there are no numbers here. Use a null supplier.
+          supplier = null;
+        } else {
+          // There may be numbers here.
+          supplier = new Supplier()
+          {
+            @Override
+            public Number get()
+            {
+              return tryParse(selector.get());
+            }
+          };
+        }
+      } else {
+        // Unhandleable ValueType (possibly STRING or COMPLEX).
+        supplier = null;
+      }
+
+      if (supplier != null) {
+        suppliers.put(columnName, supplier);
+      }
+    }
+
+    return Parser.withSuppliers(suppliers);
+  }
+
+  private static Number tryParse(final Object value)
+  {
+    if (value == null) {
+      return 0L;
+    }
+
+    if (value instanceof Number) {
+      return (Number) value;
+    }
+
+    final String stringValue = String.valueOf(value);
+    final Long longValue = GuavaUtils.tryParseLong(stringValue);
+    if (longValue != null) {
+      return longValue;
+    }
+
+    final Double doubleValue = Doubles.tryParse(stringValue);
+    if (doubleValue != null) {
+      return doubleValue;
+    }
+
+    return 0L;
+  }
+
+  @Override
+  public Class classOfObject()
+  {
+    return Number.class;
+  }
+
+  @Override
+  public Number get()
+  {
+    return expression.eval(bindings).numericValue();
+  }
+}
diff --git a/processing/src/main/java/io/druid/segment/virtual/ExpressionVirtualColumn.java b/processing/src/main/java/io/druid/segment/virtual/ExpressionVirtualColumn.java
new file mode 100644
index 000000000000..f0f1f53832af
--- /dev/null
+++ b/processing/src/main/java/io/druid/segment/virtual/ExpressionVirtualColumn.java
@@ -0,0 +1,199 @@
+/*
+ * Licensed to Metamarkets Group Inc. (Metamarkets) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. Metamarkets licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package io.druid.segment.virtual;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.google.common.base.Preconditions;
+import com.google.common.primitives.Ints;
+import io.druid.math.expr.Expr;
+import io.druid.math.expr.Parser;
+import io.druid.query.dimension.DimensionSpec;
+import io.druid.segment.ColumnSelectorFactory;
+import io.druid.segment.DimensionSelector;
+import io.druid.segment.FloatColumnSelector;
+import io.druid.segment.LongColumnSelector;
+import io.druid.segment.NullDimensionSelector;
+import io.druid.segment.ObjectColumnSelector;
+import io.druid.segment.column.ValueType;
+import org.apache.commons.codec.Charsets;
+
+import java.nio.ByteBuffer;
+import java.util.List;
+
+public class ExpressionVirtualColumn implements VirtualColumn
+{
+  private final String name;
+  private final String expression;
+  private final Expr parsedExpression;
+
+  @JsonCreator
+  public ExpressionVirtualColumn(
+      @JsonProperty("name") String name,
+      @JsonProperty("expression") String expression
+  )
+  {
+    this.name = Preconditions.checkNotNull(name, "name");
+    this.expression = Preconditions.checkNotNull(expression, "expression");
+    this.parsedExpression = Parser.parse(expression);
+  }
+
+  @JsonProperty("name")
+  @Override
+  public String getOutputName()
+  {
+    return name;
+  }
+
+  @JsonProperty
+  public String getExpression()
+  {
+    return expression;
+  }
+
+  @Override
+  public ObjectColumnSelector makeObjectColumnSelector(
+      final String columnName,
+      final ColumnSelectorFactory columnSelectorFactory
+  )
+  {
+    return new ExpressionObjectColumnSelector(parsedExpression, columnSelectorFactory);
+  }
+
+  @Override
+  public DimensionSelector makeDimensionSelector(
+      final DimensionSpec dimensionSpec,
+      final ColumnSelectorFactory columnSelectorFactory
+  )
+  {
+    return NullDimensionSelector.instance();
+  }
+
+  @Override
+  public FloatColumnSelector makeFloatColumnSelector(
+      final String columnName,
+      final ColumnSelectorFactory columnSelectorFactory
+  )
+  {
+    final ExpressionObjectColumnSelector numericSelector = new ExpressionObjectColumnSelector(
+        parsedExpression,
+        columnSelectorFactory
+    );
+    return new FloatColumnSelector()
+    {
+      @Override
+      public float get()
+      {
+        final Number number = numericSelector.get();
+        return number == null ? 0.0f : number.floatValue();
+      }
+    };
+  }
+
+  @Override
+  public LongColumnSelector makeLongColumnSelector(
+      final String columnName,
+      final ColumnSelectorFactory columnSelectorFactory
+  )
+  {
+    final ExpressionObjectColumnSelector numericSelector = new ExpressionObjectColumnSelector(
+        parsedExpression,
+        columnSelectorFactory
+    );
+    return new LongColumnSelector()
+    {
+      @Override
+      public long get()
+      {
+        final Number number = numericSelector.get();
+        return number == null ? 0L : number.longValue();
+      }
+    };
+  }
+
+  @Override
+  public ValueType nativeType(String columnName)
+  {
+    return ValueType.FLOAT;
+  }
+
+  @Override
+  public List requiredColumns()
+  {
+    return Parser.findRequiredBindings(expression);
+  }
+
+  @Override
+  public boolean usesDotNotation()
+  {
+    return false;
+  }
+
+  @Override
+  public byte[] getCacheKey()
+  {
+    final byte[] nameBytes = name.getBytes(Charsets.UTF_8);
+    final byte[] expressionBytes = expression.getBytes(Charsets.UTF_8);
+
+    return ByteBuffer
+        .allocate(1 + Ints.BYTES * 2 + nameBytes.length + expressionBytes.length)
+        .put(VirtualColumnCacheHelper.CACHE_TYPE_ID_EXPRESSION)
+        .putInt(nameBytes.length)
+        .put(nameBytes)
+        .putInt(expressionBytes.length)
+        .put(expressionBytes)
+        .array();
+  }
+
+  @Override
+  public boolean equals(Object o)
+  {
+    if (this == o) {
+      return true;
+    }
+    if (o == null || getClass() != o.getClass()) {
+      return false;
+    }
+
+    ExpressionVirtualColumn that = (ExpressionVirtualColumn) o;
+
+    if (!name.equals(that.name)) {
+      return false;
+    }
+    return expression.equals(that.expression);
+  }
+
+  @Override
+  public int hashCode()
+  {
+    int result = name.hashCode();
+    result = 31 * result + expression.hashCode();
+    return result;
+  }
+
+  @Override
+  public String toString()
+  {
+    return "ExpressionVirtualColumn{" +
+           "name='" + name + '\'' +
+           ", expression='" + expression + '\'' +
+           '}';
+  }
+}
diff --git a/processing/src/main/java/io/druid/segment/virtual/VirtualColumn.java b/processing/src/main/java/io/druid/segment/virtual/VirtualColumn.java
new file mode 100644
index 000000000000..56c7d21bba91
--- /dev/null
+++ b/processing/src/main/java/io/druid/segment/virtual/VirtualColumn.java
@@ -0,0 +1,140 @@
+/*
+ * Licensed to Metamarkets Group Inc. (Metamarkets) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. Metamarkets licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package io.druid.segment.virtual;
+
+import com.fasterxml.jackson.annotation.JsonSubTypes;
+import com.fasterxml.jackson.annotation.JsonTypeInfo;
+import io.druid.query.dimension.DimensionSpec;
+import io.druid.segment.ColumnSelectorFactory;
+import io.druid.segment.DimensionSelector;
+import io.druid.segment.FloatColumnSelector;
+import io.druid.segment.LongColumnSelector;
+import io.druid.segment.ObjectColumnSelector;
+import io.druid.segment.column.ValueType;
+
+import java.util.List;
+
+/**
+ * Virtual columns are "views" created over a ColumnSelectorFactory. They can potentially draw from multiple
+ * underlying columns, although they always present themselves as if they were a single column.
+ *
+ * A virtual column object will be shared amongst threads and must be thread safe. The selectors returned
+ * from the various makeXXXSelector methods need not be thread safe.
+ */
+@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type")
+@JsonSubTypes(value = {
+    @JsonSubTypes.Type(name = "expression", value = ExpressionVirtualColumn.class)
+})
+public interface VirtualColumn
+{
+  /**
+   * Output name of this column.
+   *
+   * @return name
+   */
+  String getOutputName();
+
+  /**
+   * Build a selector corresponding to this virtual column. Also provides the name that the
+   * virtual column was referenced with, which is useful if this column uses dot notation.
+   *
+   * @param columnName the name this virtual column was referenced with
+   * @param factory    column selector factory
+   *
+   * @return the selector, must not be null
+   */
+  ObjectColumnSelector makeObjectColumnSelector(String columnName, ColumnSelectorFactory factory);
+
+  /**
+   * Build a selector corresponding to this virtual column. Also provides the name that the
+   * virtual column was referenced with (through {@link DimensionSpec#getDimension()}, which
+   * is useful if this column uses dot notation. The virtual column is expected to apply any
+   * necessary decoration from the dimensionSpec.
+   *
+   * @param dimensionSpec the dimensionSpec this column was referenced with
+   * @param factory       column selector factory
+   *
+   * @return the selector, or null if we can't make a selector
+   */
+  DimensionSelector makeDimensionSelector(DimensionSpec dimensionSpec, ColumnSelectorFactory factory);
+
+  /**
+   * Build a selector corresponding to this virtual column. Also provides the name that the
+   * virtual column was referenced with, which is useful if this column uses dot notation.
+   *
+   * @param columnName the name this virtual column was referenced with
+   * @param factory    column selector factory
+   *
+   * @return the selector, or null if we can't make a selector
+   */
+  FloatColumnSelector makeFloatColumnSelector(String columnName, ColumnSelectorFactory factory);
+
+  /**
+   * Build a selector corresponding to this virtual column. Also provides the name that the
+   * virtual column was referenced with, which is useful if this column uses dot notation.
+   *
+   * @param columnName the name this virtual column was referenced with
+   * @param factory    column selector factory
+   *
+   * @return the selector, or null if we can't make a selector
+   */
+  LongColumnSelector makeLongColumnSelector(String columnName, ColumnSelectorFactory factory);
+
+  /**
+   * Returns the native type of this virtual column, which should match the type returned
+   * by "makeObjectColumnSelector" and should correspond to the best performing selector.
+   * May vary based on columnName if this column uses dot notation.
+   *
+   * @param columnName the name this virtual column was referenced with
+   *
+   * @return native type, must not be null
+   */
+  ValueType nativeType(String columnName);
+
+  /**
+   * Returns a list of columns that this virtual column will access. This may include the
+   * names of other virtual columns. May be empty if a virtual column doesn't access any
+   * underlying columns.
+   *
+   * Does not pass columnName because there is an assumption that the list of columns
+   * needed by a dot-notation supporting virtual column will not vary based on the
+   * columnName.
+   *
+   * @return column names
+   */
+  List requiredColumns();
+
+  /**
+   * Indicates that this virtual column can be referenced with dot notation. For example,
+   * a virtual column named "foo" could be referred to as "foo.bar" with the Cursor it is
+   * registered with. In that case, init will be called with columnName "foo.bar" rather
+   * than "foo".
+   *
+   * @return whether to use dot notation
+   */
+  boolean usesDotNotation();
+
+  /**
+   * Returns cache key
+   *
+   * @return cache key
+   */
+  byte[] getCacheKey();
+}
diff --git a/processing/src/main/java/io/druid/segment/virtual/VirtualColumnCacheHelper.java b/processing/src/main/java/io/druid/segment/virtual/VirtualColumnCacheHelper.java
new file mode 100644
index 000000000000..7c7bba2edd9d
--- /dev/null
+++ b/processing/src/main/java/io/druid/segment/virtual/VirtualColumnCacheHelper.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to Metamarkets Group Inc. (Metamarkets) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. Metamarkets licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package io.druid.segment.virtual;
+
+public class VirtualColumnCacheHelper
+{
+  public static final byte CACHE_TYPE_ID_MAP = 0x00;
+  public static final byte CACHE_TYPE_ID_EXPRESSION = 0x01;
+
+  // Starting byte 0xFF is reserved for site-specific virtual columns.
+  public static final byte CACHE_TYPE_ID_USER_DEFINED = (byte) 0xFF;
+
+  private VirtualColumnCacheHelper()
+  {
+    // No instantiation.
+  }
+}
diff --git a/processing/src/main/java/io/druid/segment/virtual/VirtualColumns.java b/processing/src/main/java/io/druid/segment/virtual/VirtualColumns.java
new file mode 100644
index 000000000000..dc1a9e81d359
--- /dev/null
+++ b/processing/src/main/java/io/druid/segment/virtual/VirtualColumns.java
@@ -0,0 +1,275 @@
+/*
+ * Licensed to Metamarkets Group Inc. (Metamarkets) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. Metamarkets licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package io.druid.segment.virtual;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonValue;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
+import com.google.common.primitives.Ints;
+import io.druid.java.util.common.IAE;
+import io.druid.java.util.common.Pair;
+import io.druid.query.dimension.DimensionSpec;
+import io.druid.segment.ColumnSelectorFactory;
+import io.druid.segment.DimensionSelector;
+import io.druid.segment.FloatColumnSelector;
+import io.druid.segment.LongColumnSelector;
+import io.druid.segment.NullDimensionSelector;
+import io.druid.segment.ObjectColumnSelector;
+import io.druid.segment.ZeroFloatColumnSelector;
+import io.druid.segment.ZeroLongColumnSelector;
+import io.druid.segment.column.Column;
+import io.druid.segment.column.ValueType;
+
+import java.nio.ByteBuffer;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * Class allowing lookup and usage of virtual columns.
+ */
+public class VirtualColumns
+{
+  public static final VirtualColumns EMPTY = new VirtualColumns(
+      ImmutableList.of(),
+      ImmutableMap.of(),
+      ImmutableMap.of()
+  );
+
+  /**
+   * Split a dot-style columnName into the "main" columnName and the subColumn name after the dot. Useful for
+   * columns that support dot notation.
+   *
+   * @param columnName columnName like "foo" or "foo.bar"
+   *
+   * @return pair of main column name (will not be null) and subColumn name (may be null)
+   */
+  public static Pair splitColumnName(String columnName)
+  {
+    final int i = columnName.indexOf('.');
+    if (i < 0) {
+      return Pair.of(columnName, null);
+    } else {
+      return Pair.of(columnName.substring(0, i), columnName.substring(i + 1));
+    }
+  }
+
+  @JsonCreator
+  public static VirtualColumns create(List virtualColumns)
+  {
+    if (virtualColumns == null || virtualColumns.isEmpty()) {
+      return EMPTY;
+    }
+    Map withDotSupport = Maps.newHashMap();
+    Map withoutDotSupport = Maps.newHashMap();
+    for (VirtualColumn vc : virtualColumns) {
+      if (vc.getOutputName().equals(Column.TIME_COLUMN_NAME)) {
+        throw new IAE("virtualColumn name[%s] not allowed", vc.getOutputName());
+      }
+
+      if (withDotSupport.containsKey(vc.getOutputName()) || withoutDotSupport.containsKey(vc.getOutputName())) {
+        throw new IAE("Duplicate virtualColumn name[%s]", vc.getOutputName());
+      }
+
+      if (vc.usesDotNotation()) {
+        withDotSupport.put(vc.getOutputName(), vc);
+      } else {
+        withoutDotSupport.put(vc.getOutputName(), vc);
+      }
+    }
+    return new VirtualColumns(ImmutableList.copyOf(virtualColumns), withDotSupport, withoutDotSupport);
+  }
+
+  private VirtualColumns(
+      List virtualColumns,
+      Map withDotSupport,
+      Map withoutDotSupport
+  )
+  {
+    this.virtualColumns = virtualColumns;
+    this.withDotSupport = withDotSupport;
+    this.withoutDotSupport = withoutDotSupport;
+
+    for (VirtualColumn virtualColumn : virtualColumns) {
+      detectCycles(virtualColumn, null);
+    }
+  }
+
+  // For equals, hashCode, toString, and serialization:
+  private final List virtualColumns;
+
+  // For getVirtualColumn:
+  private final Map withDotSupport;
+  private final Map withoutDotSupport;
+
+  public boolean exists(String columnName)
+  {
+    return getVirtualColumn(columnName) != null;
+  }
+
+  public VirtualColumn getVirtualColumn(String columnName)
+  {
+    final VirtualColumn vc = withoutDotSupport.get(columnName);
+    if (vc != null) {
+      return vc;
+    }
+    final String baseColumnName = splitColumnName(columnName).lhs;
+    return withDotSupport.get(baseColumnName);
+  }
+
+  public ObjectColumnSelector makeObjectColumnSelector(String columnName, ColumnSelectorFactory factory)
+  {
+    final VirtualColumn virtualColumn = getVirtualColumn(columnName);
+    if (virtualColumn == null) {
+      return null;
+    } else {
+      return Preconditions.checkNotNull(
+          virtualColumn.makeObjectColumnSelector(columnName, factory),
+          "VirtualColumn[%s] returned a null ObjectColumnSelector for columnName[%s]",
+          virtualColumn.getOutputName(),
+          columnName
+      );
+    }
+  }
+
+  public DimensionSelector makeDimensionSelector(DimensionSpec dimensionSpec, ColumnSelectorFactory factory)
+  {
+    final VirtualColumn virtualColumn = getVirtualColumn(dimensionSpec.getDimension());
+    if (virtualColumn == null) {
+      return dimensionSpec.decorate(NullDimensionSelector.instance());
+    } else {
+      final DimensionSelector selector = virtualColumn.makeDimensionSelector(dimensionSpec, factory);
+      return selector == null ? dimensionSpec.decorate(NullDimensionSelector.instance()) : selector;
+    }
+  }
+
+  public FloatColumnSelector makeFloatColumnSelector(String columnName, ColumnSelectorFactory factory)
+  {
+    final VirtualColumn virtualColumn = getVirtualColumn(columnName);
+    if (virtualColumn == null) {
+      return ZeroFloatColumnSelector.instance();
+    } else {
+      final FloatColumnSelector selector = virtualColumn.makeFloatColumnSelector(columnName, factory);
+      return selector == null ? ZeroFloatColumnSelector.instance() : selector;
+    }
+  }
+
+  public LongColumnSelector makeLongColumnSelector(String columnName, ColumnSelectorFactory factory)
+  {
+    final VirtualColumn virtualColumn = getVirtualColumn(columnName);
+    if (virtualColumn == null) {
+      return ZeroLongColumnSelector.instance();
+    } else {
+      final LongColumnSelector selector = virtualColumn.makeLongColumnSelector(columnName, factory);
+      return selector == null ? ZeroLongColumnSelector.instance() : selector;
+    }
+  }
+
+  public ValueType getNativeType(String columnName)
+  {
+    final VirtualColumn virtualColumn = getVirtualColumn(columnName);
+    return virtualColumn != null
+           ? Preconditions.checkNotNull(virtualColumn.nativeType(columnName), "nativeType for column[%s]", columnName)
+           : null;
+  }
+
+  public boolean isEmpty()
+  {
+    return withDotSupport.isEmpty() && withoutDotSupport.isEmpty();
+  }
+
+  @JsonValue
+  public VirtualColumn[] getVirtualColumns()
+  {
+    // VirtualColumn[] instead of List to aid Jackson serialization.
+    return virtualColumns.toArray(new VirtualColumn[]{});
+  }
+
+  public ColumnSelectorFactory wrap(final ColumnSelectorFactory baseFactory)
+  {
+    return new VirtualizedColumnSelectorFactory(baseFactory, this);
+  }
+
+  public byte[] getCacheKey()
+  {
+    final byte[][] cacheKeys = new byte[virtualColumns.size()][];
+    int len = Ints.BYTES;
+    for (int i = 0; i < virtualColumns.size(); i++) {
+      cacheKeys[i] = virtualColumns.get(i).getCacheKey();
+      len += Ints.BYTES + cacheKeys[i].length;
+    }
+    final ByteBuffer buf = ByteBuffer.allocate(len).putInt(virtualColumns.size());
+    for (byte[] cacheKey : cacheKeys) {
+      buf.putInt(cacheKey.length);
+      buf.put(cacheKey);
+    }
+    return buf.array();
+  }
+
+  private void detectCycles(VirtualColumn virtualColumn, ImmutableSet columnNames)
+  {
+    Set nextSet = columnNames == null
+                          ? Sets.newHashSet(virtualColumn.getOutputName())
+                          : Sets.newHashSet(columnNames);
+
+    for (String columnName : virtualColumn.requiredColumns()) {
+      if (!nextSet.add(columnName)) {
+        throw new IAE("Self-referential column[%s]", columnName);
+      }
+
+      final VirtualColumn dependency = getVirtualColumn(columnName);
+      if (dependency != null) {
+        detectCycles(dependency, ImmutableSet.copyOf(nextSet));
+      }
+    }
+  }
+
+  @Override
+  public boolean equals(Object o)
+  {
+    if (this == o) {
+      return true;
+    }
+    if (o == null || getClass() != o.getClass()) {
+      return false;
+    }
+
+    VirtualColumns that = (VirtualColumns) o;
+
+    return virtualColumns.equals(that.virtualColumns);
+  }
+
+  @Override
+  public int hashCode()
+  {
+    return virtualColumns.hashCode();
+  }
+
+  @Override
+  public String toString()
+  {
+    return virtualColumns.toString();
+  }
+}
diff --git a/processing/src/main/java/io/druid/segment/virtual/VirtualizedColumnSelectorFactory.java b/processing/src/main/java/io/druid/segment/virtual/VirtualizedColumnSelectorFactory.java
new file mode 100644
index 000000000000..3a4f4373e9de
--- /dev/null
+++ b/processing/src/main/java/io/druid/segment/virtual/VirtualizedColumnSelectorFactory.java
@@ -0,0 +1,91 @@
+/*
+ * Licensed to Metamarkets Group Inc. (Metamarkets) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. Metamarkets licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package io.druid.segment.virtual;
+
+import com.google.common.base.Preconditions;
+import io.druid.query.dimension.DimensionSpec;
+import io.druid.segment.ColumnSelectorFactory;
+import io.druid.segment.DimensionSelector;
+import io.druid.segment.FloatColumnSelector;
+import io.druid.segment.LongColumnSelector;
+import io.druid.segment.ObjectColumnSelector;
+import io.druid.segment.column.ValueType;
+
+public class VirtualizedColumnSelectorFactory implements ColumnSelectorFactory
+{
+  private final ColumnSelectorFactory baseFactory;
+  private final VirtualColumns virtualColumns;
+
+  VirtualizedColumnSelectorFactory(
+      ColumnSelectorFactory baseFactory,
+      VirtualColumns virtualColumns
+  )
+  {
+    this.baseFactory = Preconditions.checkNotNull(baseFactory, "baseFactory");
+    this.virtualColumns = Preconditions.checkNotNull(virtualColumns, "virtualColumns");
+  }
+
+  @Override
+  public DimensionSelector makeDimensionSelector(DimensionSpec dimensionSpec)
+  {
+    if (virtualColumns.exists(dimensionSpec.getDimension())) {
+      return virtualColumns.makeDimensionSelector(dimensionSpec, baseFactory);
+    } else {
+      return baseFactory.makeDimensionSelector(dimensionSpec);
+    }
+  }
+
+  @Override
+  public FloatColumnSelector makeFloatColumnSelector(String columnName)
+  {
+    if (virtualColumns.exists(columnName)) {
+      return virtualColumns.makeFloatColumnSelector(columnName, baseFactory);
+    } else {
+      return baseFactory.makeFloatColumnSelector(columnName);
+    }
+  }
+
+  @Override
+  public LongColumnSelector makeLongColumnSelector(String columnName)
+  {
+    if (virtualColumns.exists(columnName)) {
+      return virtualColumns.makeLongColumnSelector(columnName, baseFactory);
+    } else {
+      return baseFactory.makeLongColumnSelector(columnName);
+    }
+  }
+
+  @Override
+  public ObjectColumnSelector makeObjectColumnSelector(String columnName)
+  {
+    if (virtualColumns.exists(columnName)) {
+      return virtualColumns.makeObjectColumnSelector(columnName, baseFactory);
+    } else {
+      return baseFactory.makeObjectColumnSelector(columnName);
+    }
+  }
+
+  @Override
+  public ValueType getNativeType(String columnName)
+  {
+    final ValueType virtualType = virtualColumns.getNativeType(columnName);
+    return virtualType != null ? virtualType : baseFactory.getNativeType(columnName);
+  }
+}
diff --git a/processing/src/test/java/io/druid/query/SchemaEvolutionTest.java b/processing/src/test/java/io/druid/query/SchemaEvolutionTest.java
index 8aa5997c109a..687023840531 100644
--- a/processing/src/test/java/io/druid/query/SchemaEvolutionTest.java
+++ b/processing/src/test/java/io/druid/query/SchemaEvolutionTest.java
@@ -49,6 +49,7 @@
 import io.druid.segment.QueryableIndex;
 import io.druid.segment.QueryableIndexSegment;
 import io.druid.segment.incremental.IncrementalIndexSchema;
+import io.druid.segment.virtual.ExpressionVirtualColumn;
 import org.joda.time.DateTime;
 import org.junit.After;
 import org.junit.Assert;
@@ -257,12 +258,15 @@ public void testNumericEvolutionTimeseriesAggregation()
         .newTimeseriesQueryBuilder()
         .dataSource(DATA_SOURCE)
         .intervals("1000/3000")
+        .virtualColumns(
+            new ExpressionVirtualColumn("expr", "c1 * 1")
+        )
         .aggregators(
             ImmutableList.of(
                 new LongSumAggregatorFactory("a", "c1"),
                 new DoubleSumAggregatorFactory("b", "c1"),
-                new LongSumAggregatorFactory("c", null, "c1 * 1"),
-                new DoubleSumAggregatorFactory("d", null, "c1 * 1")
+                new LongSumAggregatorFactory("c", "expr"),
+                new DoubleSumAggregatorFactory("d", "expr")
             )
         )
         .build();
diff --git a/processing/src/test/java/io/druid/query/aggregation/FilteredAggregatorTest.java b/processing/src/test/java/io/druid/query/aggregation/FilteredAggregatorTest.java
index 042c5ed34a68..4073a0570b64 100644
--- a/processing/src/test/java/io/druid/query/aggregation/FilteredAggregatorTest.java
+++ b/processing/src/test/java/io/druid/query/aggregation/FilteredAggregatorTest.java
@@ -40,10 +40,7 @@
 import io.druid.segment.DimensionSelector;
 import io.druid.segment.FloatColumnSelector;
 import io.druid.segment.LongColumnSelector;
-import io.druid.segment.NumericColumnSelector;
 import io.druid.segment.ObjectColumnSelector;
-import io.druid.segment.column.ColumnCapabilities;
-import io.druid.segment.column.ColumnCapabilitiesImpl;
 import io.druid.segment.column.ValueType;
 import io.druid.segment.data.ArrayBasedIndexedInts;
 import io.druid.segment.data.IndexedInts;
@@ -167,27 +164,13 @@ public ObjectColumnSelector makeObjectColumnSelector(String columnName)
       }
 
       @Override
-      public ColumnCapabilities getColumnCapabilities(String columnName)
+      public ValueType getNativeType(String columnName)
       {
-        ColumnCapabilitiesImpl caps;
         if (columnName.equals("value")) {
-          caps = new ColumnCapabilitiesImpl();
-          caps.setType(ValueType.FLOAT);
-          caps.setDictionaryEncoded(false);
-          caps.setHasBitmapIndexes(false);
+          return ValueType.FLOAT;
         } else {
-          caps = new ColumnCapabilitiesImpl();
-          caps.setType(ValueType.STRING);
-          caps.setDictionaryEncoded(true);
-          caps.setHasBitmapIndexes(true);
+          return ValueType.STRING;
         }
-        return caps;
-      }
-
-      @Override
-      public NumericColumnSelector makeMathExpressionSelector(String expression)
-      {
-        throw new UnsupportedOperationException();
       }
     };
   }
diff --git a/processing/src/test/java/io/druid/query/aggregation/JavaScriptAggregatorTest.java b/processing/src/test/java/io/druid/query/aggregation/JavaScriptAggregatorTest.java
index 3bb6d26fe65a..55e8fbcae753 100644
--- a/processing/src/test/java/io/druid/query/aggregation/JavaScriptAggregatorTest.java
+++ b/processing/src/test/java/io/druid/query/aggregation/JavaScriptAggregatorTest.java
@@ -28,9 +28,8 @@
 import io.druid.segment.DimensionSelector;
 import io.druid.segment.FloatColumnSelector;
 import io.druid.segment.LongColumnSelector;
-import io.druid.segment.NumericColumnSelector;
 import io.druid.segment.ObjectColumnSelector;
-import io.druid.segment.column.ColumnCapabilities;
+import io.druid.segment.column.ValueType;
 import org.junit.Assert;
 import org.junit.Rule;
 import org.junit.Test;
@@ -73,13 +72,7 @@ public ObjectColumnSelector makeObjectColumnSelector(String columnName)
     }
 
     @Override
-    public ColumnCapabilities getColumnCapabilities(String columnName)
-    {
-      return null;
-    }
-
-    @Override
-    public NumericColumnSelector makeMathExpressionSelector(String expression)
+    public ValueType getNativeType(String columnName)
     {
       return null;
     }
diff --git a/processing/src/test/java/io/druid/query/groupby/GroupByQueryRunnerTest.java b/processing/src/test/java/io/druid/query/groupby/GroupByQueryRunnerTest.java
index 0c36dea67a3d..051d24841740 100644
--- a/processing/src/test/java/io/druid/query/groupby/GroupByQueryRunnerTest.java
+++ b/processing/src/test/java/io/druid/query/groupby/GroupByQueryRunnerTest.java
@@ -107,6 +107,7 @@
 import io.druid.query.spec.MultipleIntervalSegmentSpec;
 import io.druid.segment.TestHelper;
 import io.druid.segment.column.Column;
+import io.druid.segment.virtual.ExpressionVirtualColumn;
 import org.joda.time.DateTime;
 import org.joda.time.DateTimeZone;
 import org.joda.time.Interval;
@@ -2203,16 +2204,33 @@ public void testMergeResultsAcrossMultipleDaysWithLimitAndOrderBy()
     TestHelper.assertExpectedObjects(
         Iterables.limit(expectedResults, limit), mergeRunner.run(fullQuery, context), String.format("limit: %d", limit)
     );
+  }
 
-    builder.setAggregatorSpecs(
-        Arrays.asList(
-            QueryRunnerTestHelper.rowsCount,
-            new LongSumAggregatorFactory("idx", null, "index * 2 + indexMin / 10")
+  @Test
+  public void testMergeResultsAcrossMultipleDaysWithLimitAndOrderByUsingMathExpressions()
+  {
+    final int limit = 14;
+    GroupByQuery.Builder builder = GroupByQuery
+        .builder()
+        .setDataSource(QueryRunnerTestHelper.dataSource)
+        .setInterval(QueryRunnerTestHelper.firstToThird)
+        .setVirtualColumns(
+            new ExpressionVirtualColumn("expr", "index * 2 + indexMin / 10")
         )
-    );
-    fullQuery = builder.build();
+        .setDimensions(Lists.newArrayList(new DefaultDimensionSpec("quality", "alias")))
+        .setAggregatorSpecs(
+            Arrays.asList(
+                QueryRunnerTestHelper.rowsCount,
+                new LongSumAggregatorFactory("idx", "expr")
+            )
+        )
+        .setGranularity(QueryGranularities.DAY)
+        .setLimit(limit)
+        .addOrderByColumn("idx", OrderByColumnSpec.Direction.DESCENDING);
+
+    GroupByQuery fullQuery = builder.build();
 
-    expectedResults = Arrays.asList(
+    List expectedResults = Arrays.asList(
         GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "premium", "rows", 3L, "idx", 6090L),
         GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "mezzanine", "rows", 3L, "idx", 6030L),
         GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "entertainment", "rows", 1L, "idx", 333L),
@@ -2230,8 +2248,9 @@ public void testMergeResultsAcrossMultipleDaysWithLimitAndOrderBy()
         GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "travel", "rows", 1L, "idx", 265L)
     );
 
-    mergeRunner = factory.getToolchest().mergeResults(runner);
+    QueryRunner mergeRunner = factory.getToolchest().mergeResults(runner);
 
+    Map context = Maps.newHashMap();
     TestHelper.assertExpectedObjects(
         Iterables.limit(expectedResults, limit), mergeRunner.run(fullQuery, context), String.format("limit: %d", limit)
     );
@@ -2405,10 +2424,13 @@ public void testGroupByOrderLimit() throws Exception
     );
 
     builder.limit(Integer.MAX_VALUE)
+           .setVirtualColumns(
+               new ExpressionVirtualColumn("expr", "index / 2 + indexMin")
+           )
            .setAggregatorSpecs(
                Arrays.asList(
                    QueryRunnerTestHelper.rowsCount,
-                   new DoubleSumAggregatorFactory("idx", null, "index / 2 + indexMin")
+                   new DoubleSumAggregatorFactory("idx", "expr")
                )
            );
 
@@ -3991,13 +4013,17 @@ public void testDifferentGroupingSubquery()
         GroupByQueryRunnerTestHelper.runQuery(factory, runner, query), ""
     );
 
-    subquery = subquery.withAggregatorSpecs(
-        Arrays.asList(
-            QueryRunnerTestHelper.rowsCount,
-            new LongSumAggregatorFactory("idx", null, "-index + 100"),
-            new LongSumAggregatorFactory("indexMaxPlusTen", "indexMaxPlusTen")
+    subquery = new GroupByQuery.Builder(subquery)
+        .setVirtualColumns(
+            new ExpressionVirtualColumn("expr", "-index + 100")
         )
-    );
+        .setAggregatorSpecs(
+            Arrays.asList(
+                QueryRunnerTestHelper.rowsCount,
+                new LongSumAggregatorFactory("idx", "expr"),
+                new LongSumAggregatorFactory("indexMaxPlusTen", "indexMaxPlusTen")
+            )
+        ).build();
     query = (GroupByQuery) query.withDataSource(new QueryDataSource(subquery));
 
     expectedResults = GroupByQueryRunnerTestHelper.createExpectedRows(
@@ -5106,6 +5132,34 @@ public void testSubqueryWithContextTimeout()
     TestHelper.assertExpectedObjects(expectedResults, results, "");
   }
 
+  @Test
+  public void testSubqueryWithOuterVirtualColumns()
+  {
+    final GroupByQuery subquery = GroupByQuery
+        .builder()
+        .setDataSource(QueryRunnerTestHelper.dataSource)
+        .setQuerySegmentSpec(QueryRunnerTestHelper.fullOnInterval)
+        .setDimensions(Lists.newArrayList(new DefaultDimensionSpec("quality", "alias")))
+        .setGranularity(QueryRunnerTestHelper.dayGran)
+        .build();
+
+    final GroupByQuery query = GroupByQuery
+        .builder()
+        .setDataSource(subquery)
+        .setQuerySegmentSpec(QueryRunnerTestHelper.firstToThird)
+        .setVirtualColumns(new ExpressionVirtualColumn("expr", "1"))
+        .setDimensions(Lists.newArrayList())
+        .setAggregatorSpecs(ImmutableList.of(new LongSumAggregatorFactory("count", "expr")))
+        .setGranularity(QueryRunnerTestHelper.allGran)
+        .build();
+
+    List expectedResults = Arrays.asList(
+        GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "count", 18L)
+    );
+    Iterable results = GroupByQueryRunnerTestHelper.runQuery(factory, runner, query);
+    TestHelper.assertExpectedObjects(expectedResults, results, "");
+  }
+
   @Test
   public void testSubqueryWithOuterCardinalityAggregator()
   {
@@ -5113,8 +5167,10 @@ public void testSubqueryWithOuterCardinalityAggregator()
         .builder()
         .setDataSource(QueryRunnerTestHelper.dataSource)
         .setQuerySegmentSpec(QueryRunnerTestHelper.fullOnInterval)
-        .setDimensions(Lists.newArrayList(new DefaultDimensionSpec("market", "market"),
-                                                         new DefaultDimensionSpec("quality", "quality")))
+        .setDimensions(Lists.newArrayList(
+            new DefaultDimensionSpec("market", "market"),
+            new DefaultDimensionSpec("quality", "quality")
+        ))
         .setAggregatorSpecs(
             Arrays.asList(
                 QueryRunnerTestHelper.rowsCount,
diff --git a/processing/src/test/java/io/druid/query/groupby/epinephelinae/TestColumnSelectorFactory.java b/processing/src/test/java/io/druid/query/groupby/epinephelinae/TestColumnSelectorFactory.java
index 8373698976c0..7b2fa7f5626b 100644
--- a/processing/src/test/java/io/druid/query/groupby/epinephelinae/TestColumnSelectorFactory.java
+++ b/processing/src/test/java/io/druid/query/groupby/epinephelinae/TestColumnSelectorFactory.java
@@ -25,9 +25,8 @@
 import io.druid.segment.DimensionSelector;
 import io.druid.segment.FloatColumnSelector;
 import io.druid.segment.LongColumnSelector;
-import io.druid.segment.NumericColumnSelector;
 import io.druid.segment.ObjectColumnSelector;
-import io.druid.segment.column.ColumnCapabilities;
+import io.druid.segment.column.ValueType;
 
 public class TestColumnSelectorFactory implements ColumnSelectorFactory
 {
@@ -90,13 +89,7 @@ public Object get()
   }
 
   @Override
-  public NumericColumnSelector makeMathExpressionSelector(String expression)
-  {
-    throw new UnsupportedOperationException("expression is not supported in current context");
-  }
-
-  @Override
-  public ColumnCapabilities getColumnCapabilities(String columnName)
+  public ValueType getNativeType(String columnName)
   {
     return null;
   }
diff --git a/processing/src/test/java/io/druid/query/select/SelectQuerySpecTest.java b/processing/src/test/java/io/druid/query/select/SelectQuerySpecTest.java
index b6f918fa6baa..9c4ee4c58ec6 100644
--- a/processing/src/test/java/io/druid/query/select/SelectQuerySpecTest.java
+++ b/processing/src/test/java/io/druid/query/select/SelectQuerySpecTest.java
@@ -60,7 +60,7 @@ public void testSerializationLegacyString() throws Exception
         + "\"granularity\":{\"type\":\"all\"},"
         + "\"dimensions\":[{\"type\":\"default\",\"dimension\":\"market\",\"outputName\":\"market\"},{\"type\":\"default\",\"dimension\":\"quality\",\"outputName\":\"quality\"}],"
         + "\"metrics\":[\"index\"],"
-        + "\"virtualColumns\":null,"
+        + "\"virtualColumns\":[],"
         + "\"pagingSpec\":{\"pagingIdentifiers\":{},\"threshold\":3,\"fromNext\":false},"
         + "\"context\":null}";
 
diff --git a/processing/src/test/java/io/druid/query/timeseries/TimeseriesQueryQueryToolChestTest.java b/processing/src/test/java/io/druid/query/timeseries/TimeseriesQueryQueryToolChestTest.java
index c005abdf1180..ee0376f799b7 100644
--- a/processing/src/test/java/io/druid/query/timeseries/TimeseriesQueryQueryToolChestTest.java
+++ b/processing/src/test/java/io/druid/query/timeseries/TimeseriesQueryQueryToolChestTest.java
@@ -31,6 +31,7 @@
 import io.druid.query.aggregation.AggregatorFactory;
 import io.druid.query.aggregation.CountAggregatorFactory;
 import io.druid.query.spec.MultipleIntervalSegmentSpec;
+import io.druid.segment.virtual.VirtualColumns;
 import org.joda.time.DateTime;
 import org.joda.time.Interval;
 import org.junit.Assert;
@@ -73,6 +74,7 @@ public void testCacheStrategy() throws Exception
                     )
                 ),
                 descending,
+                VirtualColumns.EMPTY,
                 null,
                 QueryGranularities.ALL,
                 ImmutableList.of(new CountAggregatorFactory("metric1")),
diff --git a/processing/src/test/java/io/druid/query/topn/TopNQueryQueryToolChestTest.java b/processing/src/test/java/io/druid/query/topn/TopNQueryQueryToolChestTest.java
index e651bbedee64..4e3838e44be1 100644
--- a/processing/src/test/java/io/druid/query/topn/TopNQueryQueryToolChestTest.java
+++ b/processing/src/test/java/io/druid/query/topn/TopNQueryQueryToolChestTest.java
@@ -40,6 +40,7 @@
 import io.druid.query.spec.MultipleIntervalSegmentSpec;
 import io.druid.segment.IncrementalIndexSegment;
 import io.druid.segment.TestIndex;
+import io.druid.segment.virtual.VirtualColumns;
 import org.joda.time.DateTime;
 import org.joda.time.Interval;
 import org.junit.Assert;
@@ -60,6 +61,7 @@ public void testCacheStrategy() throws Exception
         new TopNQueryQueryToolChest(null, null).getCacheStrategy(
             new TopNQuery(
                 new TableDataSource("dummy"),
+                VirtualColumns.EMPTY,
                 new DefaultDimensionSpec("test", "test"),
                 new NumericTopNMetricSpec("metric1"),
                 3,
diff --git a/processing/src/test/java/io/druid/query/topn/TopNQueryRunnerTest.java b/processing/src/test/java/io/druid/query/topn/TopNQueryRunnerTest.java
index cd12aff33f5d..2cab5e85d5ff 100644
--- a/processing/src/test/java/io/druid/query/topn/TopNQueryRunnerTest.java
+++ b/processing/src/test/java/io/druid/query/topn/TopNQueryRunnerTest.java
@@ -70,6 +70,7 @@
 import io.druid.query.timeseries.TimeseriesQuery;
 import io.druid.segment.TestHelper;
 import io.druid.segment.column.Column;
+import io.druid.segment.virtual.ExpressionVirtualColumn;
 import org.joda.time.DateTime;
 import org.joda.time.Interval;
 import org.junit.Assert;
@@ -1645,12 +1646,17 @@ public void testTopNCollapsingDimExtraction()
 
     assertExpectedResults(expectedResults, query);
 
-    query = query.withAggregatorSpecs(
-        Arrays.asList(
-            QueryRunnerTestHelper.rowsCount,
-            new DoubleSumAggregatorFactory("index", null, "-index + 100")
+    query = new TopNQueryBuilder(query)
+        .virtualColumns(
+            new ExpressionVirtualColumn("expr", "-index + 100")
         )
-    );
+        .aggregators(
+            Arrays.asList(
+                QueryRunnerTestHelper.rowsCount,
+                new DoubleSumAggregatorFactory("index", "expr")
+            )
+        )
+        .build();
 
     expectedResults = Arrays.asList(
         TopNQueryRunnerTestHelper.createExpectedRows(
diff --git a/processing/src/test/java/io/druid/segment/NullDimensionSelectorTest.java b/processing/src/test/java/io/druid/segment/NullDimensionSelectorTest.java
index aab2dff99c64..ac26061786c4 100644
--- a/processing/src/test/java/io/druid/segment/NullDimensionSelectorTest.java
+++ b/processing/src/test/java/io/druid/segment/NullDimensionSelectorTest.java
@@ -27,7 +27,7 @@
 
 public class NullDimensionSelectorTest {
 
-  private final NullDimensionSelector selector = new NullDimensionSelector();
+  private final NullDimensionSelector selector = NullDimensionSelector.instance();
 
   @Test
   public void testGetRow() throws Exception {
diff --git a/processing/src/test/java/io/druid/segment/TestIndex.java b/processing/src/test/java/io/druid/segment/TestIndex.java
index 084785933807..7184d38b05a1 100644
--- a/processing/src/test/java/io/druid/segment/TestIndex.java
+++ b/processing/src/test/java/io/druid/segment/TestIndex.java
@@ -41,6 +41,9 @@
 import io.druid.segment.incremental.IncrementalIndexSchema;
 import io.druid.segment.incremental.OnheapIncrementalIndex;
 import io.druid.segment.serde.ComplexMetrics;
+import io.druid.segment.virtual.ExpressionVirtualColumn;
+import io.druid.segment.virtual.VirtualColumn;
+import io.druid.segment.virtual.VirtualColumns;
 import org.joda.time.DateTime;
 import org.joda.time.Interval;
 
@@ -78,10 +81,15 @@ public class TestIndex
   public static final String[] METRICS = new String[]{"index", "indexMin", "indexMaxPlusTen"};
   private static final Logger log = new Logger(TestIndex.class);
   private static final Interval DATA_INTERVAL = new Interval("2011-01-12T00:00:00.000Z/2011-05-01T00:00:00.000Z");
+  private static final VirtualColumns VIRTUAL_COLUMNS = VirtualColumns.create(
+      Arrays.asList(
+          new ExpressionVirtualColumn("expr", "index + 10")
+      )
+  );
   public static final AggregatorFactory[] METRIC_AGGS = new AggregatorFactory[]{
       new DoubleSumAggregatorFactory(METRICS[0], METRICS[0]),
       new DoubleMinAggregatorFactory(METRICS[1], METRICS[0]),
-      new DoubleMaxAggregatorFactory(METRICS[2], null, "index + 10"),
+      new DoubleMaxAggregatorFactory(METRICS[2], VIRTUAL_COLUMNS.getVirtualColumns()[0].getOutputName()),
       new HyperUniquesAggregatorFactory("quality_uniques", "quality")
   };
   private static final IndexSpec indexSpec = new IndexSpec();
@@ -224,6 +232,7 @@ public static IncrementalIndex makeRealtimeIndex(final CharSource source, boolea
         .withMinTimestamp(new DateTime("2011-01-12T00:00:00.000Z").getMillis())
         .withTimestampSpec(new TimestampSpec("ds", "auto", null))
         .withQueryGranularity(QueryGranularities.NONE)
+        .withVirtualColumns(VIRTUAL_COLUMNS)
         .withMetrics(METRIC_AGGS)
         .withRollup(rollup)
         .build();
diff --git a/processing/src/test/java/io/druid/segment/filter/BaseFilterTest.java b/processing/src/test/java/io/druid/segment/filter/BaseFilterTest.java
index 76e192848dd2..13cb294acd95 100644
--- a/processing/src/test/java/io/druid/segment/filter/BaseFilterTest.java
+++ b/processing/src/test/java/io/druid/segment/filter/BaseFilterTest.java
@@ -50,13 +50,13 @@
 import io.druid.segment.QueryableIndexStorageAdapter;
 import io.druid.segment.StorageAdapter;
 import io.druid.segment.TestHelper;
-import io.druid.segment.VirtualColumns;
 import io.druid.segment.data.BitmapSerdeFactory;
 import io.druid.segment.data.ConciseBitmapSerdeFactory;
 import io.druid.segment.data.IndexedInts;
 import io.druid.segment.data.RoaringBitmapSerdeFactory;
 import io.druid.segment.incremental.IncrementalIndex;
 import io.druid.segment.incremental.IncrementalIndexStorageAdapter;
+import io.druid.segment.virtual.VirtualColumns;
 import org.joda.time.Interval;
 import org.junit.Assert;
 import org.junit.Before;
diff --git a/processing/src/test/java/io/druid/segment/incremental/IncrementalIndexMultiValueSpecTest.java b/processing/src/test/java/io/druid/segment/incremental/IncrementalIndexMultiValueSpecTest.java
index 48c98f2ce892..3fc595c147b9 100644
--- a/processing/src/test/java/io/druid/segment/incremental/IncrementalIndexMultiValueSpecTest.java
+++ b/processing/src/test/java/io/druid/segment/incremental/IncrementalIndexMultiValueSpecTest.java
@@ -28,6 +28,7 @@
 import io.druid.data.input.impl.TimestampSpec;
 import io.druid.granularity.QueryGranularities;
 import io.druid.query.aggregation.AggregatorFactory;
+import io.druid.segment.virtual.VirtualColumns;
 import org.junit.Assert;
 import org.junit.Test;
 
@@ -54,6 +55,7 @@ public void test() throws IndexSizeExceededException
         0,
         new TimestampSpec("ds", "auto", null),
         QueryGranularities.ALL,
+        VirtualColumns.EMPTY,
         dimensionsSpec,
         new AggregatorFactory[0],
         false
diff --git a/processing/src/test/java/io/druid/segment/incremental/IncrementalIndexStorageAdapterTest.java b/processing/src/test/java/io/druid/segment/incremental/IncrementalIndexStorageAdapterTest.java
index 34b79506d49d..ad0ece737cab 100644
--- a/processing/src/test/java/io/druid/segment/incremental/IncrementalIndexStorageAdapterTest.java
+++ b/processing/src/test/java/io/druid/segment/incremental/IncrementalIndexStorageAdapterTest.java
@@ -49,9 +49,9 @@
 import io.druid.segment.Cursor;
 import io.druid.segment.DimensionSelector;
 import io.druid.segment.StorageAdapter;
-import io.druid.segment.VirtualColumns;
 import io.druid.segment.data.IndexedInts;
 import io.druid.segment.filter.SelectorFilter;
+import io.druid.segment.virtual.VirtualColumns;
 import org.joda.time.DateTime;
 import org.joda.time.Interval;
 import org.junit.Assert;
@@ -264,7 +264,7 @@ public void testResetSanity() throws IOException
       Sequence cursorSequence = adapter.makeCursors(
           new SelectorFilter("sally", "bo"),
           interval,
-          null,
+          VirtualColumns.EMPTY,
           QueryGranularities.NONE,
           descending
       );
diff --git a/processing/src/test/java/io/druid/segment/incremental/OnheapIncrementalIndexBenchmark.java b/processing/src/test/java/io/druid/segment/incremental/OnheapIncrementalIndexBenchmark.java
index d7e89b337260..0bb927662cfc 100644
--- a/processing/src/test/java/io/druid/segment/incremental/OnheapIncrementalIndexBenchmark.java
+++ b/processing/src/test/java/io/druid/segment/incremental/OnheapIncrementalIndexBenchmark.java
@@ -157,9 +157,7 @@ protected Integer addToFacts(
 
         for (int i = 0; i < metrics.length; i++) {
           final AggregatorFactory agg = metrics[i];
-          aggs[i] = agg.factorize(
-              makeColumnSelectorFactory(agg, rowSupplier, deserializeComplexMetrics, null)
-          );
+          aggs[i] = agg.factorize(makeColumnSelectorFactory(agg, rowSupplier, deserializeComplexMetrics));
         }
         Integer rowIndex;
 
diff --git a/processing/src/test/java/io/druid/segment/virtual/VirtualColumnsTest.java b/processing/src/test/java/io/druid/segment/virtual/VirtualColumnsTest.java
new file mode 100644
index 000000000000..387daf307439
--- /dev/null
+++ b/processing/src/test/java/io/druid/segment/virtual/VirtualColumnsTest.java
@@ -0,0 +1,375 @@
+/*
+ * Licensed to Metamarkets Group Inc. (Metamarkets) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. Metamarkets licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package io.druid.segment.virtual;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.collect.ImmutableList;
+import com.google.common.primitives.Longs;
+import io.druid.jackson.DefaultObjectMapper;
+import io.druid.query.dimension.DefaultDimensionSpec;
+import io.druid.query.dimension.DimensionSpec;
+import io.druid.query.extraction.ExtractionFn;
+import io.druid.segment.ColumnSelectorFactory;
+import io.druid.segment.DimensionSelector;
+import io.druid.segment.FloatColumnSelector;
+import io.druid.segment.LongColumnSelector;
+import io.druid.segment.ObjectColumnSelector;
+import io.druid.segment.column.ValueType;
+import io.druid.segment.data.IndexedInts;
+import it.unimi.dsi.fastutil.ints.IntIterator;
+import it.unimi.dsi.fastutil.ints.IntIterators;
+import org.junit.Assert;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
+
+public class VirtualColumnsTest
+{
+  @Rule
+  public ExpectedException expectedException = ExpectedException.none();
+
+  @Test
+  public void testMakeSelectors()
+  {
+    final VirtualColumns virtualColumns = makeVirtualColumns();
+    final ObjectColumnSelector objectSelector = virtualColumns.makeObjectColumnSelector("expr", null);
+    final DimensionSelector dimensionSelector = virtualColumns.makeDimensionSelector(
+        new DefaultDimensionSpec("expr", "x"),
+        null
+    );
+    final FloatColumnSelector floatSelector = virtualColumns.makeFloatColumnSelector("expr", null);
+    final LongColumnSelector longSelector = virtualColumns.makeLongColumnSelector("expr", null);
+
+    Assert.assertEquals(1L, objectSelector.get());
+    Assert.assertEquals(null, dimensionSelector.lookupName(dimensionSelector.getRow().get(0)));
+    Assert.assertEquals(1.0f, floatSelector.get(), 0.0f);
+    Assert.assertEquals(1L, longSelector.get());
+  }
+
+  @Test
+  public void testMakeSelectorsWithDotSupport()
+  {
+    final VirtualColumns virtualColumns = makeVirtualColumns();
+    final ObjectColumnSelector objectSelector = virtualColumns.makeObjectColumnSelector("foo.5", null);
+    final DimensionSelector dimensionSelector = virtualColumns.makeDimensionSelector(
+        new DefaultDimensionSpec("foo.5", "x"),
+        null
+    );
+    final FloatColumnSelector floatSelector = virtualColumns.makeFloatColumnSelector("foo.5", null);
+    final LongColumnSelector longSelector = virtualColumns.makeLongColumnSelector("foo.5", null);
+
+    Assert.assertEquals(5L, objectSelector.get());
+    Assert.assertEquals("5", dimensionSelector.lookupName(dimensionSelector.getRow().get(0)));
+    Assert.assertEquals(5.0f, floatSelector.get(), 0.0f);
+    Assert.assertEquals(5L, longSelector.get());
+  }
+
+  @Test
+  public void testMakeSelectorsWithDotSupportBaseNameOnly()
+  {
+    final VirtualColumns virtualColumns = makeVirtualColumns();
+    final ObjectColumnSelector objectSelector = virtualColumns.makeObjectColumnSelector("foo", null);
+    final DimensionSelector dimensionSelector = virtualColumns.makeDimensionSelector(
+        new DefaultDimensionSpec("foo", "x"),
+        null
+    );
+    final FloatColumnSelector floatSelector = virtualColumns.makeFloatColumnSelector("foo", null);
+    final LongColumnSelector longSelector = virtualColumns.makeLongColumnSelector("foo", null);
+
+    Assert.assertEquals(-1L, objectSelector.get());
+    Assert.assertEquals("-1", dimensionSelector.lookupName(dimensionSelector.getRow().get(0)));
+    Assert.assertEquals(-1.0f, floatSelector.get(), 0.0f);
+    Assert.assertEquals(-1L, longSelector.get());
+  }
+
+  @Test
+  public void testTimeNotAllowed()
+  {
+    final ExpressionVirtualColumn expr = new ExpressionVirtualColumn("__time", "x + y");
+
+    expectedException.expect(IllegalArgumentException.class);
+    expectedException.expectMessage("virtualColumn name[__time] not allowed");
+
+    VirtualColumns.create(ImmutableList.of(expr));
+  }
+
+  @Test
+  public void testDuplicateNameDetection()
+  {
+    final ExpressionVirtualColumn expr = new ExpressionVirtualColumn("expr", "x + y");
+    final ExpressionVirtualColumn expr2 = new ExpressionVirtualColumn("expr", "x * 2");
+
+    expectedException.expect(IllegalArgumentException.class);
+    expectedException.expectMessage("Duplicate virtualColumn name[expr]");
+
+    VirtualColumns.create(ImmutableList.of(expr, expr2));
+  }
+
+  @Test
+  public void testCycleDetection()
+  {
+    final ExpressionVirtualColumn expr = new ExpressionVirtualColumn("expr", "x + expr2");
+    final ExpressionVirtualColumn expr2 = new ExpressionVirtualColumn("expr2", "expr * 2");
+
+    expectedException.expect(IllegalArgumentException.class);
+    expectedException.expectMessage("Self-referential column[expr]");
+
+    VirtualColumns.create(ImmutableList.of(expr, expr2));
+  }
+
+  @Test
+  public void testGetCacheKey() throws Exception
+  {
+    final VirtualColumns virtualColumns = VirtualColumns.create(
+        ImmutableList.of(
+            new ExpressionVirtualColumn("expr", "x + y")
+        )
+    );
+
+    final VirtualColumns virtualColumns2 = VirtualColumns.create(
+        ImmutableList.of(
+            new ExpressionVirtualColumn("expr", "x + y")
+        )
+    );
+
+    Assert.assertArrayEquals(virtualColumns.getCacheKey(), virtualColumns2.getCacheKey());
+    Assert.assertFalse(Arrays.equals(virtualColumns.getCacheKey(), VirtualColumns.EMPTY.getCacheKey()));
+  }
+
+  @Test
+  public void testEqualsAndHashCode() throws Exception
+  {
+    final VirtualColumns virtualColumns = VirtualColumns.create(
+        ImmutableList.of(
+            new ExpressionVirtualColumn("expr", "x + y")
+        )
+    );
+
+    final VirtualColumns virtualColumns2 = VirtualColumns.create(
+        ImmutableList.of(
+            new ExpressionVirtualColumn("expr", "x + y")
+        )
+    );
+
+    Assert.assertEquals(virtualColumns, virtualColumns);
+    Assert.assertEquals(virtualColumns, virtualColumns2);
+    Assert.assertNotEquals(VirtualColumns.EMPTY, virtualColumns);
+    Assert.assertNotEquals(VirtualColumns.EMPTY, null);
+
+    Assert.assertEquals(virtualColumns.hashCode(), virtualColumns.hashCode());
+    Assert.assertEquals(virtualColumns.hashCode(), virtualColumns2.hashCode());
+    Assert.assertNotEquals(VirtualColumns.EMPTY.hashCode(), virtualColumns.hashCode());
+  }
+
+  @Test
+  public void testSerde() throws Exception
+  {
+    final ObjectMapper mapper = new DefaultObjectMapper();
+    final ImmutableList theColumns = ImmutableList.of(
+        new ExpressionVirtualColumn("expr", "x + y"),
+        new ExpressionVirtualColumn("expr2", "x + z")
+    );
+    final VirtualColumns virtualColumns = VirtualColumns.create(theColumns);
+
+    Assert.assertEquals(
+        virtualColumns,
+        mapper.readValue(
+            mapper.writeValueAsString(virtualColumns),
+            VirtualColumns.class
+        )
+    );
+
+    Assert.assertEquals(
+        theColumns,
+        mapper.readValue(
+            mapper.writeValueAsString(virtualColumns),
+            mapper.getTypeFactory().constructParametricType(List.class, VirtualColumn.class)
+        )
+    );
+  }
+
+  private VirtualColumns makeVirtualColumns()
+  {
+    final ExpressionVirtualColumn expr = new ExpressionVirtualColumn("expr", "1");
+    final DottyVirtualColumn dotty = new DottyVirtualColumn("foo");
+    return VirtualColumns.create(ImmutableList.of(expr, dotty));
+  }
+
+  static class DottyVirtualColumn implements VirtualColumn
+  {
+    private final String name;
+
+    public DottyVirtualColumn(String name)
+    {
+      this.name = name;
+    }
+
+    @Override
+    public String getOutputName()
+    {
+      return name;
+    }
+
+    @Override
+    public ObjectColumnSelector makeObjectColumnSelector(String columnName, ColumnSelectorFactory factory)
+    {
+      final LongColumnSelector selector = makeLongColumnSelector(columnName, factory);
+      return new ObjectColumnSelector()
+      {
+        @Override
+        public Class classOfObject()
+        {
+          return Long.class;
+        }
+
+        @Override
+        public Object get()
+        {
+          return selector.get();
+        }
+      };
+    }
+
+    @Override
+    public DimensionSelector makeDimensionSelector(DimensionSpec dimensionSpec, ColumnSelectorFactory factory)
+    {
+      final LongColumnSelector selector = makeLongColumnSelector(dimensionSpec.getDimension(), factory);
+      final ExtractionFn extractionFn = dimensionSpec.getExtractionFn();
+      final DimensionSelector dimensionSelector = new DimensionSelector()
+      {
+        @Override
+        public IndexedInts getRow()
+        {
+          return new IndexedInts()
+          {
+            @Override
+            public int size()
+            {
+              return 1;
+            }
+
+            @Override
+            public int get(int index)
+            {
+              return 0;
+            }
+
+            @Override
+            public IntIterator iterator()
+            {
+              return IntIterators.singleton(0);
+            }
+
+            @Override
+            public void fill(int index, int[] toFill)
+            {
+              throw new UnsupportedOperationException("fill not supported");
+            }
+
+            @Override
+            public void close() throws IOException
+            {
+
+            }
+          };
+        }
+
+        @Override
+        public int getValueCardinality()
+        {
+          return DimensionSelector.CARDINALITY_UNKNOWN;
+        }
+
+        @Override
+        public String lookupName(int id)
+        {
+          final String stringValue = String.valueOf(selector.get());
+          return extractionFn == null ? stringValue : extractionFn.apply(stringValue);
+        }
+
+        @Override
+        public int lookupId(String name)
+        {
+          return 0;
+        }
+      };
+
+      return dimensionSpec.decorate(dimensionSelector);
+    }
+
+    @Override
+    public FloatColumnSelector makeFloatColumnSelector(String columnName, ColumnSelectorFactory factory)
+    {
+      final LongColumnSelector selector = makeLongColumnSelector(columnName, factory);
+      return new FloatColumnSelector()
+      {
+        @Override
+        public float get()
+        {
+          return selector.get();
+        }
+      };
+    }
+
+    @Override
+    public LongColumnSelector makeLongColumnSelector(String columnName, ColumnSelectorFactory factory)
+    {
+      final String subColumn = VirtualColumns.splitColumnName(columnName).rhs;
+      final Long boxed = subColumn == null ? null : Longs.tryParse(subColumn);
+      final long theLong = boxed == null ? -1 : boxed;
+      return new LongColumnSelector()
+      {
+        @Override
+        public long get()
+        {
+          return theLong;
+        }
+      };
+    }
+
+    @Override
+    public ValueType nativeType(String columnName)
+    {
+      return ValueType.LONG;
+    }
+
+    @Override
+    public List requiredColumns()
+    {
+      return ImmutableList.of();
+    }
+
+    @Override
+    public boolean usesDotNotation()
+    {
+      return true;
+    }
+
+    @Override
+    public byte[] getCacheKey()
+    {
+      throw new UnsupportedOperationException();
+    }
+  }
+}
diff --git a/server/src/main/java/io/druid/segment/realtime/firehose/IngestSegmentFirehose.java b/server/src/main/java/io/druid/segment/realtime/firehose/IngestSegmentFirehose.java
index 30e7932d2dec..317458a55ea1 100644
--- a/server/src/main/java/io/druid/segment/realtime/firehose/IngestSegmentFirehose.java
+++ b/server/src/main/java/io/druid/segment/realtime/firehose/IngestSegmentFirehose.java
@@ -39,7 +39,7 @@
 import io.druid.segment.DimensionSelector;
 import io.druid.segment.LongColumnSelector;
 import io.druid.segment.ObjectColumnSelector;
-import io.druid.segment.VirtualColumns;
+import io.druid.segment.virtual.VirtualColumns;
 import io.druid.segment.column.Column;
 import io.druid.segment.data.IndexedInts;
 import io.druid.segment.filter.Filters;
diff --git a/services/src/main/java/io/druid/cli/DumpSegment.java b/services/src/main/java/io/druid/cli/DumpSegment.java
index 7f1d9f781fff..02506141d9ef 100644
--- a/services/src/main/java/io/druid/cli/DumpSegment.java
+++ b/services/src/main/java/io/druid/cli/DumpSegment.java
@@ -71,7 +71,7 @@
 import io.druid.segment.QueryableIndex;
 import io.druid.segment.QueryableIndexSegment;
 import io.druid.segment.QueryableIndexStorageAdapter;
-import io.druid.segment.VirtualColumns;
+import io.druid.segment.virtual.VirtualColumns;
 import io.druid.segment.column.BitmapIndex;
 import io.druid.segment.column.Column;
 import io.druid.segment.column.ColumnConfig;

From fda5588482d52754cc5732cc0cb5a66000885f9d Mon Sep 17 00:00:00 2001
From: Gian Merlino 
Date: Wed, 7 Dec 2016 13:56:22 -0800
Subject: [PATCH 2/4] Add dimension selector for ExpressionVirtualColumn.

---
 ...ctSingleValueVirtualDimensionSelector.java | 89 +++++++++++++++++
 .../ExpressionObjectColumnSelector.java       |  2 +-
 .../virtual/ExpressionVirtualColumn.java      | 16 +++-
 .../virtual/ExpressionVirtualColumnTest.java  | 96 +++++++++++++++++++
 .../segment/virtual/VirtualColumnsTest.java   |  2 +-
 5 files changed, 201 insertions(+), 4 deletions(-)
 create mode 100644 processing/src/main/java/io/druid/segment/virtual/AbstractSingleValueVirtualDimensionSelector.java
 create mode 100644 processing/src/test/java/io/druid/segment/virtual/ExpressionVirtualColumnTest.java

diff --git a/processing/src/main/java/io/druid/segment/virtual/AbstractSingleValueVirtualDimensionSelector.java b/processing/src/main/java/io/druid/segment/virtual/AbstractSingleValueVirtualDimensionSelector.java
new file mode 100644
index 000000000000..2671c5fa152c
--- /dev/null
+++ b/processing/src/main/java/io/druid/segment/virtual/AbstractSingleValueVirtualDimensionSelector.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to Metamarkets Group Inc. (Metamarkets) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. Metamarkets licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package io.druid.segment.virtual;
+
+import io.druid.segment.DimensionSelector;
+import io.druid.segment.data.IndexedInts;
+import it.unimi.dsi.fastutil.ints.IntIterator;
+import it.unimi.dsi.fastutil.ints.IntIterators;
+
+import java.io.IOException;
+
+public abstract class AbstractSingleValueVirtualDimensionSelector implements DimensionSelector
+{
+  private static final IndexedInts ALWAYS_ZERO = new IndexedInts()
+  {
+    @Override
+    public int size()
+    {
+      return 1;
+    }
+
+    @Override
+    public int get(int index)
+    {
+      return 0;
+    }
+
+    @Override
+    public IntIterator iterator()
+    {
+      return IntIterators.singleton(0);
+    }
+
+    @Override
+    public void fill(int index, int[] toFill)
+    {
+      throw new UnsupportedOperationException("Cannot fill");
+    }
+
+    @Override
+    public void close() throws IOException
+    {
+
+    }
+  };
+
+  protected abstract String getValue();
+
+  @Override
+  public IndexedInts getRow()
+  {
+    return ALWAYS_ZERO;
+  }
+
+  @Override
+  public int getValueCardinality()
+  {
+    return DimensionSelector.CARDINALITY_UNKNOWN;
+  }
+
+  @Override
+  public String lookupName(int id)
+  {
+    return getValue();
+  }
+
+  @Override
+  public int lookupId(String name)
+  {
+    throw new UnsupportedOperationException("Cannot lookupId on virtual dimensions");
+  }
+}
diff --git a/processing/src/main/java/io/druid/segment/virtual/ExpressionObjectColumnSelector.java b/processing/src/main/java/io/druid/segment/virtual/ExpressionObjectColumnSelector.java
index 9bd7b7e0f9ee..1c4f2d7d288b 100644
--- a/processing/src/main/java/io/druid/segment/virtual/ExpressionObjectColumnSelector.java
+++ b/processing/src/main/java/io/druid/segment/virtual/ExpressionObjectColumnSelector.java
@@ -34,7 +34,7 @@
 
 import java.util.Map;
 
-public class ExpressionObjectColumnSelector implements ObjectColumnSelector
+class ExpressionObjectColumnSelector implements ObjectColumnSelector
 {
   private final Expr expression;
   private final Expr.ObjectBinding bindings;
diff --git a/processing/src/main/java/io/druid/segment/virtual/ExpressionVirtualColumn.java b/processing/src/main/java/io/druid/segment/virtual/ExpressionVirtualColumn.java
index f0f1f53832af..7bcb27faad11 100644
--- a/processing/src/main/java/io/druid/segment/virtual/ExpressionVirtualColumn.java
+++ b/processing/src/main/java/io/druid/segment/virtual/ExpressionVirtualColumn.java
@@ -30,7 +30,6 @@
 import io.druid.segment.DimensionSelector;
 import io.druid.segment.FloatColumnSelector;
 import io.druid.segment.LongColumnSelector;
-import io.druid.segment.NullDimensionSelector;
 import io.druid.segment.ObjectColumnSelector;
 import io.druid.segment.column.ValueType;
 import org.apache.commons.codec.Charsets;
@@ -83,7 +82,20 @@ public DimensionSelector makeDimensionSelector(
       final ColumnSelectorFactory columnSelectorFactory
   )
   {
-    return NullDimensionSelector.instance();
+    final ExpressionObjectColumnSelector numericSelector = new ExpressionObjectColumnSelector(
+        parsedExpression,
+        columnSelectorFactory
+    );
+    class ExpressionDimensionSelector extends AbstractSingleValueVirtualDimensionSelector
+    {
+      @Override
+      protected String getValue()
+      {
+        final Number number = numericSelector.get();
+        return number == null ? null : String.valueOf(number);
+      }
+    }
+    return new ExpressionDimensionSelector();
   }
 
   @Override
diff --git a/processing/src/test/java/io/druid/segment/virtual/ExpressionVirtualColumnTest.java b/processing/src/test/java/io/druid/segment/virtual/ExpressionVirtualColumnTest.java
new file mode 100644
index 000000000000..b7a227f33308
--- /dev/null
+++ b/processing/src/test/java/io/druid/segment/virtual/ExpressionVirtualColumnTest.java
@@ -0,0 +1,96 @@
+/*
+ * Licensed to Metamarkets Group Inc. (Metamarkets) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. Metamarkets licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package io.druid.segment.virtual;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import io.druid.data.input.MapBasedInputRow;
+import io.druid.query.dimension.DefaultDimensionSpec;
+import io.druid.query.groupby.epinephelinae.TestColumnSelectorFactory;
+import io.druid.segment.DimensionSelector;
+import io.druid.segment.FloatColumnSelector;
+import io.druid.segment.LongColumnSelector;
+import io.druid.segment.ObjectColumnSelector;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class ExpressionVirtualColumnTest
+{
+  @Test
+  public void testMakeSelectors()
+  {
+    final TestColumnSelectorFactory columnSelectorFactory = new TestColumnSelectorFactory();
+    final ExpressionVirtualColumn virtualColumn = new ExpressionVirtualColumn("expr", "x + y");
+
+    final ObjectColumnSelector objectSelector = virtualColumn.makeObjectColumnSelector("expr", columnSelectorFactory);
+    final DimensionSelector dimensionSelector = virtualColumn.makeDimensionSelector(
+        new DefaultDimensionSpec("expr", "x"),
+        columnSelectorFactory
+    );
+    final FloatColumnSelector floatSelector = virtualColumn.makeFloatColumnSelector("expr", columnSelectorFactory);
+    final LongColumnSelector longSelector = virtualColumn.makeLongColumnSelector("expr", columnSelectorFactory);
+
+    columnSelectorFactory.setRow(
+        new MapBasedInputRow(
+            0,
+            ImmutableList.of(),
+            ImmutableMap.of()
+        )
+    );
+
+    Assert.assertEquals(0L, objectSelector.get());
+    Assert.assertEquals("0", dimensionSelector.lookupName(dimensionSelector.getRow().get(0)));
+    Assert.assertEquals(0.0f, floatSelector.get(), 0.0f);
+    Assert.assertEquals(0L, longSelector.get());
+
+    columnSelectorFactory.setRow(
+        new MapBasedInputRow(
+            0,
+            ImmutableList.of(),
+            ImmutableMap.of("x", 4)
+        )
+    );
+
+    Assert.assertEquals(4L, objectSelector.get());
+    Assert.assertEquals("4", dimensionSelector.lookupName(dimensionSelector.getRow().get(0)));
+    Assert.assertEquals(4f, floatSelector.get(), 0.0f);
+    Assert.assertEquals(4L, longSelector.get());
+
+    columnSelectorFactory.setRow(
+        new MapBasedInputRow(
+            0,
+            ImmutableList.of(),
+            ImmutableMap.of("x", 2.1, "y", 3L)
+        )
+    );
+
+    Assert.assertEquals(5.1d, objectSelector.get());
+    Assert.assertEquals("5.1", dimensionSelector.lookupName(dimensionSelector.getRow().get(0)));
+    Assert.assertEquals(5.1f, floatSelector.get(), 0.0f);
+    Assert.assertEquals(5L, longSelector.get());
+  }
+
+  @Test
+  public void testRequiredColumns()
+  {
+    final ExpressionVirtualColumn virtualColumn = new ExpressionVirtualColumn("expr", "x + y");
+    Assert.assertEquals(ImmutableList.of("x", "y"), virtualColumn.requiredColumns());
+  }
+}
diff --git a/processing/src/test/java/io/druid/segment/virtual/VirtualColumnsTest.java b/processing/src/test/java/io/druid/segment/virtual/VirtualColumnsTest.java
index 387daf307439..99f427093f39 100644
--- a/processing/src/test/java/io/druid/segment/virtual/VirtualColumnsTest.java
+++ b/processing/src/test/java/io/druid/segment/virtual/VirtualColumnsTest.java
@@ -62,7 +62,7 @@ public void testMakeSelectors()
     final LongColumnSelector longSelector = virtualColumns.makeLongColumnSelector("expr", null);
 
     Assert.assertEquals(1L, objectSelector.get());
-    Assert.assertEquals(null, dimensionSelector.lookupName(dimensionSelector.getRow().get(0)));
+    Assert.assertEquals("1", dimensionSelector.lookupName(dimensionSelector.getRow().get(0)));
     Assert.assertEquals(1.0f, floatSelector.get(), 0.0f);
     Assert.assertEquals(1L, longSelector.get());
   }

From 5899da81cf6f3e655cf48f9c58f776de59ecaa4e Mon Sep 17 00:00:00 2001
From: Gian Merlino 
Date: Wed, 7 Dec 2016 17:23:32 -0800
Subject: [PATCH 3/4] Move VirtualColumn, VirtualColumns back to
 io.druid.segment package.

---
 .../io/druid/benchmark/FilterPartitionBenchmark.java  |  2 +-
 .../indexing/IncrementalIndexReadBenchmark.java       |  2 +-
 .../main/java/io/druid/segment/MapVirtualColumn.java  |  2 --
 .../java/io/druid/segment/MapVirtualColumnTest.java   |  1 -
 .../src/main/java/io/druid/indexer/InputRowSerde.java |  2 +-
 processing/src/main/java/io/druid/query/Druids.java   |  4 ++--
 .../main/java/io/druid/query/QueryRunnerHelper.java   |  2 +-
 .../java/io/druid/query/groupby/GroupByQuery.java     |  4 ++--
 .../java/io/druid/query/metadata/SegmentAnalyzer.java |  2 +-
 .../java/io/druid/query/search/SearchQueryRunner.java |  2 +-
 .../main/java/io/druid/query/select/SelectQuery.java  |  2 +-
 .../timeboundary/TimeBoundaryQueryRunnerFactory.java  |  2 +-
 .../io/druid/query/timeseries/TimeseriesQuery.java    |  2 +-
 .../src/main/java/io/druid/query/topn/TopNQuery.java  |  2 +-
 .../java/io/druid/query/topn/TopNQueryBuilder.java    |  4 ++--
 .../src/main/java/io/druid/segment/CursorFactory.java |  1 -
 .../druid/segment/QueryableIndexStorageAdapter.java   |  1 -
 .../io/druid/segment/{virtual => }/VirtualColumn.java |  3 ++-
 .../druid/segment/{virtual => }/VirtualColumns.java   | 11 ++---------
 .../druid/segment/incremental/IncrementalIndex.java   |  2 +-
 .../segment/incremental/IncrementalIndexSchema.java   |  2 +-
 .../incremental/IncrementalIndexStorageAdapter.java   |  2 +-
 .../segment/virtual/ExpressionVirtualColumn.java      |  1 +
 .../virtual/VirtualizedColumnSelectorFactory.java     |  1 +
 .../timeseries/TimeseriesQueryQueryToolChestTest.java |  2 +-
 .../druid/query/topn/TopNQueryQueryToolChestTest.java |  2 +-
 .../src/test/java/io/druid/segment/TestIndex.java     |  2 --
 .../java/io/druid/segment/filter/BaseFilterTest.java  |  2 +-
 .../IncrementalIndexMultiValueSpecTest.java           |  2 +-
 .../IncrementalIndexStorageAdapterTest.java           |  2 +-
 .../io/druid/segment/virtual/VirtualColumnsTest.java  |  2 ++
 .../realtime/firehose/IngestSegmentFirehose.java      |  2 +-
 services/src/main/java/io/druid/cli/DumpSegment.java  |  2 +-
 33 files changed, 34 insertions(+), 43 deletions(-)
 rename processing/src/main/java/io/druid/segment/{virtual => }/VirtualColumn.java (98%)
 rename processing/src/main/java/io/druid/segment/{virtual => }/VirtualColumns.java (95%)

diff --git a/benchmarks/src/main/java/io/druid/benchmark/FilterPartitionBenchmark.java b/benchmarks/src/main/java/io/druid/benchmark/FilterPartitionBenchmark.java
index b5ff728555d1..825940f221c2 100644
--- a/benchmarks/src/main/java/io/druid/benchmark/FilterPartitionBenchmark.java
+++ b/benchmarks/src/main/java/io/druid/benchmark/FilterPartitionBenchmark.java
@@ -61,7 +61,7 @@
 import io.druid.segment.QueryableIndex;
 import io.druid.segment.QueryableIndexStorageAdapter;
 import io.druid.segment.StorageAdapter;
-import io.druid.segment.virtual.VirtualColumns;
+import io.druid.segment.VirtualColumns;
 import io.druid.segment.column.Column;
 import io.druid.segment.column.ColumnConfig;
 import io.druid.segment.data.IndexedInts;
diff --git a/benchmarks/src/main/java/io/druid/benchmark/indexing/IncrementalIndexReadBenchmark.java b/benchmarks/src/main/java/io/druid/benchmark/indexing/IncrementalIndexReadBenchmark.java
index 7a6a768f1ed2..6cafa63ec464 100644
--- a/benchmarks/src/main/java/io/druid/benchmark/indexing/IncrementalIndexReadBenchmark.java
+++ b/benchmarks/src/main/java/io/druid/benchmark/indexing/IncrementalIndexReadBenchmark.java
@@ -45,7 +45,7 @@
 import io.druid.query.search.search.ContainsSearchQuerySpec;
 import io.druid.segment.Cursor;
 import io.druid.segment.DimensionSelector;
-import io.druid.segment.virtual.VirtualColumns;
+import io.druid.segment.VirtualColumns;
 import io.druid.segment.data.IndexedInts;
 import io.druid.segment.incremental.IncrementalIndex;
 import io.druid.segment.incremental.IncrementalIndexSchema;
diff --git a/extensions-contrib/virtual-columns/src/main/java/io/druid/segment/MapVirtualColumn.java b/extensions-contrib/virtual-columns/src/main/java/io/druid/segment/MapVirtualColumn.java
index 659912f24004..7fc360ba7437 100644
--- a/extensions-contrib/virtual-columns/src/main/java/io/druid/segment/MapVirtualColumn.java
+++ b/extensions-contrib/virtual-columns/src/main/java/io/druid/segment/MapVirtualColumn.java
@@ -30,9 +30,7 @@
 import io.druid.query.filter.DimFilterUtils;
 import io.druid.segment.column.ValueType;
 import io.druid.segment.data.IndexedInts;
-import io.druid.segment.virtual.VirtualColumn;
 import io.druid.segment.virtual.VirtualColumnCacheHelper;
-import io.druid.segment.virtual.VirtualColumns;
 
 import java.nio.ByteBuffer;
 import java.util.List;
diff --git a/extensions-contrib/virtual-columns/src/test/java/io/druid/segment/MapVirtualColumnTest.java b/extensions-contrib/virtual-columns/src/test/java/io/druid/segment/MapVirtualColumnTest.java
index c7ca1a93e068..2f3ff107883e 100644
--- a/extensions-contrib/virtual-columns/src/test/java/io/druid/segment/MapVirtualColumnTest.java
+++ b/extensions-contrib/virtual-columns/src/test/java/io/druid/segment/MapVirtualColumnTest.java
@@ -44,7 +44,6 @@
 import io.druid.segment.incremental.IncrementalIndex;
 import io.druid.segment.incremental.IncrementalIndexSchema;
 import io.druid.segment.incremental.OnheapIncrementalIndex;
-import io.druid.segment.virtual.VirtualColumn;
 import org.joda.time.DateTime;
 import org.junit.Assert;
 import org.junit.Test;
diff --git a/indexing-hadoop/src/main/java/io/druid/indexer/InputRowSerde.java b/indexing-hadoop/src/main/java/io/druid/indexer/InputRowSerde.java
index 9831a1ecb997..937c0db262db 100644
--- a/indexing-hadoop/src/main/java/io/druid/indexer/InputRowSerde.java
+++ b/indexing-hadoop/src/main/java/io/druid/indexer/InputRowSerde.java
@@ -36,7 +36,7 @@
 import io.druid.segment.incremental.IncrementalIndex;
 import io.druid.segment.serde.ComplexMetricSerde;
 import io.druid.segment.serde.ComplexMetrics;
-import io.druid.segment.virtual.VirtualColumns;
+import io.druid.segment.VirtualColumns;
 import org.apache.hadoop.io.ArrayWritable;
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.io.WritableUtils;
diff --git a/processing/src/main/java/io/druid/query/Druids.java b/processing/src/main/java/io/druid/query/Druids.java
index 80f5d647951d..e37e8ebcae40 100644
--- a/processing/src/main/java/io/druid/query/Druids.java
+++ b/processing/src/main/java/io/druid/query/Druids.java
@@ -54,8 +54,8 @@
 import io.druid.query.timeboundary.TimeBoundaryQuery;
 import io.druid.query.timeboundary.TimeBoundaryResultValue;
 import io.druid.query.timeseries.TimeseriesQuery;
-import io.druid.segment.virtual.VirtualColumn;
-import io.druid.segment.virtual.VirtualColumns;
+import io.druid.segment.VirtualColumn;
+import io.druid.segment.VirtualColumns;
 import org.joda.time.DateTime;
 import org.joda.time.Interval;
 
diff --git a/processing/src/main/java/io/druid/query/QueryRunnerHelper.java b/processing/src/main/java/io/druid/query/QueryRunnerHelper.java
index 2b15b012ef55..0d1cd979d9c9 100644
--- a/processing/src/main/java/io/druid/query/QueryRunnerHelper.java
+++ b/processing/src/main/java/io/druid/query/QueryRunnerHelper.java
@@ -30,7 +30,7 @@
 import io.druid.query.filter.Filter;
 import io.druid.segment.Cursor;
 import io.druid.segment.StorageAdapter;
-import io.druid.segment.virtual.VirtualColumns;
+import io.druid.segment.VirtualColumns;
 import org.joda.time.Interval;
 
 import java.io.Closeable;
diff --git a/processing/src/main/java/io/druid/query/groupby/GroupByQuery.java b/processing/src/main/java/io/druid/query/groupby/GroupByQuery.java
index 5c1af08da634..fe6103623e3a 100644
--- a/processing/src/main/java/io/druid/query/groupby/GroupByQuery.java
+++ b/processing/src/main/java/io/druid/query/groupby/GroupByQuery.java
@@ -56,8 +56,8 @@
 import io.druid.query.groupby.orderby.OrderByColumnSpec;
 import io.druid.query.spec.LegacySegmentSpec;
 import io.druid.query.spec.QuerySegmentSpec;
-import io.druid.segment.virtual.VirtualColumn;
-import io.druid.segment.virtual.VirtualColumns;
+import io.druid.segment.VirtualColumn;
+import io.druid.segment.VirtualColumns;
 import org.joda.time.Interval;
 
 import java.util.Arrays;
diff --git a/processing/src/main/java/io/druid/query/metadata/SegmentAnalyzer.java b/processing/src/main/java/io/druid/query/metadata/SegmentAnalyzer.java
index de7f0bf1ad3b..4e038e4ba01e 100644
--- a/processing/src/main/java/io/druid/query/metadata/SegmentAnalyzer.java
+++ b/processing/src/main/java/io/druid/query/metadata/SegmentAnalyzer.java
@@ -48,7 +48,7 @@
 import io.druid.segment.data.IndexedInts;
 import io.druid.segment.serde.ComplexMetricSerde;
 import io.druid.segment.serde.ComplexMetrics;
-import io.druid.segment.virtual.VirtualColumns;
+import io.druid.segment.VirtualColumns;
 import org.joda.time.Interval;
 
 import javax.annotation.Nullable;
diff --git a/processing/src/main/java/io/druid/query/search/SearchQueryRunner.java b/processing/src/main/java/io/druid/query/search/SearchQueryRunner.java
index f00c0b922115..954b4de49a97 100644
--- a/processing/src/main/java/io/druid/query/search/SearchQueryRunner.java
+++ b/processing/src/main/java/io/druid/query/search/SearchQueryRunner.java
@@ -57,7 +57,7 @@
 import io.druid.segment.column.GenericColumn;
 import io.druid.segment.data.IndexedInts;
 import io.druid.segment.filter.Filters;
-import io.druid.segment.virtual.VirtualColumns;
+import io.druid.segment.VirtualColumns;
 import org.apache.commons.lang.mutable.MutableInt;
 import org.joda.time.Interval;
 
diff --git a/processing/src/main/java/io/druid/query/select/SelectQuery.java b/processing/src/main/java/io/druid/query/select/SelectQuery.java
index bbc2cf04cbd6..d33ff67fe7d8 100644
--- a/processing/src/main/java/io/druid/query/select/SelectQuery.java
+++ b/processing/src/main/java/io/druid/query/select/SelectQuery.java
@@ -31,7 +31,7 @@
 import io.druid.query.dimension.DimensionSpec;
 import io.druid.query.filter.DimFilter;
 import io.druid.query.spec.QuerySegmentSpec;
-import io.druid.segment.virtual.VirtualColumns;
+import io.druid.segment.VirtualColumns;
 
 import java.util.List;
 import java.util.Map;
diff --git a/processing/src/main/java/io/druid/query/timeboundary/TimeBoundaryQueryRunnerFactory.java b/processing/src/main/java/io/druid/query/timeboundary/TimeBoundaryQueryRunnerFactory.java
index c77b5f8d3e52..9773a08852f8 100644
--- a/processing/src/main/java/io/druid/query/timeboundary/TimeBoundaryQueryRunnerFactory.java
+++ b/processing/src/main/java/io/druid/query/timeboundary/TimeBoundaryQueryRunnerFactory.java
@@ -41,7 +41,7 @@
 import io.druid.segment.StorageAdapter;
 import io.druid.segment.column.Column;
 import io.druid.segment.filter.Filters;
-import io.druid.segment.virtual.VirtualColumns;
+import io.druid.segment.VirtualColumns;
 import org.joda.time.DateTime;
 
 import java.util.Iterator;
diff --git a/processing/src/main/java/io/druid/query/timeseries/TimeseriesQuery.java b/processing/src/main/java/io/druid/query/timeseries/TimeseriesQuery.java
index 0a90f17e2426..01f04638b687 100644
--- a/processing/src/main/java/io/druid/query/timeseries/TimeseriesQuery.java
+++ b/processing/src/main/java/io/druid/query/timeseries/TimeseriesQuery.java
@@ -33,7 +33,7 @@
 import io.druid.query.aggregation.PostAggregator;
 import io.druid.query.filter.DimFilter;
 import io.druid.query.spec.QuerySegmentSpec;
-import io.druid.segment.virtual.VirtualColumns;
+import io.druid.segment.VirtualColumns;
 
 import java.util.List;
 import java.util.Map;
diff --git a/processing/src/main/java/io/druid/query/topn/TopNQuery.java b/processing/src/main/java/io/druid/query/topn/TopNQuery.java
index 41915b122c6e..e0991425b46d 100644
--- a/processing/src/main/java/io/druid/query/topn/TopNQuery.java
+++ b/processing/src/main/java/io/druid/query/topn/TopNQuery.java
@@ -34,7 +34,7 @@
 import io.druid.query.dimension.DimensionSpec;
 import io.druid.query.filter.DimFilter;
 import io.druid.query.spec.QuerySegmentSpec;
-import io.druid.segment.virtual.VirtualColumns;
+import io.druid.segment.VirtualColumns;
 
 import java.util.List;
 import java.util.Map;
diff --git a/processing/src/main/java/io/druid/query/topn/TopNQueryBuilder.java b/processing/src/main/java/io/druid/query/topn/TopNQueryBuilder.java
index 2d0f077c6868..dfe68b262361 100644
--- a/processing/src/main/java/io/druid/query/topn/TopNQueryBuilder.java
+++ b/processing/src/main/java/io/druid/query/topn/TopNQueryBuilder.java
@@ -33,8 +33,8 @@
 import io.druid.query.filter.SelectorDimFilter;
 import io.druid.query.spec.LegacySegmentSpec;
 import io.druid.query.spec.QuerySegmentSpec;
-import io.druid.segment.virtual.VirtualColumn;
-import io.druid.segment.virtual.VirtualColumns;
+import io.druid.segment.VirtualColumn;
+import io.druid.segment.VirtualColumns;
 import org.joda.time.Interval;
 
 import java.util.Arrays;
diff --git a/processing/src/main/java/io/druid/segment/CursorFactory.java b/processing/src/main/java/io/druid/segment/CursorFactory.java
index f8137d2fddca..b9e409f05651 100644
--- a/processing/src/main/java/io/druid/segment/CursorFactory.java
+++ b/processing/src/main/java/io/druid/segment/CursorFactory.java
@@ -22,7 +22,6 @@
 import io.druid.granularity.QueryGranularity;
 import io.druid.java.util.common.guava.Sequence;
 import io.druid.query.filter.Filter;
-import io.druid.segment.virtual.VirtualColumns;
 import org.joda.time.Interval;
 
 /**
diff --git a/processing/src/main/java/io/druid/segment/QueryableIndexStorageAdapter.java b/processing/src/main/java/io/druid/segment/QueryableIndexStorageAdapter.java
index b7fcd6c78c44..ca54f8add4e3 100644
--- a/processing/src/main/java/io/druid/segment/QueryableIndexStorageAdapter.java
+++ b/processing/src/main/java/io/druid/segment/QueryableIndexStorageAdapter.java
@@ -56,7 +56,6 @@
 import io.druid.segment.filter.AndFilter;
 import io.druid.segment.filter.BooleanValueMatcher;
 import io.druid.segment.filter.Filters;
-import io.druid.segment.virtual.VirtualColumns;
 import it.unimi.dsi.fastutil.ints.IntIterators;
 import org.joda.time.DateTime;
 import org.joda.time.Interval;
diff --git a/processing/src/main/java/io/druid/segment/virtual/VirtualColumn.java b/processing/src/main/java/io/druid/segment/VirtualColumn.java
similarity index 98%
rename from processing/src/main/java/io/druid/segment/virtual/VirtualColumn.java
rename to processing/src/main/java/io/druid/segment/VirtualColumn.java
index 56c7d21bba91..38c91d9bc1eb 100644
--- a/processing/src/main/java/io/druid/segment/virtual/VirtualColumn.java
+++ b/processing/src/main/java/io/druid/segment/VirtualColumn.java
@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package io.druid.segment.virtual;
+package io.druid.segment;
 
 import com.fasterxml.jackson.annotation.JsonSubTypes;
 import com.fasterxml.jackson.annotation.JsonTypeInfo;
@@ -28,6 +28,7 @@
 import io.druid.segment.LongColumnSelector;
 import io.druid.segment.ObjectColumnSelector;
 import io.druid.segment.column.ValueType;
+import io.druid.segment.virtual.ExpressionVirtualColumn;
 
 import java.util.List;
 
diff --git a/processing/src/main/java/io/druid/segment/virtual/VirtualColumns.java b/processing/src/main/java/io/druid/segment/VirtualColumns.java
similarity index 95%
rename from processing/src/main/java/io/druid/segment/virtual/VirtualColumns.java
rename to processing/src/main/java/io/druid/segment/VirtualColumns.java
index dc1a9e81d359..0f3a9e6934c5 100644
--- a/processing/src/main/java/io/druid/segment/virtual/VirtualColumns.java
+++ b/processing/src/main/java/io/druid/segment/VirtualColumns.java
@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package io.druid.segment.virtual;
+package io.druid.segment;
 
 import com.fasterxml.jackson.annotation.JsonCreator;
 import com.fasterxml.jackson.annotation.JsonValue;
@@ -31,16 +31,9 @@
 import io.druid.java.util.common.IAE;
 import io.druid.java.util.common.Pair;
 import io.druid.query.dimension.DimensionSpec;
-import io.druid.segment.ColumnSelectorFactory;
-import io.druid.segment.DimensionSelector;
-import io.druid.segment.FloatColumnSelector;
-import io.druid.segment.LongColumnSelector;
-import io.druid.segment.NullDimensionSelector;
-import io.druid.segment.ObjectColumnSelector;
-import io.druid.segment.ZeroFloatColumnSelector;
-import io.druid.segment.ZeroLongColumnSelector;
 import io.druid.segment.column.Column;
 import io.druid.segment.column.ValueType;
+import io.druid.segment.virtual.VirtualizedColumnSelectorFactory;
 
 import java.nio.ByteBuffer;
 import java.util.List;
diff --git a/processing/src/main/java/io/druid/segment/incremental/IncrementalIndex.java b/processing/src/main/java/io/druid/segment/incremental/IncrementalIndex.java
index dfe82e605e9b..0aef437c874e 100644
--- a/processing/src/main/java/io/druid/segment/incremental/IncrementalIndex.java
+++ b/processing/src/main/java/io/druid/segment/incremental/IncrementalIndex.java
@@ -61,7 +61,7 @@
 import io.druid.segment.serde.ComplexMetricExtractor;
 import io.druid.segment.serde.ComplexMetricSerde;
 import io.druid.segment.serde.ComplexMetrics;
-import io.druid.segment.virtual.VirtualColumns;
+import io.druid.segment.VirtualColumns;
 import it.unimi.dsi.fastutil.ints.IntIterator;
 import it.unimi.dsi.fastutil.ints.IntIterators;
 import org.joda.time.DateTime;
diff --git a/processing/src/main/java/io/druid/segment/incremental/IncrementalIndexSchema.java b/processing/src/main/java/io/druid/segment/incremental/IncrementalIndexSchema.java
index 3750629abfc8..ca26a95b9d2b 100644
--- a/processing/src/main/java/io/druid/segment/incremental/IncrementalIndexSchema.java
+++ b/processing/src/main/java/io/druid/segment/incremental/IncrementalIndexSchema.java
@@ -25,7 +25,7 @@
 import io.druid.granularity.QueryGranularities;
 import io.druid.granularity.QueryGranularity;
 import io.druid.query.aggregation.AggregatorFactory;
-import io.druid.segment.virtual.VirtualColumns;
+import io.druid.segment.VirtualColumns;
 
 /**
  */
diff --git a/processing/src/main/java/io/druid/segment/incremental/IncrementalIndexStorageAdapter.java b/processing/src/main/java/io/druid/segment/incremental/IncrementalIndexStorageAdapter.java
index c3f21d7bdcc8..bc984f5b4ebe 100644
--- a/processing/src/main/java/io/druid/segment/incremental/IncrementalIndexStorageAdapter.java
+++ b/processing/src/main/java/io/druid/segment/incremental/IncrementalIndexStorageAdapter.java
@@ -57,7 +57,7 @@
 import io.druid.segment.data.ListIndexed;
 import io.druid.segment.filter.BooleanValueMatcher;
 import io.druid.segment.filter.Filters;
-import io.druid.segment.virtual.VirtualColumns;
+import io.druid.segment.VirtualColumns;
 import org.joda.time.DateTime;
 import org.joda.time.Interval;
 
diff --git a/processing/src/main/java/io/druid/segment/virtual/ExpressionVirtualColumn.java b/processing/src/main/java/io/druid/segment/virtual/ExpressionVirtualColumn.java
index 7bcb27faad11..5ae7a8014511 100644
--- a/processing/src/main/java/io/druid/segment/virtual/ExpressionVirtualColumn.java
+++ b/processing/src/main/java/io/druid/segment/virtual/ExpressionVirtualColumn.java
@@ -31,6 +31,7 @@
 import io.druid.segment.FloatColumnSelector;
 import io.druid.segment.LongColumnSelector;
 import io.druid.segment.ObjectColumnSelector;
+import io.druid.segment.VirtualColumn;
 import io.druid.segment.column.ValueType;
 import org.apache.commons.codec.Charsets;
 
diff --git a/processing/src/main/java/io/druid/segment/virtual/VirtualizedColumnSelectorFactory.java b/processing/src/main/java/io/druid/segment/virtual/VirtualizedColumnSelectorFactory.java
index 3a4f4373e9de..1641968bbf3d 100644
--- a/processing/src/main/java/io/druid/segment/virtual/VirtualizedColumnSelectorFactory.java
+++ b/processing/src/main/java/io/druid/segment/virtual/VirtualizedColumnSelectorFactory.java
@@ -26,6 +26,7 @@
 import io.druid.segment.FloatColumnSelector;
 import io.druid.segment.LongColumnSelector;
 import io.druid.segment.ObjectColumnSelector;
+import io.druid.segment.VirtualColumns;
 import io.druid.segment.column.ValueType;
 
 public class VirtualizedColumnSelectorFactory implements ColumnSelectorFactory
diff --git a/processing/src/test/java/io/druid/query/timeseries/TimeseriesQueryQueryToolChestTest.java b/processing/src/test/java/io/druid/query/timeseries/TimeseriesQueryQueryToolChestTest.java
index ee0376f799b7..724b6e7fb4a7 100644
--- a/processing/src/test/java/io/druid/query/timeseries/TimeseriesQueryQueryToolChestTest.java
+++ b/processing/src/test/java/io/druid/query/timeseries/TimeseriesQueryQueryToolChestTest.java
@@ -31,7 +31,7 @@
 import io.druid.query.aggregation.AggregatorFactory;
 import io.druid.query.aggregation.CountAggregatorFactory;
 import io.druid.query.spec.MultipleIntervalSegmentSpec;
-import io.druid.segment.virtual.VirtualColumns;
+import io.druid.segment.VirtualColumns;
 import org.joda.time.DateTime;
 import org.joda.time.Interval;
 import org.junit.Assert;
diff --git a/processing/src/test/java/io/druid/query/topn/TopNQueryQueryToolChestTest.java b/processing/src/test/java/io/druid/query/topn/TopNQueryQueryToolChestTest.java
index 4e3838e44be1..393942767f4e 100644
--- a/processing/src/test/java/io/druid/query/topn/TopNQueryQueryToolChestTest.java
+++ b/processing/src/test/java/io/druid/query/topn/TopNQueryQueryToolChestTest.java
@@ -40,7 +40,7 @@
 import io.druid.query.spec.MultipleIntervalSegmentSpec;
 import io.druid.segment.IncrementalIndexSegment;
 import io.druid.segment.TestIndex;
-import io.druid.segment.virtual.VirtualColumns;
+import io.druid.segment.VirtualColumns;
 import org.joda.time.DateTime;
 import org.joda.time.Interval;
 import org.junit.Assert;
diff --git a/processing/src/test/java/io/druid/segment/TestIndex.java b/processing/src/test/java/io/druid/segment/TestIndex.java
index 7184d38b05a1..efc7a6405aeb 100644
--- a/processing/src/test/java/io/druid/segment/TestIndex.java
+++ b/processing/src/test/java/io/druid/segment/TestIndex.java
@@ -42,8 +42,6 @@
 import io.druid.segment.incremental.OnheapIncrementalIndex;
 import io.druid.segment.serde.ComplexMetrics;
 import io.druid.segment.virtual.ExpressionVirtualColumn;
-import io.druid.segment.virtual.VirtualColumn;
-import io.druid.segment.virtual.VirtualColumns;
 import org.joda.time.DateTime;
 import org.joda.time.Interval;
 
diff --git a/processing/src/test/java/io/druid/segment/filter/BaseFilterTest.java b/processing/src/test/java/io/druid/segment/filter/BaseFilterTest.java
index 13cb294acd95..0223b97b261e 100644
--- a/processing/src/test/java/io/druid/segment/filter/BaseFilterTest.java
+++ b/processing/src/test/java/io/druid/segment/filter/BaseFilterTest.java
@@ -56,7 +56,7 @@
 import io.druid.segment.data.RoaringBitmapSerdeFactory;
 import io.druid.segment.incremental.IncrementalIndex;
 import io.druid.segment.incremental.IncrementalIndexStorageAdapter;
-import io.druid.segment.virtual.VirtualColumns;
+import io.druid.segment.VirtualColumns;
 import org.joda.time.Interval;
 import org.junit.Assert;
 import org.junit.Before;
diff --git a/processing/src/test/java/io/druid/segment/incremental/IncrementalIndexMultiValueSpecTest.java b/processing/src/test/java/io/druid/segment/incremental/IncrementalIndexMultiValueSpecTest.java
index 3fc595c147b9..00d0e66a0995 100644
--- a/processing/src/test/java/io/druid/segment/incremental/IncrementalIndexMultiValueSpecTest.java
+++ b/processing/src/test/java/io/druid/segment/incremental/IncrementalIndexMultiValueSpecTest.java
@@ -28,7 +28,7 @@
 import io.druid.data.input.impl.TimestampSpec;
 import io.druid.granularity.QueryGranularities;
 import io.druid.query.aggregation.AggregatorFactory;
-import io.druid.segment.virtual.VirtualColumns;
+import io.druid.segment.VirtualColumns;
 import org.junit.Assert;
 import org.junit.Test;
 
diff --git a/processing/src/test/java/io/druid/segment/incremental/IncrementalIndexStorageAdapterTest.java b/processing/src/test/java/io/druid/segment/incremental/IncrementalIndexStorageAdapterTest.java
index ad0ece737cab..de26954395de 100644
--- a/processing/src/test/java/io/druid/segment/incremental/IncrementalIndexStorageAdapterTest.java
+++ b/processing/src/test/java/io/druid/segment/incremental/IncrementalIndexStorageAdapterTest.java
@@ -51,7 +51,7 @@
 import io.druid.segment.StorageAdapter;
 import io.druid.segment.data.IndexedInts;
 import io.druid.segment.filter.SelectorFilter;
-import io.druid.segment.virtual.VirtualColumns;
+import io.druid.segment.VirtualColumns;
 import org.joda.time.DateTime;
 import org.joda.time.Interval;
 import org.junit.Assert;
diff --git a/processing/src/test/java/io/druid/segment/virtual/VirtualColumnsTest.java b/processing/src/test/java/io/druid/segment/virtual/VirtualColumnsTest.java
index 99f427093f39..70a8d54620af 100644
--- a/processing/src/test/java/io/druid/segment/virtual/VirtualColumnsTest.java
+++ b/processing/src/test/java/io/druid/segment/virtual/VirtualColumnsTest.java
@@ -31,6 +31,8 @@
 import io.druid.segment.FloatColumnSelector;
 import io.druid.segment.LongColumnSelector;
 import io.druid.segment.ObjectColumnSelector;
+import io.druid.segment.VirtualColumn;
+import io.druid.segment.VirtualColumns;
 import io.druid.segment.column.ValueType;
 import io.druid.segment.data.IndexedInts;
 import it.unimi.dsi.fastutil.ints.IntIterator;
diff --git a/server/src/main/java/io/druid/segment/realtime/firehose/IngestSegmentFirehose.java b/server/src/main/java/io/druid/segment/realtime/firehose/IngestSegmentFirehose.java
index 317458a55ea1..30e7932d2dec 100644
--- a/server/src/main/java/io/druid/segment/realtime/firehose/IngestSegmentFirehose.java
+++ b/server/src/main/java/io/druid/segment/realtime/firehose/IngestSegmentFirehose.java
@@ -39,7 +39,7 @@
 import io.druid.segment.DimensionSelector;
 import io.druid.segment.LongColumnSelector;
 import io.druid.segment.ObjectColumnSelector;
-import io.druid.segment.virtual.VirtualColumns;
+import io.druid.segment.VirtualColumns;
 import io.druid.segment.column.Column;
 import io.druid.segment.data.IndexedInts;
 import io.druid.segment.filter.Filters;
diff --git a/services/src/main/java/io/druid/cli/DumpSegment.java b/services/src/main/java/io/druid/cli/DumpSegment.java
index 02506141d9ef..7f1d9f781fff 100644
--- a/services/src/main/java/io/druid/cli/DumpSegment.java
+++ b/services/src/main/java/io/druid/cli/DumpSegment.java
@@ -71,7 +71,7 @@
 import io.druid.segment.QueryableIndex;
 import io.druid.segment.QueryableIndexSegment;
 import io.druid.segment.QueryableIndexStorageAdapter;
-import io.druid.segment.virtual.VirtualColumns;
+import io.druid.segment.VirtualColumns;
 import io.druid.segment.column.BitmapIndex;
 import io.druid.segment.column.Column;
 import io.druid.segment.column.ColumnConfig;

From c0d6472c42bbddd5537831f912356d908b55b9d0 Mon Sep 17 00:00:00 2001
From: Gian Merlino 
Date: Wed, 7 Dec 2016 17:30:56 -0800
Subject: [PATCH 4/4] Fix some imports.

---
 processing/src/main/java/io/druid/segment/VirtualColumn.java | 5 -----
 .../segment/virtual/VirtualizedColumnSelectorFactory.java    | 2 +-
 2 files changed, 1 insertion(+), 6 deletions(-)

diff --git a/processing/src/main/java/io/druid/segment/VirtualColumn.java b/processing/src/main/java/io/druid/segment/VirtualColumn.java
index 38c91d9bc1eb..97160d99e9e1 100644
--- a/processing/src/main/java/io/druid/segment/VirtualColumn.java
+++ b/processing/src/main/java/io/druid/segment/VirtualColumn.java
@@ -22,11 +22,6 @@
 import com.fasterxml.jackson.annotation.JsonSubTypes;
 import com.fasterxml.jackson.annotation.JsonTypeInfo;
 import io.druid.query.dimension.DimensionSpec;
-import io.druid.segment.ColumnSelectorFactory;
-import io.druid.segment.DimensionSelector;
-import io.druid.segment.FloatColumnSelector;
-import io.druid.segment.LongColumnSelector;
-import io.druid.segment.ObjectColumnSelector;
 import io.druid.segment.column.ValueType;
 import io.druid.segment.virtual.ExpressionVirtualColumn;
 
diff --git a/processing/src/main/java/io/druid/segment/virtual/VirtualizedColumnSelectorFactory.java b/processing/src/main/java/io/druid/segment/virtual/VirtualizedColumnSelectorFactory.java
index 1641968bbf3d..967cd176cf78 100644
--- a/processing/src/main/java/io/druid/segment/virtual/VirtualizedColumnSelectorFactory.java
+++ b/processing/src/main/java/io/druid/segment/virtual/VirtualizedColumnSelectorFactory.java
@@ -34,7 +34,7 @@ public class VirtualizedColumnSelectorFactory implements ColumnSelectorFactory
   private final ColumnSelectorFactory baseFactory;
   private final VirtualColumns virtualColumns;
 
-  VirtualizedColumnSelectorFactory(
+  public VirtualizedColumnSelectorFactory(
       ColumnSelectorFactory baseFactory,
       VirtualColumns virtualColumns
   )