From c01b81e727d13ef7f0cdd90b790a7fcecba9e9c6 Mon Sep 17 00:00:00 2001 From: Gian Merlino Date: Fri, 6 Jan 2017 10:31:24 -0800 Subject: [PATCH 1/5] Add virtual column types, holder serde, and safety features. Virtual columns: - add long, float, dimension selectors - put cache IDs in VirtualColumnCacheHelper - adjust serde so VirtualColumns can be the holder object for Jackson - add fail-fast validation for cycle detection and duplicates - add expression virtual column in core Storage adapters: - move virtual column hooks before checking base columns, to prevent surprises when a new base column is added that happens to have the same name as a virtual column. --- .../main/java/io/druid/math/expr/Evals.java | 22 - .../io/druid/segment/MapVirtualColumn.java | 52 ++- .../java/io/druid/indexer/InputRowSerde.java | 2 + .../src/main/java/io/druid/query/Druids.java | 15 +- .../io/druid/query/select/SelectQuery.java | 10 +- .../druid/query/select/SelectQueryEngine.java | 3 +- .../select/SelectQueryQueryToolChest.java | 22 +- .../druid/segment/ColumnSelectorFactory.java | 25 +- .../druid/segment/NullDimensionSelector.java | 12 + .../segment/QueryableIndexStorageAdapter.java | 59 ++- .../java/io/druid/segment/VirtualColumn.java | 86 +++- .../java/io/druid/segment/VirtualColumns.java | 212 +++++++++- .../segment/ZeroFloatColumnSelector.java | 41 ++ .../druid/segment/ZeroLongColumnSelector.java | 41 ++ .../io/druid/segment/column/ValueType.java | 14 +- .../segment/incremental/IncrementalIndex.java | 29 +- .../incremental/IncrementalIndexSchema.java | 19 +- .../IncrementalIndexStorageAdapter.java | 59 ++- .../incremental/OffheapIncrementalIndex.java | 25 -- .../BaseSingleValueDimensionSelector.java | 53 +++ .../segment/virtual/ExpressionSelectors.java | 29 +- .../virtual/ExpressionVirtualColumn.java | 179 +++++++++ .../virtual/VirtualColumnCacheHelper.java | 34 ++ .../VirtualizedColumnSelectorFactory.java | 99 +++++ .../query/select/SelectQuerySpecTest.java | 2 +- .../segment/NullDimensionSelectorTest.java | 2 +- .../test/java/io/druid/segment/TestIndex.java | 9 +- .../IncrementalIndexMultiValueSpecTest.java | 2 + .../IncrementalIndexStorageAdapterTest.java | 4 +- .../virtual/ExpressionVirtualColumnTest.java | 96 +++++ .../segment/virtual/VirtualColumnsTest.java | 379 ++++++++++++++++++ 31 files changed, 1437 insertions(+), 199 deletions(-) create mode 100644 processing/src/main/java/io/druid/segment/ZeroFloatColumnSelector.java create mode 100644 processing/src/main/java/io/druid/segment/ZeroLongColumnSelector.java create mode 100644 processing/src/main/java/io/druid/segment/virtual/BaseSingleValueDimensionSelector.java create mode 100644 processing/src/main/java/io/druid/segment/virtual/ExpressionVirtualColumn.java create mode 100644 processing/src/main/java/io/druid/segment/virtual/VirtualColumnCacheHelper.java create mode 100644 processing/src/main/java/io/druid/segment/virtual/VirtualizedColumnSelectorFactory.java create mode 100644 processing/src/test/java/io/druid/segment/virtual/ExpressionVirtualColumnTest.java create mode 100644 processing/src/test/java/io/druid/segment/virtual/VirtualColumnsTest.java diff --git a/common/src/main/java/io/druid/math/expr/Evals.java b/common/src/main/java/io/druid/math/expr/Evals.java index 702037309a91..1ff14385738b 100644 --- a/common/src/main/java/io/druid/math/expr/Evals.java +++ b/common/src/main/java/io/druid/math/expr/Evals.java @@ -20,7 +20,6 @@ package io.druid.math.expr; import com.google.common.base.Strings; -import io.druid.common.guava.GuavaUtils; import io.druid.java.util.common.logger.Logger; import java.util.Arrays; @@ -32,27 +31,6 @@ public class Evals { private static final Logger log = new Logger(Evals.class); - public static Number toNumber(Object value) - { - if (value == null) { - return 0L; - } - if (value instanceof Number) { - return (Number) value; - } - String stringValue = String.valueOf(value); - Long longValue = GuavaUtils.tryParseLong(stringValue); - if (longValue == null) { - return Double.valueOf(stringValue); - } - return longValue; - } - - public static boolean isConstant(Expr expr) - { - return expr instanceof ConstantExpr; - } - public static boolean isAllConstants(Expr... exprs) { return isAllConstants(Arrays.asList(exprs)); diff --git a/extensions-contrib/virtual-columns/src/main/java/io/druid/segment/MapVirtualColumn.java b/extensions-contrib/virtual-columns/src/main/java/io/druid/segment/MapVirtualColumn.java index 75fe13ad6c82..13a1ff7034d4 100644 --- a/extensions-contrib/virtual-columns/src/main/java/io/druid/segment/MapVirtualColumn.java +++ b/extensions-contrib/virtual-columns/src/main/java/io/druid/segment/MapVirtualColumn.java @@ -22,21 +22,26 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; import com.google.common.collect.Maps; import com.metamx.common.StringUtils; import io.druid.query.dimension.DefaultDimensionSpec; +import io.druid.query.dimension.DimensionSpec; import io.druid.query.filter.DimFilterUtils; +import io.druid.segment.column.ColumnCapabilities; +import io.druid.segment.column.ColumnCapabilitiesImpl; +import io.druid.segment.column.ValueType; import io.druid.segment.data.IndexedInts; +import io.druid.segment.virtual.VirtualColumnCacheHelper; import java.nio.ByteBuffer; +import java.util.List; import java.util.Map; /** */ public class MapVirtualColumn implements VirtualColumn { - private static final byte VC_TYPE_ID = 0x00; - private final String outputName; private final String keyDimension; private final String valueDimension; @@ -58,13 +63,14 @@ public MapVirtualColumn( } @Override - public ObjectColumnSelector init(String dimension, ColumnSelectorFactory factory) + public ObjectColumnSelector makeObjectColumnSelector(String dimension, ColumnSelectorFactory factory) { final DimensionSelector keySelector = factory.makeDimensionSelector(DefaultDimensionSpec.of(keyDimension)); final DimensionSelector valueSelector = factory.makeDimensionSelector(DefaultDimensionSpec.of(valueDimension)); - int index = dimension.indexOf('.'); - if (index < 0) { + final String subColumnName = VirtualColumns.splitColumnName(dimension).rhs; + + if (subColumnName == null) { return new ObjectColumnSelector() { @Override @@ -94,7 +100,7 @@ public Map get() }; } - final int keyId = keySelector.lookupId(dimension.substring(index + 1)); + final int keyId = keySelector.lookupId(subColumnName); return new ObjectColumnSelector() { @@ -123,6 +129,38 @@ public String get() }; } + @Override + public DimensionSelector makeDimensionSelector(DimensionSpec dimensionSpec, ColumnSelectorFactory factory) + { + // Could probably do something useful here if the column name is dot-style. But for now just return nothing. + return null; + } + + @Override + public FloatColumnSelector makeFloatColumnSelector(String columnName, ColumnSelectorFactory factory) + { + return null; + } + + @Override + public LongColumnSelector makeLongColumnSelector(String columnName, ColumnSelectorFactory factory) + { + return null; + } + + @Override + public ColumnCapabilities capabilities(String columnName) + { + final ValueType valueType = columnName.indexOf('.') < 0 ? ValueType.COMPLEX : ValueType.STRING; + return new ColumnCapabilitiesImpl().setType(valueType); + } + + @Override + public List requiredColumns() + { + return ImmutableList.of(keyDimension, valueDimension); + } + @Override public boolean usesDotNotation() { @@ -137,7 +175,7 @@ public byte[] getCacheKey() byte[] output = StringUtils.toUtf8(outputName); return ByteBuffer.allocate(3 + key.length + value.length + output.length) - .put(VC_TYPE_ID) + .put(VirtualColumnCacheHelper.CACHE_TYPE_ID_MAP) .put(key).put(DimFilterUtils.STRING_SEPARATOR) .put(value).put(DimFilterUtils.STRING_SEPARATOR) .put(output) diff --git a/indexing-hadoop/src/main/java/io/druid/indexer/InputRowSerde.java b/indexing-hadoop/src/main/java/io/druid/indexer/InputRowSerde.java index a8b25460b1c4..937c0db262db 100644 --- a/indexing-hadoop/src/main/java/io/druid/indexer/InputRowSerde.java +++ b/indexing-hadoop/src/main/java/io/druid/indexer/InputRowSerde.java @@ -36,6 +36,7 @@ import io.druid.segment.incremental.IncrementalIndex; import io.druid.segment.serde.ComplexMetricSerde; import io.druid.segment.serde.ComplexMetrics; +import io.druid.segment.VirtualColumns; import org.apache.hadoop.io.ArrayWritable; import org.apache.hadoop.io.Text; import org.apache.hadoop.io.WritableUtils; @@ -87,6 +88,7 @@ public InputRow get() Aggregator agg = aggFactory.factorize( IncrementalIndex.makeColumnSelectorFactory( + VirtualColumns.EMPTY, aggFactory, supplier, true diff --git a/processing/src/main/java/io/druid/query/Druids.java b/processing/src/main/java/io/druid/query/Druids.java index dfa7d238fb50..b9f9db4b8a04 100644 --- a/processing/src/main/java/io/druid/query/Druids.java +++ b/processing/src/main/java/io/druid/query/Druids.java @@ -55,6 +55,7 @@ import io.druid.query.timeboundary.TimeBoundaryResultValue; import io.druid.query.timeseries.TimeseriesQuery; import io.druid.segment.VirtualColumn; +import io.druid.segment.VirtualColumns; import org.joda.time.DateTime; import org.joda.time.Interval; @@ -1104,7 +1105,7 @@ public static class SelectQueryBuilder private QueryGranularity granularity; private List dimensions; private List metrics; - private List virtualColumns; + private VirtualColumns virtualColumns; private PagingSpec pagingSpec; public SelectQueryBuilder() @@ -1233,12 +1234,22 @@ public SelectQueryBuilder metrics(List m) return this; } - public SelectQueryBuilder virtualColumns(List vcs) + public SelectQueryBuilder virtualColumns(VirtualColumns vcs) { virtualColumns = vcs; return this; } + public SelectQueryBuilder virtualColumns(List vcs) + { + return virtualColumns(VirtualColumns.create(vcs)); + } + + public SelectQueryBuilder virtualColumns(VirtualColumn... vcs) + { + return virtualColumns(VirtualColumns.create(Arrays.asList(vcs))); + } + public SelectQueryBuilder pagingSpec(PagingSpec p) { pagingSpec = p; diff --git a/processing/src/main/java/io/druid/query/select/SelectQuery.java b/processing/src/main/java/io/druid/query/select/SelectQuery.java index 4dd27a593083..d33ff67fe7d8 100644 --- a/processing/src/main/java/io/druid/query/select/SelectQuery.java +++ b/processing/src/main/java/io/druid/query/select/SelectQuery.java @@ -31,7 +31,7 @@ import io.druid.query.dimension.DimensionSpec; import io.druid.query.filter.DimFilter; import io.druid.query.spec.QuerySegmentSpec; -import io.druid.segment.VirtualColumn; +import io.druid.segment.VirtualColumns; import java.util.List; import java.util.Map; @@ -46,7 +46,7 @@ public class SelectQuery extends BaseQuery> private final QueryGranularity granularity; private final List dimensions; private final List metrics; - private final List virtualColumns; + private final VirtualColumns virtualColumns; private final PagingSpec pagingSpec; @JsonCreator @@ -58,7 +58,7 @@ public SelectQuery( @JsonProperty("granularity") QueryGranularity granularity, @JsonProperty("dimensions") List dimensions, @JsonProperty("metrics") List metrics, - @JsonProperty("virtualColumns") List virtualColumns, + @JsonProperty("virtualColumns") VirtualColumns virtualColumns, @JsonProperty("pagingSpec") PagingSpec pagingSpec, @JsonProperty("context") Map context ) @@ -67,7 +67,7 @@ public SelectQuery( this.dimFilter = dimFilter; this.granularity = granularity; this.dimensions = dimensions; - this.virtualColumns = virtualColumns; + this.virtualColumns = virtualColumns == null ? VirtualColumns.EMPTY : virtualColumns; this.metrics = metrics; this.pagingSpec = pagingSpec; @@ -134,7 +134,7 @@ public List getMetrics() } @JsonProperty - public List getVirtualColumns() + public VirtualColumns getVirtualColumns() { return virtualColumns; } diff --git a/processing/src/main/java/io/druid/query/select/SelectQueryEngine.java b/processing/src/main/java/io/druid/query/select/SelectQueryEngine.java index 411e5db119c6..ad675ecc3ba2 100644 --- a/processing/src/main/java/io/druid/query/select/SelectQueryEngine.java +++ b/processing/src/main/java/io/druid/query/select/SelectQueryEngine.java @@ -43,7 +43,6 @@ import io.druid.segment.ObjectColumnSelector; import io.druid.segment.Segment; import io.druid.segment.StorageAdapter; -import io.druid.segment.VirtualColumns; import io.druid.segment.column.Column; import io.druid.segment.column.ColumnCapabilities; import io.druid.segment.column.ValueType; @@ -161,7 +160,7 @@ public Sequence> process(final SelectQuery query, fina adapter, query.getQuerySegmentSpec().getIntervals(), filter, - VirtualColumns.valueOf(query.getVirtualColumns()), + query.getVirtualColumns(), query.isDescending(), query.getGranularity(), new Function>() diff --git a/processing/src/main/java/io/druid/query/select/SelectQueryQueryToolChest.java b/processing/src/main/java/io/druid/query/select/SelectQueryQueryToolChest.java index 99ff72251fd4..2bb009835e2d 100644 --- a/processing/src/main/java/io/druid/query/select/SelectQueryQueryToolChest.java +++ b/processing/src/main/java/io/druid/query/select/SelectQueryQueryToolChest.java @@ -49,7 +49,6 @@ import io.druid.query.dimension.DimensionSpec; import io.druid.query.filter.DimFilter; import io.druid.timeline.DataSegmentUtils; -import io.druid.segment.VirtualColumn; import io.druid.timeline.LogicalSegment; import org.joda.time.DateTime; import org.joda.time.Interval; @@ -192,20 +191,7 @@ public byte[] computeCacheKey(SelectQuery query) ++index; } - List virtualColumns = query.getVirtualColumns(); - if (virtualColumns == null) { - virtualColumns = Collections.emptyList(); - } - - final byte[][] virtualColumnsBytes = new byte[virtualColumns.size()][]; - int virtualColumnsBytesSize = 0; - index = 0; - for (VirtualColumn vc : virtualColumns) { - virtualColumnsBytes[index] = vc.getCacheKey(); - virtualColumnsBytesSize += virtualColumnsBytes[index].length; - ++index; - } - + final byte[] virtualColumnsCacheKey = query.getVirtualColumns().getCacheKey(); final ByteBuffer queryCacheKey = ByteBuffer .allocate( 1 @@ -214,7 +200,7 @@ public byte[] computeCacheKey(SelectQuery query) + query.getPagingSpec().getCacheKey().length + dimensionsBytesSize + metricBytesSize - + virtualColumnsBytesSize + + virtualColumnsCacheKey.length ) .put(SELECT_QUERY) .put(granularityBytes) @@ -229,9 +215,7 @@ public byte[] computeCacheKey(SelectQuery query) queryCacheKey.put(metricByte); } - for (byte[] vcByte : virtualColumnsBytes) { - queryCacheKey.put(vcByte); - } + queryCacheKey.put(virtualColumnsCacheKey); return queryCacheKey.array(); } diff --git a/processing/src/main/java/io/druid/segment/ColumnSelectorFactory.java b/processing/src/main/java/io/druid/segment/ColumnSelectorFactory.java index f550fef14e5c..a13afabc253c 100644 --- a/processing/src/main/java/io/druid/segment/ColumnSelectorFactory.java +++ b/processing/src/main/java/io/druid/segment/ColumnSelectorFactory.java @@ -22,14 +22,29 @@ import io.druid.query.dimension.DimensionSpec; import io.druid.segment.column.ColumnCapabilities; +import javax.annotation.Nullable; + /** * Factory class for MetricSelectors */ public interface ColumnSelectorFactory { - public DimensionSelector makeDimensionSelector(DimensionSpec dimensionSpec); - public FloatColumnSelector makeFloatColumnSelector(String columnName); - public LongColumnSelector makeLongColumnSelector(String columnName); - public ObjectColumnSelector makeObjectColumnSelector(String columnName); - public ColumnCapabilities getColumnCapabilities(String columnName); + DimensionSelector makeDimensionSelector(DimensionSpec dimensionSpec); + FloatColumnSelector makeFloatColumnSelector(String columnName); + LongColumnSelector makeLongColumnSelector(String columnName); + + @Nullable + ObjectColumnSelector makeObjectColumnSelector(String columnName); + + /** + * Returns capabilities of a particular column, if known. May be null if the column doesn't exist, or if + * the column does exist but the capabilities are unknown. The latter is possible with dynamically discovered + * columns. + * + * @param column column name + * + * @return capabilities, or null + */ + @Nullable + ColumnCapabilities getColumnCapabilities(String column); } diff --git a/processing/src/main/java/io/druid/segment/NullDimensionSelector.java b/processing/src/main/java/io/druid/segment/NullDimensionSelector.java index f5cbe064686e..dced44f9500a 100644 --- a/processing/src/main/java/io/druid/segment/NullDimensionSelector.java +++ b/processing/src/main/java/io/druid/segment/NullDimensionSelector.java @@ -25,6 +25,18 @@ public class NullDimensionSelector implements DimensionSelector { + private static final NullDimensionSelector INSTANCE = new NullDimensionSelector(); + + private NullDimensionSelector() + { + // Singleton. + } + + public static NullDimensionSelector instance() + { + return INSTANCE; + } + @Override public IndexedInts getRow() { diff --git a/processing/src/main/java/io/druid/segment/QueryableIndexStorageAdapter.java b/processing/src/main/java/io/druid/segment/QueryableIndexStorageAdapter.java index 4750ba771dad..0c175fa7c449 100644 --- a/processing/src/main/java/io/druid/segment/QueryableIndexStorageAdapter.java +++ b/processing/src/main/java/io/druid/segment/QueryableIndexStorageAdapter.java @@ -40,7 +40,6 @@ import io.druid.segment.column.BitmapIndex; import io.druid.segment.column.Column; import io.druid.segment.column.ColumnCapabilities; -import io.druid.segment.column.ColumnCapabilitiesImpl; import io.druid.segment.column.ComplexColumn; import io.druid.segment.column.DictionaryEncodedColumn; import io.druid.segment.column.GenericColumn; @@ -55,7 +54,6 @@ import org.joda.time.Interval; import org.roaringbitmap.IntIterator; -import javax.annotation.Nullable; import java.io.Closeable; import java.io.IOException; import java.util.ArrayList; @@ -66,8 +64,6 @@ */ public class QueryableIndexStorageAdapter implements StorageAdapter { - private static final NullDimensionSelector NULL_DIMENSION_SELECTOR = new NullDimensionSelector(); - private final QueryableIndex index; public QueryableIndexStorageAdapter( @@ -428,6 +424,10 @@ public DimensionSelector makeDimensionSelector( DimensionSpec dimensionSpec ) { + if (virtualColumns.exists(dimensionSpec.getDimension())) { + return virtualColumns.makeDimensionSelector(dimensionSpec, this); + } + return dimensionSpec.decorate(makeDimensionSelectorUndecorated(dimensionSpec)); } @@ -440,7 +440,7 @@ private DimensionSelector makeDimensionSelectorUndecorated( final Column columnDesc = index.getColumn(dimension); if (columnDesc == null) { - return NULL_DIMENSION_SELECTOR; + return NullDimensionSelector.instance(); } if (dimension.equals(Column.TIME_COLUMN_NAME)) { @@ -461,7 +461,7 @@ private DimensionSelector makeDimensionSelectorUndecorated( final DictionaryEncodedColumn column = cachedColumn; if (column == null) { - return NULL_DIMENSION_SELECTOR; + return NullDimensionSelector.instance(); } else if (columnDesc.getCapabilities().hasMultipleValues()) { return new DimensionSelector() { @@ -569,6 +569,10 @@ public int lookupId(String name) @Override public FloatColumnSelector makeFloatColumnSelector(String columnName) { + if (virtualColumns.exists(columnName)) { + return virtualColumns.makeFloatColumnSelector(columnName, this); + } + GenericColumn cachedMetricVals = genericColumnCache.get(columnName); if (cachedMetricVals == null) { @@ -582,14 +586,7 @@ public FloatColumnSelector makeFloatColumnSelector(String columnName) } if (cachedMetricVals == null) { - return new FloatColumnSelector() - { - @Override - public float get() - { - return 0.0f; - } - }; + return ZeroFloatColumnSelector.instance(); } final GenericColumn metricVals = cachedMetricVals; @@ -606,6 +603,10 @@ public float get() @Override public LongColumnSelector makeLongColumnSelector(String columnName) { + if (virtualColumns.exists(columnName)) { + return virtualColumns.makeLongColumnSelector(columnName, this); + } + GenericColumn cachedMetricVals = genericColumnCache.get(columnName); if (cachedMetricVals == null) { @@ -619,14 +620,7 @@ public LongColumnSelector makeLongColumnSelector(String columnName) } if (cachedMetricVals == null) { - return new LongColumnSelector() - { - @Override - public long get() - { - return 0L; - } - }; + return ZeroLongColumnSelector.instance(); } final GenericColumn metricVals = cachedMetricVals; @@ -644,6 +638,10 @@ public long get() @Override public ObjectColumnSelector makeObjectColumnSelector(String column) { + if (virtualColumns.exists(column)) { + return virtualColumns.makeObjectColumnSelector(column, this); + } + Object cachedColumnVals = objectColumnCache.get(column); if (cachedColumnVals == null) { @@ -668,10 +666,6 @@ public ObjectColumnSelector makeObjectColumnSelector(String column) } if (cachedColumnVals == null) { - VirtualColumn vc = virtualColumns.getVirtualColumn(column); - if (vc != null) { - return vc.init(column, this); - } return null; } @@ -798,19 +792,14 @@ public Object get() }; } - @Nullable @Override public ColumnCapabilities getColumnCapabilities(String columnName) { - ColumnCapabilities capabilities = getColumnCapabilites(index, columnName); - if (capabilities == null && !virtualColumns.isEmpty()) { - VirtualColumn virtualColumn = virtualColumns.getVirtualColumn(columnName); - if (virtualColumn != null) { - Class clazz = virtualColumn.init(columnName, this).classOfObject(); - capabilities = new ColumnCapabilitiesImpl().setType(ValueType.typeFor(clazz)); - } + if (virtualColumns.exists(columnName)) { + return virtualColumns.getColumnCapabilities(columnName); } - return capabilities; + + return getColumnCapabilites(index, columnName); } } diff --git a/processing/src/main/java/io/druid/segment/VirtualColumn.java b/processing/src/main/java/io/druid/segment/VirtualColumn.java index 855affe8bf4f..beef428fef41 100644 --- a/processing/src/main/java/io/druid/segment/VirtualColumn.java +++ b/processing/src/main/java/io/druid/segment/VirtualColumn.java @@ -19,15 +19,26 @@ package io.druid.segment; +import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonTypeInfo; +import io.druid.query.dimension.DimensionSpec; +import io.druid.segment.column.ColumnCapabilities; +import io.druid.segment.virtual.ExpressionVirtualColumn; + +import javax.annotation.Nullable; +import java.util.List; -/** - */ -@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type") /** * Virtual columns are "views" created over a ColumnSelectorFactory. They can potentially draw from multiple * underlying columns, although they always present themselves as if they were a single column. + * + * A virtual column object will be shared amongst threads and must be thread safe. The selectors returned + * from the various makeXXXSelector methods need not be thread safe. */ +@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type") +@JsonSubTypes(value = { + @JsonSubTypes.Type(name = "expression", value = ExpressionVirtualColumn.class) +}) public interface VirtualColumn { /** @@ -42,10 +53,73 @@ public interface VirtualColumn * virtual column was referenced with, which is useful if this column uses dot notation. * * @param columnName the name this virtual column was referenced with - * @param factory column selector factory - * @return the selector + * @param factory column selector factory + * + * @return the selector, must not be null + */ + ObjectColumnSelector makeObjectColumnSelector(String columnName, ColumnSelectorFactory factory); + + /** + * Build a selector corresponding to this virtual column. Also provides the name that the + * virtual column was referenced with (through {@link DimensionSpec#getDimension()}, which + * is useful if this column uses dot notation. The virtual column is expected to apply any + * necessary decoration from the dimensionSpec. + * + * @param dimensionSpec the dimensionSpec this column was referenced with + * @param factory column selector factory + * + * @return the selector, or null if we can't make a selector + */ + @Nullable + DimensionSelector makeDimensionSelector(DimensionSpec dimensionSpec, ColumnSelectorFactory factory); + + /** + * Build a selector corresponding to this virtual column. Also provides the name that the + * virtual column was referenced with, which is useful if this column uses dot notation. + * + * @param columnName the name this virtual column was referenced with + * @param factory column selector factory + * + * @return the selector, or null if we can't make a selector + */ + @Nullable + FloatColumnSelector makeFloatColumnSelector(String columnName, ColumnSelectorFactory factory); + + /** + * Build a selector corresponding to this virtual column. Also provides the name that the + * virtual column was referenced with, which is useful if this column uses dot notation. + * + * @param columnName the name this virtual column was referenced with + * @param factory column selector factory + * + * @return the selector, or null if we can't make a selector + */ + @Nullable + LongColumnSelector makeLongColumnSelector(String columnName, ColumnSelectorFactory factory); + + /** + * Returns the capabilities of this virtual column, which includes a type that should match + * the type returned by "makeObjectColumnSelector" and should correspond to the best + * performing selector. May vary based on columnName if this column uses dot notation. + * + * @param columnName the name this virtual column was referenced with + * + * @return capabilities, must not be null + */ + ColumnCapabilities capabilities(String columnName); + + /** + * Returns a list of columns that this virtual column will access. This may include the + * names of other virtual columns. May be empty if a virtual column doesn't access any + * underlying columns. + * + * Does not pass columnName because there is an assumption that the list of columns + * needed by a dot-notation supporting virtual column will not vary based on the + * columnName. + * + * @return column names */ - ObjectColumnSelector init(String columnName, ColumnSelectorFactory factory); + List requiredColumns(); /** * Indicates that this virtual column can be referenced with dot notation. For example, diff --git a/processing/src/main/java/io/druid/segment/VirtualColumns.java b/processing/src/main/java/io/druid/segment/VirtualColumns.java index 799a0ca366e7..02482eb8c0b4 100644 --- a/processing/src/main/java/io/druid/segment/VirtualColumns.java +++ b/processing/src/main/java/io/druid/segment/VirtualColumns.java @@ -19,62 +19,250 @@ package io.druid.segment; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonValue; +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; import com.google.common.collect.Maps; +import com.google.common.collect.Sets; +import com.google.common.primitives.Ints; +import io.druid.java.util.common.IAE; +import io.druid.java.util.common.Pair; +import io.druid.query.dimension.DimensionSpec; +import io.druid.segment.column.Column; +import io.druid.segment.column.ColumnCapabilities; +import io.druid.segment.virtual.VirtualizedColumnSelectorFactory; +import java.nio.ByteBuffer; import java.util.List; import java.util.Map; +import java.util.Set; /** + * Class allowing lookup and usage of virtual columns. */ public class VirtualColumns { public static final VirtualColumns EMPTY = new VirtualColumns( - ImmutableMap.of(), ImmutableMap.of() + ImmutableList.of(), + ImmutableMap.of(), + ImmutableMap.of() ); - public static VirtualColumns valueOf(List virtualColumns) { + /** + * Split a dot-style columnName into the "main" columnName and the subColumn name after the dot. Useful for + * columns that support dot notation. + * + * @param columnName columnName like "foo" or "foo.bar" + * + * @return pair of main column name (will not be null) and subColumn name (may be null) + */ + public static Pair splitColumnName(String columnName) + { + final int i = columnName.indexOf('.'); + if (i < 0) { + return Pair.of(columnName, null); + } else { + return Pair.of(columnName.substring(0, i), columnName.substring(i + 1)); + } + } + + @JsonCreator + public static VirtualColumns create(List virtualColumns) + { if (virtualColumns == null || virtualColumns.isEmpty()) { return EMPTY; } Map withDotSupport = Maps.newHashMap(); Map withoutDotSupport = Maps.newHashMap(); for (VirtualColumn vc : virtualColumns) { + if (vc.getOutputName().equals(Column.TIME_COLUMN_NAME)) { + throw new IAE("virtualColumn name[%s] not allowed", vc.getOutputName()); + } + + if (withDotSupport.containsKey(vc.getOutputName()) || withoutDotSupport.containsKey(vc.getOutputName())) { + throw new IAE("Duplicate virtualColumn name[%s]", vc.getOutputName()); + } + if (vc.usesDotNotation()) { withDotSupport.put(vc.getOutputName(), vc); } else { withoutDotSupport.put(vc.getOutputName(), vc); } } - return new VirtualColumns(withDotSupport, withoutDotSupport); + return new VirtualColumns(ImmutableList.copyOf(virtualColumns), withDotSupport, withoutDotSupport); } - public VirtualColumns(Map withDotSupport, Map withoutDotSupport) + private VirtualColumns( + List virtualColumns, + Map withDotSupport, + Map withoutDotSupport + ) { + this.virtualColumns = virtualColumns; this.withDotSupport = withDotSupport; this.withoutDotSupport = withoutDotSupport; + + for (VirtualColumn virtualColumn : virtualColumns) { + detectCycles(virtualColumn, null); + } } + // For equals, hashCode, toString, and serialization: + private final List virtualColumns; + + // For getVirtualColumn: private final Map withDotSupport; private final Map withoutDotSupport; - public VirtualColumn getVirtualColumn(String dimension) + public boolean exists(String columnName) + { + return getVirtualColumn(columnName) != null; + } + + public VirtualColumn getVirtualColumn(String columnName) { - VirtualColumn vc = withoutDotSupport.get(dimension); + final VirtualColumn vc = withoutDotSupport.get(columnName); if (vc != null) { return vc; } - for (int index = dimension.indexOf('.'); index >= 0; index = dimension.indexOf('.', index + 1)) { - vc = withDotSupport.get(dimension.substring(0, index)); - if (vc != null) { - return vc; - } + final String baseColumnName = splitColumnName(columnName).lhs; + return withDotSupport.get(baseColumnName); + } + + public ObjectColumnSelector makeObjectColumnSelector(String columnName, ColumnSelectorFactory factory) + { + final VirtualColumn virtualColumn = getVirtualColumn(columnName); + if (virtualColumn == null) { + return null; + } else { + return Preconditions.checkNotNull( + virtualColumn.makeObjectColumnSelector(columnName, factory), + "VirtualColumn[%s] returned a null ObjectColumnSelector for columnName[%s]", + virtualColumn.getOutputName(), + columnName + ); + } + } + + public DimensionSelector makeDimensionSelector(DimensionSpec dimensionSpec, ColumnSelectorFactory factory) + { + final VirtualColumn virtualColumn = getVirtualColumn(dimensionSpec.getDimension()); + if (virtualColumn == null) { + return dimensionSpec.decorate(NullDimensionSelector.instance()); + } else { + final DimensionSelector selector = virtualColumn.makeDimensionSelector(dimensionSpec, factory); + return selector == null ? dimensionSpec.decorate(NullDimensionSelector.instance()) : selector; + } + } + + public FloatColumnSelector makeFloatColumnSelector(String columnName, ColumnSelectorFactory factory) + { + final VirtualColumn virtualColumn = getVirtualColumn(columnName); + if (virtualColumn == null) { + return ZeroFloatColumnSelector.instance(); + } else { + final FloatColumnSelector selector = virtualColumn.makeFloatColumnSelector(columnName, factory); + return selector == null ? ZeroFloatColumnSelector.instance() : selector; + } + } + + public LongColumnSelector makeLongColumnSelector(String columnName, ColumnSelectorFactory factory) + { + final VirtualColumn virtualColumn = getVirtualColumn(columnName); + if (virtualColumn == null) { + return ZeroLongColumnSelector.instance(); + } else { + final LongColumnSelector selector = virtualColumn.makeLongColumnSelector(columnName, factory); + return selector == null ? ZeroLongColumnSelector.instance() : selector; } - return withDotSupport.get(dimension); + } + + public ColumnCapabilities getColumnCapabilities(String columnName) + { + final VirtualColumn virtualColumn = getVirtualColumn(columnName); + return virtualColumn != null + ? Preconditions.checkNotNull(virtualColumn.capabilities(columnName), "capabilities for column[%s]", columnName) + : null; } public boolean isEmpty() { return withDotSupport.isEmpty() && withoutDotSupport.isEmpty(); } + + @JsonValue + public VirtualColumn[] getVirtualColumns() + { + // VirtualColumn[] instead of List to aid Jackson serialization. + return virtualColumns.toArray(new VirtualColumn[]{}); + } + + public ColumnSelectorFactory wrap(final ColumnSelectorFactory baseFactory) + { + return new VirtualizedColumnSelectorFactory(baseFactory, this); + } + + public byte[] getCacheKey() + { + final byte[][] cacheKeys = new byte[virtualColumns.size()][]; + int len = Ints.BYTES; + for (int i = 0; i < virtualColumns.size(); i++) { + cacheKeys[i] = virtualColumns.get(i).getCacheKey(); + len += Ints.BYTES + cacheKeys[i].length; + } + final ByteBuffer buf = ByteBuffer.allocate(len).putInt(virtualColumns.size()); + for (byte[] cacheKey : cacheKeys) { + buf.putInt(cacheKey.length); + buf.put(cacheKey); + } + return buf.array(); + } + + private void detectCycles(VirtualColumn virtualColumn, ImmutableSet columnNames) + { + Set nextSet = columnNames == null + ? Sets.newHashSet(virtualColumn.getOutputName()) + : Sets.newHashSet(columnNames); + + for (String columnName : virtualColumn.requiredColumns()) { + if (!nextSet.add(columnName)) { + throw new IAE("Self-referential column[%s]", columnName); + } + + final VirtualColumn dependency = getVirtualColumn(columnName); + if (dependency != null) { + detectCycles(dependency, ImmutableSet.copyOf(nextSet)); + } + } + } + + @Override + public boolean equals(Object o) + { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + + VirtualColumns that = (VirtualColumns) o; + + return virtualColumns.equals(that.virtualColumns); + } + + @Override + public int hashCode() + { + return virtualColumns.hashCode(); + } + + @Override + public String toString() + { + return virtualColumns.toString(); + } } diff --git a/processing/src/main/java/io/druid/segment/ZeroFloatColumnSelector.java b/processing/src/main/java/io/druid/segment/ZeroFloatColumnSelector.java new file mode 100644 index 000000000000..8e8eb99a4f8b --- /dev/null +++ b/processing/src/main/java/io/druid/segment/ZeroFloatColumnSelector.java @@ -0,0 +1,41 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.segment; + +public class ZeroFloatColumnSelector implements FloatColumnSelector +{ + private static final ZeroFloatColumnSelector INSTANCE = new ZeroFloatColumnSelector(); + + private ZeroFloatColumnSelector() + { + // No instantiation. + } + + public static ZeroFloatColumnSelector instance() + { + return INSTANCE; + } + + @Override + public float get() + { + return 0.0f; + } +} diff --git a/processing/src/main/java/io/druid/segment/ZeroLongColumnSelector.java b/processing/src/main/java/io/druid/segment/ZeroLongColumnSelector.java new file mode 100644 index 000000000000..a24f16111c9f --- /dev/null +++ b/processing/src/main/java/io/druid/segment/ZeroLongColumnSelector.java @@ -0,0 +1,41 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.segment; + +public class ZeroLongColumnSelector implements LongColumnSelector +{ + private static final ZeroLongColumnSelector INSTANCE = new ZeroLongColumnSelector(); + + private ZeroLongColumnSelector() + { + // No instantiation. + } + + public static ZeroLongColumnSelector instance() + { + return INSTANCE; + } + + @Override + public long get() + { + return 0; + } +} diff --git a/processing/src/main/java/io/druid/segment/column/ValueType.java b/processing/src/main/java/io/druid/segment/column/ValueType.java index 1760fea8c725..aa9919a129b9 100644 --- a/processing/src/main/java/io/druid/segment/column/ValueType.java +++ b/processing/src/main/java/io/druid/segment/column/ValueType.java @@ -26,17 +26,5 @@ public enum ValueType FLOAT, LONG, STRING, - COMPLEX; - - public static ValueType typeFor(Class clazz) - { - if (clazz == String.class) { - return STRING; - } else if (clazz == float.class || clazz == Float.TYPE) { - return FLOAT; - } else if (clazz == long.class || clazz == Long.TYPE) { - return LONG; - } - return COMPLEX; - } + COMPLEX } diff --git a/processing/src/main/java/io/druid/segment/incremental/IncrementalIndex.java b/processing/src/main/java/io/druid/segment/incremental/IncrementalIndex.java index 431ebe18df67..24d0ce390a33 100644 --- a/processing/src/main/java/io/druid/segment/incremental/IncrementalIndex.java +++ b/processing/src/main/java/io/druid/segment/incremental/IncrementalIndex.java @@ -53,6 +53,7 @@ import io.druid.segment.LongColumnSelector; import io.druid.segment.Metadata; import io.druid.segment.ObjectColumnSelector; +import io.druid.segment.VirtualColumns; import io.druid.segment.column.Column; import io.druid.segment.column.ColumnCapabilities; import io.druid.segment.column.ColumnCapabilitiesImpl; @@ -100,14 +101,25 @@ public abstract class IncrementalIndex implements Iterable, .put(DimensionSchema.ValueType.STRING, ValueType.STRING) .build(); + /** + * Column selector used at ingestion time for inputs to aggregators. + * + * @param agg the aggregator + * @param in ingestion-time input row supplier + * @param deserializeComplexMetrics whether complex objects should be deserialized by a {@link ComplexMetricExtractor} + * + * @return column selector factory + */ public static ColumnSelectorFactory makeColumnSelectorFactory( + final VirtualColumns virtualColumns, final AggregatorFactory agg, final Supplier in, final boolean deserializeComplexMetrics ) { final RowBasedColumnSelectorFactory baseSelectorFactory = RowBasedColumnSelectorFactory.create(in, null); - return new ColumnSelectorFactory() + + class IncrementalIndexInputRowColumnSelectorFactory implements ColumnSelectorFactory { @Override public LongColumnSelector makeLongColumnSelector(final String columnName) @@ -167,13 +179,16 @@ public ColumnCapabilities getColumnCapabilities(String columnName) { return baseSelectorFactory.getColumnCapabilities(columnName); } - }; + } + + return virtualColumns.wrap(new IncrementalIndexInputRowColumnSelectorFactory()); } private final long minTimestamp; private final QueryGranularity gran; private final boolean rollup; private final List> rowTransformers; + private final VirtualColumns virtualColumns; private final AggregatorFactory[] metrics; private final AggregatorType[] aggs; private final boolean deserializeComplexMetrics; @@ -217,6 +232,7 @@ public IncrementalIndex( this.minTimestamp = incrementalIndexSchema.getMinTimestamp(); this.gran = incrementalIndexSchema.getGran(); this.rollup = incrementalIndexSchema.isRollup(); + this.virtualColumns = incrementalIndexSchema.getVirtualColumns(); this.metrics = incrementalIndexSchema.getMetrics(); this.rowTransformers = new CopyOnWriteArrayList<>(); this.deserializeComplexMetrics = deserializeComplexMetrics; @@ -894,6 +910,15 @@ public int hashCode() } } + protected ColumnSelectorFactory makeColumnSelectorFactory( + final AggregatorFactory agg, + final Supplier in, + final boolean deserializeComplexMetrics + ) + { + return makeColumnSelectorFactory(virtualColumns, agg, in, deserializeComplexMetrics); + } + protected final Comparator dimsComparator() { return new TimeAndDimsComp(dimensionDescsList); diff --git a/processing/src/main/java/io/druid/segment/incremental/IncrementalIndexSchema.java b/processing/src/main/java/io/druid/segment/incremental/IncrementalIndexSchema.java index b3bbfb187896..ca26a95b9d2b 100644 --- a/processing/src/main/java/io/druid/segment/incremental/IncrementalIndexSchema.java +++ b/processing/src/main/java/io/druid/segment/incremental/IncrementalIndexSchema.java @@ -25,6 +25,7 @@ import io.druid.granularity.QueryGranularities; import io.druid.granularity.QueryGranularity; import io.druid.query.aggregation.AggregatorFactory; +import io.druid.segment.VirtualColumns; /** */ @@ -34,6 +35,7 @@ public class IncrementalIndexSchema private final long minTimestamp; private final TimestampSpec timestampSpec; private final QueryGranularity gran; + private final VirtualColumns virtualColumns; private final DimensionsSpec dimensionsSpec; private final AggregatorFactory[] metrics; private final boolean rollup; @@ -42,6 +44,7 @@ public IncrementalIndexSchema( long minTimestamp, TimestampSpec timestampSpec, QueryGranularity gran, + VirtualColumns virtualColumns, DimensionsSpec dimensionsSpec, AggregatorFactory[] metrics, boolean rollup @@ -50,6 +53,7 @@ public IncrementalIndexSchema( this.minTimestamp = minTimestamp; this.timestampSpec = timestampSpec; this.gran = gran; + this.virtualColumns = virtualColumns == null ? VirtualColumns.EMPTY : virtualColumns; this.dimensionsSpec = dimensionsSpec; this.metrics = metrics; this.rollup = rollup; @@ -70,6 +74,11 @@ public QueryGranularity getGran() return gran; } + public VirtualColumns getVirtualColumns() + { + return virtualColumns; + } + public DimensionsSpec getDimensionsSpec() { return dimensionsSpec; @@ -90,6 +99,7 @@ public static class Builder private long minTimestamp; private TimestampSpec timestampSpec; private QueryGranularity gran; + private VirtualColumns virtualColumns; private DimensionsSpec dimensionsSpec; private AggregatorFactory[] metrics; private boolean rollup; @@ -98,6 +108,7 @@ public Builder() { this.minTimestamp = 0L; this.gran = QueryGranularities.NONE; + this.virtualColumns = VirtualColumns.EMPTY; this.dimensionsSpec = new DimensionsSpec(null, null, null); this.metrics = new AggregatorFactory[]{}; this.rollup = true; @@ -133,6 +144,12 @@ public Builder withQueryGranularity(QueryGranularity gran) return this; } + public Builder withVirtualColumns(VirtualColumns virtualColumns) + { + this.virtualColumns = virtualColumns; + return this; + } + public Builder withDimensionsSpec(DimensionsSpec dimensionsSpec) { this.dimensionsSpec = dimensionsSpec == null ? DimensionsSpec.ofEmpty() : dimensionsSpec; @@ -167,7 +184,7 @@ public Builder withRollup(boolean rollup) public IncrementalIndexSchema build() { return new IncrementalIndexSchema( - minTimestamp, timestampSpec, gran, dimensionsSpec, metrics, rollup + minTimestamp, timestampSpec, gran, virtualColumns, dimensionsSpec, metrics, rollup ); } } diff --git a/processing/src/main/java/io/druid/segment/incremental/IncrementalIndexStorageAdapter.java b/processing/src/main/java/io/druid/segment/incremental/IncrementalIndexStorageAdapter.java index 364445a0f1fd..f24d3eaf7efd 100644 --- a/processing/src/main/java/io/druid/segment/incremental/IncrementalIndexStorageAdapter.java +++ b/processing/src/main/java/io/druid/segment/incremental/IncrementalIndexStorageAdapter.java @@ -44,12 +44,11 @@ import io.druid.segment.ObjectColumnSelector; import io.druid.segment.SingleScanTimeDimSelector; import io.druid.segment.StorageAdapter; -import io.druid.segment.VirtualColumn; import io.druid.segment.VirtualColumns; +import io.druid.segment.ZeroFloatColumnSelector; +import io.druid.segment.ZeroLongColumnSelector; import io.druid.segment.column.Column; import io.druid.segment.column.ColumnCapabilities; -import io.druid.segment.column.ColumnCapabilitiesImpl; -import io.druid.segment.column.ValueType; import io.druid.segment.data.Indexed; import io.druid.segment.data.ListIndexed; import io.druid.segment.filter.BooleanValueMatcher; @@ -64,8 +63,6 @@ */ public class IncrementalIndexStorageAdapter implements StorageAdapter { - private static final NullDimensionSelector NULL_DIMENSION_SELECTOR = new NullDimensionSelector(); - private final IncrementalIndex index; public IncrementalIndexStorageAdapter( @@ -340,6 +337,10 @@ public DimensionSelector makeDimensionSelector( DimensionSpec dimensionSpec ) { + if (virtualColumns.exists(dimensionSpec.getDimension())) { + return virtualColumns.makeDimensionSelector(dimensionSpec, this); + } + final String dimension = dimensionSpec.getDimension(); final ExtractionFn extractionFn = dimensionSpec.getExtractionFn(); @@ -354,7 +355,7 @@ public DimensionSelector makeDimensionSelector( final IncrementalIndex.DimensionDesc dimensionDesc = index.getDimension(dimensionSpec.getDimension()); if (dimensionDesc == null) { - return dimensionSpec.decorate(NULL_DIMENSION_SELECTOR); + return dimensionSpec.decorate(NullDimensionSelector.instance()); } final DimensionIndexer indexer = dimensionDesc.getIndexer(); @@ -364,6 +365,10 @@ public DimensionSelector makeDimensionSelector( @Override public FloatColumnSelector makeFloatColumnSelector(String columnName) { + if (virtualColumns.exists(columnName)) { + return virtualColumns.makeFloatColumnSelector(columnName, this); + } + final Integer dimIndex = index.getDimensionIndex(columnName); if (dimIndex != null) { final IncrementalIndex.DimensionDesc dimensionDesc = index.getDimension(columnName); @@ -377,14 +382,7 @@ public FloatColumnSelector makeFloatColumnSelector(String columnName) final Integer metricIndexInt = index.getMetricIndex(columnName); if (metricIndexInt == null) { - return new FloatColumnSelector() - { - @Override - public float get() - { - return 0.0f; - } - }; + return ZeroFloatColumnSelector.instance(); } final int metricIndex = metricIndexInt; @@ -401,6 +399,10 @@ public float get() @Override public LongColumnSelector makeLongColumnSelector(String columnName) { + if (virtualColumns.exists(columnName)) { + return virtualColumns.makeLongColumnSelector(columnName, this); + } + if (columnName.equals(Column.TIME_COLUMN_NAME)) { return new LongColumnSelector() { @@ -425,14 +427,7 @@ public long get() final Integer metricIndexInt = index.getMetricIndex(columnName); if (metricIndexInt == null) { - return new LongColumnSelector() - { - @Override - public long get() - { - return 0L; - } - }; + return ZeroLongColumnSelector.instance(); } final int metricIndex = metricIndexInt; @@ -453,6 +448,10 @@ public long get() @Override public ObjectColumnSelector makeObjectColumnSelector(String column) { + if (virtualColumns.exists(column)) { + return virtualColumns.makeObjectColumnSelector(column, this); + } + if (column.equals(Column.TIME_COLUMN_NAME)) { return new ObjectColumnSelector() { @@ -496,10 +495,6 @@ public Object get() IncrementalIndex.DimensionDesc dimensionDesc = index.getDimension(column); if (dimensionDesc == null) { - VirtualColumn virtualColumn = virtualColumns.getVirtualColumn(column); - if (virtualColumn != null) { - return virtualColumn.init(column, this); - } return null; } else { @@ -539,15 +534,11 @@ public Object get() @Override public ColumnCapabilities getColumnCapabilities(String columnName) { - ColumnCapabilities capabilities = index.getCapabilities(columnName); - if (capabilities == null && !virtualColumns.isEmpty()) { - VirtualColumn virtualColumn = virtualColumns.getVirtualColumn(columnName); - if (virtualColumn != null) { - Class clazz = virtualColumn.init(columnName, this).classOfObject(); - capabilities = new ColumnCapabilitiesImpl().setType(ValueType.typeFor(clazz)); - } + if (virtualColumns.exists(columnName)) { + return virtualColumns.getColumnCapabilities(columnName); } - return capabilities; + + return index.getCapabilities(columnName); } }; } diff --git a/processing/src/main/java/io/druid/segment/incremental/OffheapIncrementalIndex.java b/processing/src/main/java/io/druid/segment/incremental/OffheapIncrementalIndex.java index 2100e8ff0df1..4655d7c27623 100644 --- a/processing/src/main/java/io/druid/segment/incremental/OffheapIncrementalIndex.java +++ b/processing/src/main/java/io/druid/segment/incremental/OffheapIncrementalIndex.java @@ -91,31 +91,6 @@ public OffheapIncrementalIndex( aggBuffers.add(bb); } - public OffheapIncrementalIndex( - long minTimestamp, - QueryGranularity gran, - final AggregatorFactory[] metrics, - boolean deserializeComplexMetrics, - boolean reportParseExceptions, - boolean sortFacts, - int maxRowCount, - StupidPool bufferPool - ) - { - this( - new IncrementalIndexSchema.Builder().withMinTimestamp(minTimestamp) - .withQueryGranularity(gran) - .withMetrics(metrics) - .withRollup(IncrementalIndexSchema.DEFAULT_ROLLUP) - .build(), - deserializeComplexMetrics, - reportParseExceptions, - sortFacts, - maxRowCount, - bufferPool - ); - } - public OffheapIncrementalIndex( long minTimestamp, QueryGranularity gran, diff --git a/processing/src/main/java/io/druid/segment/virtual/BaseSingleValueDimensionSelector.java b/processing/src/main/java/io/druid/segment/virtual/BaseSingleValueDimensionSelector.java new file mode 100644 index 000000000000..57416f1e9d07 --- /dev/null +++ b/processing/src/main/java/io/druid/segment/virtual/BaseSingleValueDimensionSelector.java @@ -0,0 +1,53 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.segment.virtual; + +import io.druid.segment.DimensionSelector; +import io.druid.segment.data.IndexedInts; +import io.druid.segment.data.ZeroIndexedInts; + +public abstract class BaseSingleValueDimensionSelector implements DimensionSelector +{ + protected abstract String getValue(); + + @Override + public IndexedInts getRow() + { + return ZeroIndexedInts.instance(); + } + + @Override + public int getValueCardinality() + { + return DimensionSelector.CARDINALITY_UNKNOWN; + } + + @Override + public String lookupName(int id) + { + return getValue(); + } + + @Override + public int lookupId(String name) + { + throw new UnsupportedOperationException("Cannot lookupId"); + } +} diff --git a/processing/src/main/java/io/druid/segment/virtual/ExpressionSelectors.java b/processing/src/main/java/io/druid/segment/virtual/ExpressionSelectors.java index 7273d95759ed..137761dcca15 100644 --- a/processing/src/main/java/io/druid/segment/virtual/ExpressionSelectors.java +++ b/processing/src/main/java/io/druid/segment/virtual/ExpressionSelectors.java @@ -21,6 +21,7 @@ import io.druid.math.expr.Expr; import io.druid.segment.ColumnSelectorFactory; +import io.druid.segment.DimensionSelector; import io.druid.segment.FloatColumnSelector; import io.druid.segment.LongColumnSelector; @@ -46,7 +47,7 @@ public static LongColumnSelector makeLongColumnSelector( ) { final ExpressionObjectSelector baseSelector = ExpressionObjectSelector.from(columnSelectorFactory, expression); - return new LongColumnSelector() + class ExpressionLongColumnSelector implements LongColumnSelector { @Override public long get() @@ -54,7 +55,8 @@ public long get() final Number number = baseSelector.get(); return number != null ? number.longValue() : nullValue; } - }; + } + return new ExpressionLongColumnSelector(); } public static FloatColumnSelector makeFloatColumnSelector( @@ -64,7 +66,7 @@ public static FloatColumnSelector makeFloatColumnSelector( ) { final ExpressionObjectSelector baseSelector = ExpressionObjectSelector.from(columnSelectorFactory, expression); - return new FloatColumnSelector() + class ExpressionFloatColumnSelector implements FloatColumnSelector { @Override public float get() @@ -72,6 +74,25 @@ public float get() final Number number = baseSelector.get(); return number != null ? number.floatValue() : nullValue; } - }; + } + return new ExpressionFloatColumnSelector(); + } + + public static DimensionSelector makeDimensionSelector( + final ColumnSelectorFactory columnSelectorFactory, + final Expr expression + ) + { + final ExpressionObjectSelector baseSelector = ExpressionObjectSelector.from(columnSelectorFactory, expression); + class ExpressionDimensionSelector extends BaseSingleValueDimensionSelector + { + @Override + protected String getValue() + { + final Number number = baseSelector.get(); + return number == null ? null : String.valueOf(number); + } + } + return new ExpressionDimensionSelector(); } } diff --git a/processing/src/main/java/io/druid/segment/virtual/ExpressionVirtualColumn.java b/processing/src/main/java/io/druid/segment/virtual/ExpressionVirtualColumn.java new file mode 100644 index 000000000000..dd3f748525f3 --- /dev/null +++ b/processing/src/main/java/io/druid/segment/virtual/ExpressionVirtualColumn.java @@ -0,0 +1,179 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.segment.virtual; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.Preconditions; +import com.google.common.primitives.Ints; +import io.druid.math.expr.Expr; +import io.druid.math.expr.Parser; +import io.druid.query.dimension.DimensionSpec; +import io.druid.segment.ColumnSelectorFactory; +import io.druid.segment.DimensionSelector; +import io.druid.segment.FloatColumnSelector; +import io.druid.segment.LongColumnSelector; +import io.druid.segment.ObjectColumnSelector; +import io.druid.segment.VirtualColumn; +import io.druid.segment.column.ColumnCapabilities; +import io.druid.segment.column.ColumnCapabilitiesImpl; +import io.druid.segment.column.ValueType; +import org.apache.commons.codec.Charsets; + +import java.nio.ByteBuffer; +import java.util.List; + +public class ExpressionVirtualColumn implements VirtualColumn +{ + private static final ColumnCapabilities CAPABILITIES = new ColumnCapabilitiesImpl().setType(ValueType.FLOAT); + + private final String name; + private final String expression; + private final Expr parsedExpression; + + @JsonCreator + public ExpressionVirtualColumn( + @JsonProperty("name") String name, + @JsonProperty("expression") String expression + ) + { + this.name = Preconditions.checkNotNull(name, "name"); + this.expression = Preconditions.checkNotNull(expression, "expression"); + this.parsedExpression = Parser.parse(expression); + } + + @JsonProperty("name") + @Override + public String getOutputName() + { + return name; + } + + @JsonProperty + public String getExpression() + { + return expression; + } + + @Override + public ObjectColumnSelector makeObjectColumnSelector( + final String columnName, + final ColumnSelectorFactory columnSelectorFactory + ) + { + return ExpressionSelectors.makeObjectColumnSelector(columnSelectorFactory, parsedExpression); + } + + @Override + public DimensionSelector makeDimensionSelector( + final DimensionSpec dimensionSpec, + final ColumnSelectorFactory columnSelectorFactory + ) + { + return ExpressionSelectors.makeDimensionSelector(columnSelectorFactory, parsedExpression); + } + + @Override + public FloatColumnSelector makeFloatColumnSelector( + final String columnName, + final ColumnSelectorFactory columnSelectorFactory + ) + { + return ExpressionSelectors.makeFloatColumnSelector(columnSelectorFactory, parsedExpression, 0.0f); + } + + @Override + public LongColumnSelector makeLongColumnSelector( + final String columnName, + final ColumnSelectorFactory columnSelectorFactory + ) + { + return ExpressionSelectors.makeLongColumnSelector(columnSelectorFactory, parsedExpression, 0L); + } + + @Override + public ColumnCapabilities capabilities(String columnName) + { + return CAPABILITIES; + } + + @Override + public List requiredColumns() + { + return Parser.findRequiredBindings(expression); + } + + @Override + public boolean usesDotNotation() + { + return false; + } + + @Override + public byte[] getCacheKey() + { + final byte[] nameBytes = name.getBytes(Charsets.UTF_8); + final byte[] expressionBytes = expression.getBytes(Charsets.UTF_8); + + return ByteBuffer + .allocate(1 + Ints.BYTES * 2 + nameBytes.length + expressionBytes.length) + .put(VirtualColumnCacheHelper.CACHE_TYPE_ID_EXPRESSION) + .putInt(nameBytes.length) + .put(nameBytes) + .putInt(expressionBytes.length) + .put(expressionBytes) + .array(); + } + + @Override + public boolean equals(Object o) + { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + + ExpressionVirtualColumn that = (ExpressionVirtualColumn) o; + + if (!name.equals(that.name)) { + return false; + } + return expression.equals(that.expression); + } + + @Override + public int hashCode() + { + int result = name.hashCode(); + result = 31 * result + expression.hashCode(); + return result; + } + + @Override + public String toString() + { + return "ExpressionVirtualColumn{" + + "name='" + name + '\'' + + ", expression='" + expression + '\'' + + '}'; + } +} diff --git a/processing/src/main/java/io/druid/segment/virtual/VirtualColumnCacheHelper.java b/processing/src/main/java/io/druid/segment/virtual/VirtualColumnCacheHelper.java new file mode 100644 index 000000000000..7c7bba2edd9d --- /dev/null +++ b/processing/src/main/java/io/druid/segment/virtual/VirtualColumnCacheHelper.java @@ -0,0 +1,34 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.segment.virtual; + +public class VirtualColumnCacheHelper +{ + public static final byte CACHE_TYPE_ID_MAP = 0x00; + public static final byte CACHE_TYPE_ID_EXPRESSION = 0x01; + + // Starting byte 0xFF is reserved for site-specific virtual columns. + public static final byte CACHE_TYPE_ID_USER_DEFINED = (byte) 0xFF; + + private VirtualColumnCacheHelper() + { + // No instantiation. + } +} diff --git a/processing/src/main/java/io/druid/segment/virtual/VirtualizedColumnSelectorFactory.java b/processing/src/main/java/io/druid/segment/virtual/VirtualizedColumnSelectorFactory.java new file mode 100644 index 000000000000..fd01c0ef6fd2 --- /dev/null +++ b/processing/src/main/java/io/druid/segment/virtual/VirtualizedColumnSelectorFactory.java @@ -0,0 +1,99 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.segment.virtual; + +import com.google.common.base.Preconditions; +import io.druid.query.dimension.DimensionSpec; +import io.druid.segment.ColumnSelectorFactory; +import io.druid.segment.DimensionSelector; +import io.druid.segment.FloatColumnSelector; +import io.druid.segment.LongColumnSelector; +import io.druid.segment.ObjectColumnSelector; +import io.druid.segment.VirtualColumns; +import io.druid.segment.column.ColumnCapabilities; + +import javax.annotation.Nullable; + +public class VirtualizedColumnSelectorFactory implements ColumnSelectorFactory +{ + private final ColumnSelectorFactory baseFactory; + private final VirtualColumns virtualColumns; + + public VirtualizedColumnSelectorFactory( + ColumnSelectorFactory baseFactory, + VirtualColumns virtualColumns + ) + { + this.baseFactory = Preconditions.checkNotNull(baseFactory, "baseFactory"); + this.virtualColumns = Preconditions.checkNotNull(virtualColumns, "virtualColumns"); + } + + @Override + public DimensionSelector makeDimensionSelector(DimensionSpec dimensionSpec) + { + if (virtualColumns.exists(dimensionSpec.getDimension())) { + return virtualColumns.makeDimensionSelector(dimensionSpec, baseFactory); + } else { + return baseFactory.makeDimensionSelector(dimensionSpec); + } + } + + @Override + public FloatColumnSelector makeFloatColumnSelector(String columnName) + { + if (virtualColumns.exists(columnName)) { + return virtualColumns.makeFloatColumnSelector(columnName, baseFactory); + } else { + return baseFactory.makeFloatColumnSelector(columnName); + } + } + + @Override + public LongColumnSelector makeLongColumnSelector(String columnName) + { + if (virtualColumns.exists(columnName)) { + return virtualColumns.makeLongColumnSelector(columnName, baseFactory); + } else { + return baseFactory.makeLongColumnSelector(columnName); + } + } + + @Nullable + @Override + public ObjectColumnSelector makeObjectColumnSelector(String columnName) + { + if (virtualColumns.exists(columnName)) { + return virtualColumns.makeObjectColumnSelector(columnName, baseFactory); + } else { + return baseFactory.makeObjectColumnSelector(columnName); + } + } + + @Nullable + @Override + public ColumnCapabilities getColumnCapabilities(String columnName) + { + if (virtualColumns.exists(columnName)) { + return virtualColumns.getColumnCapabilities(columnName); + } else { + return baseFactory.getColumnCapabilities(columnName); + } + } +} diff --git a/processing/src/test/java/io/druid/query/select/SelectQuerySpecTest.java b/processing/src/test/java/io/druid/query/select/SelectQuerySpecTest.java index b6f918fa6baa..9c4ee4c58ec6 100644 --- a/processing/src/test/java/io/druid/query/select/SelectQuerySpecTest.java +++ b/processing/src/test/java/io/druid/query/select/SelectQuerySpecTest.java @@ -60,7 +60,7 @@ public void testSerializationLegacyString() throws Exception + "\"granularity\":{\"type\":\"all\"}," + "\"dimensions\":[{\"type\":\"default\",\"dimension\":\"market\",\"outputName\":\"market\"},{\"type\":\"default\",\"dimension\":\"quality\",\"outputName\":\"quality\"}]," + "\"metrics\":[\"index\"]," - + "\"virtualColumns\":null," + + "\"virtualColumns\":[]," + "\"pagingSpec\":{\"pagingIdentifiers\":{},\"threshold\":3,\"fromNext\":false}," + "\"context\":null}"; diff --git a/processing/src/test/java/io/druid/segment/NullDimensionSelectorTest.java b/processing/src/test/java/io/druid/segment/NullDimensionSelectorTest.java index aab2dff99c64..ac26061786c4 100644 --- a/processing/src/test/java/io/druid/segment/NullDimensionSelectorTest.java +++ b/processing/src/test/java/io/druid/segment/NullDimensionSelectorTest.java @@ -27,7 +27,7 @@ public class NullDimensionSelectorTest { - private final NullDimensionSelector selector = new NullDimensionSelector(); + private final NullDimensionSelector selector = NullDimensionSelector.instance(); @Test public void testGetRow() throws Exception { diff --git a/processing/src/test/java/io/druid/segment/TestIndex.java b/processing/src/test/java/io/druid/segment/TestIndex.java index 084785933807..efc7a6405aeb 100644 --- a/processing/src/test/java/io/druid/segment/TestIndex.java +++ b/processing/src/test/java/io/druid/segment/TestIndex.java @@ -41,6 +41,7 @@ import io.druid.segment.incremental.IncrementalIndexSchema; import io.druid.segment.incremental.OnheapIncrementalIndex; import io.druid.segment.serde.ComplexMetrics; +import io.druid.segment.virtual.ExpressionVirtualColumn; import org.joda.time.DateTime; import org.joda.time.Interval; @@ -78,10 +79,15 @@ public class TestIndex public static final String[] METRICS = new String[]{"index", "indexMin", "indexMaxPlusTen"}; private static final Logger log = new Logger(TestIndex.class); private static final Interval DATA_INTERVAL = new Interval("2011-01-12T00:00:00.000Z/2011-05-01T00:00:00.000Z"); + private static final VirtualColumns VIRTUAL_COLUMNS = VirtualColumns.create( + Arrays.asList( + new ExpressionVirtualColumn("expr", "index + 10") + ) + ); public static final AggregatorFactory[] METRIC_AGGS = new AggregatorFactory[]{ new DoubleSumAggregatorFactory(METRICS[0], METRICS[0]), new DoubleMinAggregatorFactory(METRICS[1], METRICS[0]), - new DoubleMaxAggregatorFactory(METRICS[2], null, "index + 10"), + new DoubleMaxAggregatorFactory(METRICS[2], VIRTUAL_COLUMNS.getVirtualColumns()[0].getOutputName()), new HyperUniquesAggregatorFactory("quality_uniques", "quality") }; private static final IndexSpec indexSpec = new IndexSpec(); @@ -224,6 +230,7 @@ public static IncrementalIndex makeRealtimeIndex(final CharSource source, boolea .withMinTimestamp(new DateTime("2011-01-12T00:00:00.000Z").getMillis()) .withTimestampSpec(new TimestampSpec("ds", "auto", null)) .withQueryGranularity(QueryGranularities.NONE) + .withVirtualColumns(VIRTUAL_COLUMNS) .withMetrics(METRIC_AGGS) .withRollup(rollup) .build(); diff --git a/processing/src/test/java/io/druid/segment/incremental/IncrementalIndexMultiValueSpecTest.java b/processing/src/test/java/io/druid/segment/incremental/IncrementalIndexMultiValueSpecTest.java index 48c98f2ce892..00d0e66a0995 100644 --- a/processing/src/test/java/io/druid/segment/incremental/IncrementalIndexMultiValueSpecTest.java +++ b/processing/src/test/java/io/druid/segment/incremental/IncrementalIndexMultiValueSpecTest.java @@ -28,6 +28,7 @@ import io.druid.data.input.impl.TimestampSpec; import io.druid.granularity.QueryGranularities; import io.druid.query.aggregation.AggregatorFactory; +import io.druid.segment.VirtualColumns; import org.junit.Assert; import org.junit.Test; @@ -54,6 +55,7 @@ public void test() throws IndexSizeExceededException 0, new TimestampSpec("ds", "auto", null), QueryGranularities.ALL, + VirtualColumns.EMPTY, dimensionsSpec, new AggregatorFactory[0], false diff --git a/processing/src/test/java/io/druid/segment/incremental/IncrementalIndexStorageAdapterTest.java b/processing/src/test/java/io/druid/segment/incremental/IncrementalIndexStorageAdapterTest.java index eaaa14da3da7..c041e842547c 100644 --- a/processing/src/test/java/io/druid/segment/incremental/IncrementalIndexStorageAdapterTest.java +++ b/processing/src/test/java/io/druid/segment/incremental/IncrementalIndexStorageAdapterTest.java @@ -49,9 +49,9 @@ import io.druid.segment.Cursor; import io.druid.segment.DimensionSelector; import io.druid.segment.StorageAdapter; -import io.druid.segment.VirtualColumns; import io.druid.segment.data.IndexedInts; import io.druid.segment.filter.SelectorFilter; +import io.druid.segment.VirtualColumns; import org.joda.time.DateTime; import org.joda.time.Interval; import org.junit.Assert; @@ -265,7 +265,7 @@ public void testResetSanity() throws IOException Sequence cursorSequence = adapter.makeCursors( new SelectorFilter("sally", "bo"), interval, - null, + VirtualColumns.EMPTY, QueryGranularities.NONE, descending ); diff --git a/processing/src/test/java/io/druid/segment/virtual/ExpressionVirtualColumnTest.java b/processing/src/test/java/io/druid/segment/virtual/ExpressionVirtualColumnTest.java new file mode 100644 index 000000000000..8b7c9cb61eec --- /dev/null +++ b/processing/src/test/java/io/druid/segment/virtual/ExpressionVirtualColumnTest.java @@ -0,0 +1,96 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.segment.virtual; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import io.druid.data.input.MapBasedInputRow; +import io.druid.query.dimension.DefaultDimensionSpec; +import io.druid.query.groupby.epinephelinae.TestColumnSelectorFactory; +import io.druid.segment.DimensionSelector; +import io.druid.segment.FloatColumnSelector; +import io.druid.segment.LongColumnSelector; +import io.druid.segment.ObjectColumnSelector; +import org.junit.Assert; +import org.junit.Test; + +public class ExpressionVirtualColumnTest +{ + @Test + public void testMakeSelectors() + { + final TestColumnSelectorFactory columnSelectorFactory = new TestColumnSelectorFactory(); + final ExpressionVirtualColumn virtualColumn = new ExpressionVirtualColumn("expr", "x + y"); + + final ObjectColumnSelector objectSelector = virtualColumn.makeObjectColumnSelector("expr", columnSelectorFactory); + final DimensionSelector dimensionSelector = virtualColumn.makeDimensionSelector( + new DefaultDimensionSpec("expr", "x"), + columnSelectorFactory + ); + final FloatColumnSelector floatSelector = virtualColumn.makeFloatColumnSelector("expr", columnSelectorFactory); + final LongColumnSelector longSelector = virtualColumn.makeLongColumnSelector("expr", columnSelectorFactory); + + columnSelectorFactory.setRow( + new MapBasedInputRow( + 0, + ImmutableList.of(), + ImmutableMap.of() + ) + ); + + Assert.assertEquals(null, objectSelector.get()); + Assert.assertEquals(null, dimensionSelector.lookupName(dimensionSelector.getRow().get(0))); + Assert.assertEquals(0.0f, floatSelector.get(), 0.0f); + Assert.assertEquals(0L, longSelector.get()); + + columnSelectorFactory.setRow( + new MapBasedInputRow( + 0, + ImmutableList.of(), + ImmutableMap.of("x", 4) + ) + ); + + Assert.assertEquals(null, objectSelector.get()); + Assert.assertEquals(null, dimensionSelector.lookupName(dimensionSelector.getRow().get(0))); + Assert.assertEquals(0.0f, floatSelector.get(), 0.0f); + Assert.assertEquals(0L, longSelector.get()); + + columnSelectorFactory.setRow( + new MapBasedInputRow( + 0, + ImmutableList.of(), + ImmutableMap.of("x", 2.1, "y", 3L) + ) + ); + + Assert.assertEquals(5.1d, objectSelector.get()); + Assert.assertEquals("5.1", dimensionSelector.lookupName(dimensionSelector.getRow().get(0))); + Assert.assertEquals(5.1f, floatSelector.get(), 0.0f); + Assert.assertEquals(5L, longSelector.get()); + } + + @Test + public void testRequiredColumns() + { + final ExpressionVirtualColumn virtualColumn = new ExpressionVirtualColumn("expr", "x + y"); + Assert.assertEquals(ImmutableList.of("x", "y"), virtualColumn.requiredColumns()); + } +} diff --git a/processing/src/test/java/io/druid/segment/virtual/VirtualColumnsTest.java b/processing/src/test/java/io/druid/segment/virtual/VirtualColumnsTest.java new file mode 100644 index 000000000000..61770e5074cf --- /dev/null +++ b/processing/src/test/java/io/druid/segment/virtual/VirtualColumnsTest.java @@ -0,0 +1,379 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.segment.virtual; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.collect.ImmutableList; +import com.google.common.primitives.Longs; +import io.druid.jackson.DefaultObjectMapper; +import io.druid.query.dimension.DefaultDimensionSpec; +import io.druid.query.dimension.DimensionSpec; +import io.druid.query.extraction.ExtractionFn; +import io.druid.segment.ColumnSelectorFactory; +import io.druid.segment.DimensionSelector; +import io.druid.segment.FloatColumnSelector; +import io.druid.segment.LongColumnSelector; +import io.druid.segment.ObjectColumnSelector; +import io.druid.segment.VirtualColumn; +import io.druid.segment.VirtualColumns; +import io.druid.segment.column.ColumnCapabilities; +import io.druid.segment.column.ColumnCapabilitiesImpl; +import io.druid.segment.column.ValueType; +import io.druid.segment.data.IndexedInts; +import it.unimi.dsi.fastutil.ints.IntIterator; +import it.unimi.dsi.fastutil.ints.IntIterators; +import org.junit.Assert; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; + +import java.io.IOException; +import java.util.Arrays; +import java.util.List; + +public class VirtualColumnsTest +{ + @Rule + public ExpectedException expectedException = ExpectedException.none(); + + @Test + public void testMakeSelectors() + { + final VirtualColumns virtualColumns = makeVirtualColumns(); + final ObjectColumnSelector objectSelector = virtualColumns.makeObjectColumnSelector("expr", null); + final DimensionSelector dimensionSelector = virtualColumns.makeDimensionSelector( + new DefaultDimensionSpec("expr", "x"), + null + ); + final FloatColumnSelector floatSelector = virtualColumns.makeFloatColumnSelector("expr", null); + final LongColumnSelector longSelector = virtualColumns.makeLongColumnSelector("expr", null); + + Assert.assertEquals(1L, objectSelector.get()); + Assert.assertEquals("1", dimensionSelector.lookupName(dimensionSelector.getRow().get(0))); + Assert.assertEquals(1.0f, floatSelector.get(), 0.0f); + Assert.assertEquals(1L, longSelector.get()); + } + + @Test + public void testMakeSelectorsWithDotSupport() + { + final VirtualColumns virtualColumns = makeVirtualColumns(); + final ObjectColumnSelector objectSelector = virtualColumns.makeObjectColumnSelector("foo.5", null); + final DimensionSelector dimensionSelector = virtualColumns.makeDimensionSelector( + new DefaultDimensionSpec("foo.5", "x"), + null + ); + final FloatColumnSelector floatSelector = virtualColumns.makeFloatColumnSelector("foo.5", null); + final LongColumnSelector longSelector = virtualColumns.makeLongColumnSelector("foo.5", null); + + Assert.assertEquals(5L, objectSelector.get()); + Assert.assertEquals("5", dimensionSelector.lookupName(dimensionSelector.getRow().get(0))); + Assert.assertEquals(5.0f, floatSelector.get(), 0.0f); + Assert.assertEquals(5L, longSelector.get()); + } + + @Test + public void testMakeSelectorsWithDotSupportBaseNameOnly() + { + final VirtualColumns virtualColumns = makeVirtualColumns(); + final ObjectColumnSelector objectSelector = virtualColumns.makeObjectColumnSelector("foo", null); + final DimensionSelector dimensionSelector = virtualColumns.makeDimensionSelector( + new DefaultDimensionSpec("foo", "x"), + null + ); + final FloatColumnSelector floatSelector = virtualColumns.makeFloatColumnSelector("foo", null); + final LongColumnSelector longSelector = virtualColumns.makeLongColumnSelector("foo", null); + + Assert.assertEquals(-1L, objectSelector.get()); + Assert.assertEquals("-1", dimensionSelector.lookupName(dimensionSelector.getRow().get(0))); + Assert.assertEquals(-1.0f, floatSelector.get(), 0.0f); + Assert.assertEquals(-1L, longSelector.get()); + } + + @Test + public void testTimeNotAllowed() + { + final ExpressionVirtualColumn expr = new ExpressionVirtualColumn("__time", "x + y"); + + expectedException.expect(IllegalArgumentException.class); + expectedException.expectMessage("virtualColumn name[__time] not allowed"); + + VirtualColumns.create(ImmutableList.of(expr)); + } + + @Test + public void testDuplicateNameDetection() + { + final ExpressionVirtualColumn expr = new ExpressionVirtualColumn("expr", "x + y"); + final ExpressionVirtualColumn expr2 = new ExpressionVirtualColumn("expr", "x * 2"); + + expectedException.expect(IllegalArgumentException.class); + expectedException.expectMessage("Duplicate virtualColumn name[expr]"); + + VirtualColumns.create(ImmutableList.of(expr, expr2)); + } + + @Test + public void testCycleDetection() + { + final ExpressionVirtualColumn expr = new ExpressionVirtualColumn("expr", "x + expr2"); + final ExpressionVirtualColumn expr2 = new ExpressionVirtualColumn("expr2", "expr * 2"); + + expectedException.expect(IllegalArgumentException.class); + expectedException.expectMessage("Self-referential column[expr]"); + + VirtualColumns.create(ImmutableList.of(expr, expr2)); + } + + @Test + public void testGetCacheKey() throws Exception + { + final VirtualColumns virtualColumns = VirtualColumns.create( + ImmutableList.of( + new ExpressionVirtualColumn("expr", "x + y") + ) + ); + + final VirtualColumns virtualColumns2 = VirtualColumns.create( + ImmutableList.of( + new ExpressionVirtualColumn("expr", "x + y") + ) + ); + + Assert.assertArrayEquals(virtualColumns.getCacheKey(), virtualColumns2.getCacheKey()); + Assert.assertFalse(Arrays.equals(virtualColumns.getCacheKey(), VirtualColumns.EMPTY.getCacheKey())); + } + + @Test + public void testEqualsAndHashCode() throws Exception + { + final VirtualColumns virtualColumns = VirtualColumns.create( + ImmutableList.of( + new ExpressionVirtualColumn("expr", "x + y") + ) + ); + + final VirtualColumns virtualColumns2 = VirtualColumns.create( + ImmutableList.of( + new ExpressionVirtualColumn("expr", "x + y") + ) + ); + + Assert.assertEquals(virtualColumns, virtualColumns); + Assert.assertEquals(virtualColumns, virtualColumns2); + Assert.assertNotEquals(VirtualColumns.EMPTY, virtualColumns); + Assert.assertNotEquals(VirtualColumns.EMPTY, null); + + Assert.assertEquals(virtualColumns.hashCode(), virtualColumns.hashCode()); + Assert.assertEquals(virtualColumns.hashCode(), virtualColumns2.hashCode()); + Assert.assertNotEquals(VirtualColumns.EMPTY.hashCode(), virtualColumns.hashCode()); + } + + @Test + public void testSerde() throws Exception + { + final ObjectMapper mapper = new DefaultObjectMapper(); + final ImmutableList theColumns = ImmutableList.of( + new ExpressionVirtualColumn("expr", "x + y"), + new ExpressionVirtualColumn("expr2", "x + z") + ); + final VirtualColumns virtualColumns = VirtualColumns.create(theColumns); + + Assert.assertEquals( + virtualColumns, + mapper.readValue( + mapper.writeValueAsString(virtualColumns), + VirtualColumns.class + ) + ); + + Assert.assertEquals( + theColumns, + mapper.readValue( + mapper.writeValueAsString(virtualColumns), + mapper.getTypeFactory().constructParametricType(List.class, VirtualColumn.class) + ) + ); + } + + private VirtualColumns makeVirtualColumns() + { + final ExpressionVirtualColumn expr = new ExpressionVirtualColumn("expr", "1"); + final DottyVirtualColumn dotty = new DottyVirtualColumn("foo"); + return VirtualColumns.create(ImmutableList.of(expr, dotty)); + } + + static class DottyVirtualColumn implements VirtualColumn + { + private final String name; + + public DottyVirtualColumn(String name) + { + this.name = name; + } + + @Override + public String getOutputName() + { + return name; + } + + @Override + public ObjectColumnSelector makeObjectColumnSelector(String columnName, ColumnSelectorFactory factory) + { + final LongColumnSelector selector = makeLongColumnSelector(columnName, factory); + return new ObjectColumnSelector() + { + @Override + public Class classOfObject() + { + return Long.class; + } + + @Override + public Object get() + { + return selector.get(); + } + }; + } + + @Override + public DimensionSelector makeDimensionSelector(DimensionSpec dimensionSpec, ColumnSelectorFactory factory) + { + final LongColumnSelector selector = makeLongColumnSelector(dimensionSpec.getDimension(), factory); + final ExtractionFn extractionFn = dimensionSpec.getExtractionFn(); + final DimensionSelector dimensionSelector = new DimensionSelector() + { + @Override + public IndexedInts getRow() + { + return new IndexedInts() + { + @Override + public int size() + { + return 1; + } + + @Override + public int get(int index) + { + return 0; + } + + @Override + public IntIterator iterator() + { + return IntIterators.singleton(0); + } + + @Override + public void fill(int index, int[] toFill) + { + throw new UnsupportedOperationException("fill not supported"); + } + + @Override + public void close() throws IOException + { + + } + }; + } + + @Override + public int getValueCardinality() + { + return DimensionSelector.CARDINALITY_UNKNOWN; + } + + @Override + public String lookupName(int id) + { + final String stringValue = String.valueOf(selector.get()); + return extractionFn == null ? stringValue : extractionFn.apply(stringValue); + } + + @Override + public int lookupId(String name) + { + return 0; + } + }; + + return dimensionSpec.decorate(dimensionSelector); + } + + @Override + public FloatColumnSelector makeFloatColumnSelector(String columnName, ColumnSelectorFactory factory) + { + final LongColumnSelector selector = makeLongColumnSelector(columnName, factory); + return new FloatColumnSelector() + { + @Override + public float get() + { + return selector.get(); + } + }; + } + + @Override + public LongColumnSelector makeLongColumnSelector(String columnName, ColumnSelectorFactory factory) + { + final String subColumn = VirtualColumns.splitColumnName(columnName).rhs; + final Long boxed = subColumn == null ? null : Longs.tryParse(subColumn); + final long theLong = boxed == null ? -1 : boxed; + return new LongColumnSelector() + { + @Override + public long get() + { + return theLong; + } + }; + } + + @Override + public ColumnCapabilities capabilities(String columnName) + { + return new ColumnCapabilitiesImpl().setType(ValueType.LONG); + } + + @Override + public List requiredColumns() + { + return ImmutableList.of(); + } + + @Override + public boolean usesDotNotation() + { + return true; + } + + @Override + public byte[] getCacheKey() + { + throw new UnsupportedOperationException(); + } + } +} From 5f181e8db0e10db25d7d097fc623a34b21880fe6 Mon Sep 17 00:00:00 2001 From: Gian Merlino Date: Fri, 6 Jan 2017 17:33:25 -0800 Subject: [PATCH 2/5] Fix ExtractionDimensionSpecs with virtual dimensions. --- .../query/extraction/BucketExtractionFn.java | 2 +- .../segment/virtual/ExpressionSelectors.java | 31 ++++++++++++++----- .../virtual/ExpressionVirtualColumn.java | 8 ++++- .../virtual/ExpressionVirtualColumnTest.java | 11 +++++++ .../segment/virtual/VirtualColumnsTest.java | 9 ++++++ 5 files changed, 51 insertions(+), 10 deletions(-) diff --git a/processing/src/main/java/io/druid/query/extraction/BucketExtractionFn.java b/processing/src/main/java/io/druid/query/extraction/BucketExtractionFn.java index e734412725dd..2e163fe0bee5 100644 --- a/processing/src/main/java/io/druid/query/extraction/BucketExtractionFn.java +++ b/processing/src/main/java/io/druid/query/extraction/BucketExtractionFn.java @@ -60,7 +60,7 @@ public double getOffset() public String apply(Object value) { if (value instanceof Number) { - return bucket((Double) value); + return bucket(((Number) value).doubleValue()); } else if (value instanceof String) { return apply(value); } diff --git a/processing/src/main/java/io/druid/segment/virtual/ExpressionSelectors.java b/processing/src/main/java/io/druid/segment/virtual/ExpressionSelectors.java index 137761dcca15..f1d02439b4e0 100644 --- a/processing/src/main/java/io/druid/segment/virtual/ExpressionSelectors.java +++ b/processing/src/main/java/io/druid/segment/virtual/ExpressionSelectors.java @@ -20,6 +20,7 @@ package io.druid.segment.virtual; import io.druid.math.expr.Expr; +import io.druid.query.extraction.ExtractionFn; import io.druid.segment.ColumnSelectorFactory; import io.druid.segment.DimensionSelector; import io.druid.segment.FloatColumnSelector; @@ -80,19 +81,33 @@ public float get() public static DimensionSelector makeDimensionSelector( final ColumnSelectorFactory columnSelectorFactory, - final Expr expression + final Expr expression, + final ExtractionFn extractionFn ) { final ExpressionObjectSelector baseSelector = ExpressionObjectSelector.from(columnSelectorFactory, expression); - class ExpressionDimensionSelector extends BaseSingleValueDimensionSelector - { - @Override - protected String getValue() + + if (extractionFn == null) { + class DefaultExpressionDimensionSelector extends BaseSingleValueDimensionSelector { - final Number number = baseSelector.get(); - return number == null ? null : String.valueOf(number); + @Override + protected String getValue() + { + final Number number = baseSelector.get(); + return number == null ? null : String.valueOf(number); + } + } + return new DefaultExpressionDimensionSelector(); + } else { + class ExtractionExpressionDimensionSelector extends BaseSingleValueDimensionSelector + { + @Override + protected String getValue() + { + return extractionFn.apply(baseSelector.get()); + } } + return new ExtractionExpressionDimensionSelector(); } - return new ExpressionDimensionSelector(); } } diff --git a/processing/src/main/java/io/druid/segment/virtual/ExpressionVirtualColumn.java b/processing/src/main/java/io/druid/segment/virtual/ExpressionVirtualColumn.java index dd3f748525f3..650063cdb669 100644 --- a/processing/src/main/java/io/druid/segment/virtual/ExpressionVirtualColumn.java +++ b/processing/src/main/java/io/druid/segment/virtual/ExpressionVirtualColumn.java @@ -87,7 +87,13 @@ public DimensionSelector makeDimensionSelector( final ColumnSelectorFactory columnSelectorFactory ) { - return ExpressionSelectors.makeDimensionSelector(columnSelectorFactory, parsedExpression); + return dimensionSpec.decorate( + ExpressionSelectors.makeDimensionSelector( + columnSelectorFactory, + parsedExpression, + dimensionSpec.getExtractionFn() + ) + ); } @Override diff --git a/processing/src/test/java/io/druid/segment/virtual/ExpressionVirtualColumnTest.java b/processing/src/test/java/io/druid/segment/virtual/ExpressionVirtualColumnTest.java index 8b7c9cb61eec..cb8df8a9112b 100644 --- a/processing/src/test/java/io/druid/segment/virtual/ExpressionVirtualColumnTest.java +++ b/processing/src/test/java/io/druid/segment/virtual/ExpressionVirtualColumnTest.java @@ -21,8 +21,12 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Sets; import io.druid.data.input.MapBasedInputRow; import io.druid.query.dimension.DefaultDimensionSpec; +import io.druid.query.dimension.ExtractionDimensionSpec; +import io.druid.query.dimension.ListFilteredDimensionSpec; +import io.druid.query.extraction.BucketExtractionFn; import io.druid.query.groupby.epinephelinae.TestColumnSelectorFactory; import io.druid.segment.DimensionSelector; import io.druid.segment.FloatColumnSelector; @@ -44,6 +48,10 @@ public void testMakeSelectors() new DefaultDimensionSpec("expr", "x"), columnSelectorFactory ); + final DimensionSelector extractionDimensionSelector = virtualColumn.makeDimensionSelector( + new ExtractionDimensionSpec("expr", "x", new BucketExtractionFn(1.0, 0.0)), + columnSelectorFactory + ); final FloatColumnSelector floatSelector = virtualColumn.makeFloatColumnSelector("expr", columnSelectorFactory); final LongColumnSelector longSelector = virtualColumn.makeLongColumnSelector("expr", columnSelectorFactory); @@ -57,6 +65,7 @@ public void testMakeSelectors() Assert.assertEquals(null, objectSelector.get()); Assert.assertEquals(null, dimensionSelector.lookupName(dimensionSelector.getRow().get(0))); + Assert.assertEquals(null, extractionDimensionSelector.lookupName(extractionDimensionSelector.getRow().get(0))); Assert.assertEquals(0.0f, floatSelector.get(), 0.0f); Assert.assertEquals(0L, longSelector.get()); @@ -70,6 +79,7 @@ public void testMakeSelectors() Assert.assertEquals(null, objectSelector.get()); Assert.assertEquals(null, dimensionSelector.lookupName(dimensionSelector.getRow().get(0))); + Assert.assertEquals(null, extractionDimensionSelector.lookupName(extractionDimensionSelector.getRow().get(0))); Assert.assertEquals(0.0f, floatSelector.get(), 0.0f); Assert.assertEquals(0L, longSelector.get()); @@ -83,6 +93,7 @@ public void testMakeSelectors() Assert.assertEquals(5.1d, objectSelector.get()); Assert.assertEquals("5.1", dimensionSelector.lookupName(dimensionSelector.getRow().get(0))); + Assert.assertEquals("5", extractionDimensionSelector.lookupName(extractionDimensionSelector.getRow().get(0))); Assert.assertEquals(5.1f, floatSelector.get(), 0.0f); Assert.assertEquals(5L, longSelector.get()); } diff --git a/processing/src/test/java/io/druid/segment/virtual/VirtualColumnsTest.java b/processing/src/test/java/io/druid/segment/virtual/VirtualColumnsTest.java index 61770e5074cf..0948985ef2db 100644 --- a/processing/src/test/java/io/druid/segment/virtual/VirtualColumnsTest.java +++ b/processing/src/test/java/io/druid/segment/virtual/VirtualColumnsTest.java @@ -21,10 +21,14 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.ImmutableList; +import com.google.common.collect.Sets; import com.google.common.primitives.Longs; import io.druid.jackson.DefaultObjectMapper; import io.druid.query.dimension.DefaultDimensionSpec; import io.druid.query.dimension.DimensionSpec; +import io.druid.query.dimension.ExtractionDimensionSpec; +import io.druid.query.dimension.ListFilteredDimensionSpec; +import io.druid.query.extraction.BucketExtractionFn; import io.druid.query.extraction.ExtractionFn; import io.druid.segment.ColumnSelectorFactory; import io.druid.segment.DimensionSelector; @@ -62,11 +66,16 @@ public void testMakeSelectors() new DefaultDimensionSpec("expr", "x"), null ); + final DimensionSelector extractionDimensionSelector = virtualColumns.makeDimensionSelector( + new ExtractionDimensionSpec("expr", "x", new BucketExtractionFn(1.0, 0.5)), + null + ); final FloatColumnSelector floatSelector = virtualColumns.makeFloatColumnSelector("expr", null); final LongColumnSelector longSelector = virtualColumns.makeLongColumnSelector("expr", null); Assert.assertEquals(1L, objectSelector.get()); Assert.assertEquals("1", dimensionSelector.lookupName(dimensionSelector.getRow().get(0))); + Assert.assertEquals("0.5", extractionDimensionSelector.lookupName(extractionDimensionSelector.getRow().get(0))); Assert.assertEquals(1.0f, floatSelector.get(), 0.0f); Assert.assertEquals(1L, longSelector.get()); } From beadde302864e9f2017b5d3c7aeeefb6660a2c54 Mon Sep 17 00:00:00 2001 From: Gian Merlino Date: Sat, 7 Jan 2017 08:49:03 -0800 Subject: [PATCH 3/5] Fix unused imports. --- .../io/druid/segment/virtual/ExpressionVirtualColumnTest.java | 2 -- .../test/java/io/druid/segment/virtual/VirtualColumnsTest.java | 2 -- 2 files changed, 4 deletions(-) diff --git a/processing/src/test/java/io/druid/segment/virtual/ExpressionVirtualColumnTest.java b/processing/src/test/java/io/druid/segment/virtual/ExpressionVirtualColumnTest.java index cb8df8a9112b..663777584dfe 100644 --- a/processing/src/test/java/io/druid/segment/virtual/ExpressionVirtualColumnTest.java +++ b/processing/src/test/java/io/druid/segment/virtual/ExpressionVirtualColumnTest.java @@ -21,11 +21,9 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Sets; import io.druid.data.input.MapBasedInputRow; import io.druid.query.dimension.DefaultDimensionSpec; import io.druid.query.dimension.ExtractionDimensionSpec; -import io.druid.query.dimension.ListFilteredDimensionSpec; import io.druid.query.extraction.BucketExtractionFn; import io.druid.query.groupby.epinephelinae.TestColumnSelectorFactory; import io.druid.segment.DimensionSelector; diff --git a/processing/src/test/java/io/druid/segment/virtual/VirtualColumnsTest.java b/processing/src/test/java/io/druid/segment/virtual/VirtualColumnsTest.java index 0948985ef2db..5d62faa911b1 100644 --- a/processing/src/test/java/io/druid/segment/virtual/VirtualColumnsTest.java +++ b/processing/src/test/java/io/druid/segment/virtual/VirtualColumnsTest.java @@ -21,13 +21,11 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.ImmutableList; -import com.google.common.collect.Sets; import com.google.common.primitives.Longs; import io.druid.jackson.DefaultObjectMapper; import io.druid.query.dimension.DefaultDimensionSpec; import io.druid.query.dimension.DimensionSpec; import io.druid.query.dimension.ExtractionDimensionSpec; -import io.druid.query.dimension.ListFilteredDimensionSpec; import io.druid.query.extraction.BucketExtractionFn; import io.druid.query.extraction.ExtractionFn; import io.druid.segment.ColumnSelectorFactory; From dfc814dde10c3bb4f490c0caac6047887528beec Mon Sep 17 00:00:00 2001 From: Gian Merlino Date: Wed, 18 Jan 2017 11:37:10 -0800 Subject: [PATCH 4/5] CR comments --- .../java/io/druid/segment/VirtualColumns.java | 24 ++++++++++++------- .../segment/ZeroFloatColumnSelector.java | 2 +- .../druid/segment/ZeroLongColumnSelector.java | 2 +- 3 files changed, 17 insertions(+), 11 deletions(-) diff --git a/processing/src/main/java/io/druid/segment/VirtualColumns.java b/processing/src/main/java/io/druid/segment/VirtualColumns.java index 02482eb8c0b4..bf4acccb76b0 100644 --- a/processing/src/main/java/io/druid/segment/VirtualColumns.java +++ b/processing/src/main/java/io/druid/segment/VirtualColumns.java @@ -24,7 +24,6 @@ import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import com.google.common.collect.ImmutableSet; import com.google.common.collect.Maps; import com.google.common.collect.Sets; import com.google.common.primitives.Ints; @@ -183,9 +182,15 @@ public LongColumnSelector makeLongColumnSelector(String columnName, ColumnSelect public ColumnCapabilities getColumnCapabilities(String columnName) { final VirtualColumn virtualColumn = getVirtualColumn(columnName); - return virtualColumn != null - ? Preconditions.checkNotNull(virtualColumn.capabilities(columnName), "capabilities for column[%s]", columnName) - : null; + if (virtualColumn != null) { + return Preconditions.checkNotNull( + virtualColumn.capabilities(columnName), + "capabilities for column[%s]", + columnName + ); + } else { + return null; + } } public boolean isEmpty() @@ -221,11 +226,12 @@ public byte[] getCacheKey() return buf.array(); } - private void detectCycles(VirtualColumn virtualColumn, ImmutableSet columnNames) + private void detectCycles(VirtualColumn virtualColumn, Set columnNames) { - Set nextSet = columnNames == null - ? Sets.newHashSet(virtualColumn.getOutputName()) - : Sets.newHashSet(columnNames); + // Copy columnNames to avoid modifying it + final Set nextSet = columnNames == null + ? Sets.newHashSet(virtualColumn.getOutputName()) + : Sets.newHashSet(columnNames); for (String columnName : virtualColumn.requiredColumns()) { if (!nextSet.add(columnName)) { @@ -234,7 +240,7 @@ private void detectCycles(VirtualColumn virtualColumn, ImmutableSet colu final VirtualColumn dependency = getVirtualColumn(columnName); if (dependency != null) { - detectCycles(dependency, ImmutableSet.copyOf(nextSet)); + detectCycles(dependency, nextSet); } } } diff --git a/processing/src/main/java/io/druid/segment/ZeroFloatColumnSelector.java b/processing/src/main/java/io/druid/segment/ZeroFloatColumnSelector.java index 8e8eb99a4f8b..888214862f81 100644 --- a/processing/src/main/java/io/druid/segment/ZeroFloatColumnSelector.java +++ b/processing/src/main/java/io/druid/segment/ZeroFloatColumnSelector.java @@ -19,7 +19,7 @@ package io.druid.segment; -public class ZeroFloatColumnSelector implements FloatColumnSelector +public final class ZeroFloatColumnSelector implements FloatColumnSelector { private static final ZeroFloatColumnSelector INSTANCE = new ZeroFloatColumnSelector(); diff --git a/processing/src/main/java/io/druid/segment/ZeroLongColumnSelector.java b/processing/src/main/java/io/druid/segment/ZeroLongColumnSelector.java index a24f16111c9f..911c0e24b265 100644 --- a/processing/src/main/java/io/druid/segment/ZeroLongColumnSelector.java +++ b/processing/src/main/java/io/druid/segment/ZeroLongColumnSelector.java @@ -19,7 +19,7 @@ package io.druid.segment; -public class ZeroLongColumnSelector implements LongColumnSelector +public final class ZeroLongColumnSelector implements LongColumnSelector { private static final ZeroLongColumnSelector INSTANCE = new ZeroLongColumnSelector(); From f70fa9ec8f31e08b0b74eb785ea27867f84f8fa4 Mon Sep 17 00:00:00 2001 From: Gian Merlino Date: Thu, 26 Jan 2017 12:09:06 -0800 Subject: [PATCH 5/5] Merge one more time, with feeling. --- .../io/druid/segment/MapVirtualColumn.java | 3 +- .../segment/QueryableIndexStorageAdapter.java | 1 + .../BaseSingleValueDimensionSelector.java | 43 ++++- .../virtual/ExpressionVirtualColumnTest.java | 168 +++++++++++++----- .../segment/virtual/VirtualColumnsTest.java | 35 +++- 5 files changed, 195 insertions(+), 55 deletions(-) diff --git a/extensions-contrib/virtual-columns/src/main/java/io/druid/segment/MapVirtualColumn.java b/extensions-contrib/virtual-columns/src/main/java/io/druid/segment/MapVirtualColumn.java index 23192d7ce923..33ce97c231ee 100644 --- a/extensions-contrib/virtual-columns/src/main/java/io/druid/segment/MapVirtualColumn.java +++ b/extensions-contrib/virtual-columns/src/main/java/io/druid/segment/MapVirtualColumn.java @@ -133,7 +133,6 @@ public String get() } }; } else { - final String key = dimension.substring(index + 1); return new ObjectColumnSelector() { @Override @@ -152,7 +151,7 @@ public String get() } final int limit = Math.min(keyIndices.size(), valueIndices.size()); for (int i = 0; i < limit; i++) { - if (Objects.equals(keySelector.lookupName(keyIndices.get(i)), key)) { + if (Objects.equals(keySelector.lookupName(keyIndices.get(i)), subColumnName)) { return valueSelector.lookupName(valueIndices.get(i)); } } diff --git a/processing/src/main/java/io/druid/segment/QueryableIndexStorageAdapter.java b/processing/src/main/java/io/druid/segment/QueryableIndexStorageAdapter.java index a2e9418a44f1..6e778b7aa399 100644 --- a/processing/src/main/java/io/druid/segment/QueryableIndexStorageAdapter.java +++ b/processing/src/main/java/io/druid/segment/QueryableIndexStorageAdapter.java @@ -55,6 +55,7 @@ import org.joda.time.Interval; import org.roaringbitmap.IntIterator; +import javax.annotation.Nullable; import java.io.Closeable; import java.io.IOException; import java.util.ArrayList; diff --git a/processing/src/main/java/io/druid/segment/virtual/BaseSingleValueDimensionSelector.java b/processing/src/main/java/io/druid/segment/virtual/BaseSingleValueDimensionSelector.java index 57416f1e9d07..5fb64c653eaf 100644 --- a/processing/src/main/java/io/druid/segment/virtual/BaseSingleValueDimensionSelector.java +++ b/processing/src/main/java/io/druid/segment/virtual/BaseSingleValueDimensionSelector.java @@ -19,10 +19,16 @@ package io.druid.segment.virtual; +import com.google.common.base.Predicate; +import io.druid.query.filter.ValueMatcher; import io.druid.segment.DimensionSelector; +import io.druid.segment.IdLookup; import io.druid.segment.data.IndexedInts; import io.druid.segment.data.ZeroIndexedInts; +import javax.annotation.Nullable; +import java.util.Objects; + public abstract class BaseSingleValueDimensionSelector implements DimensionSelector { protected abstract String getValue(); @@ -46,8 +52,41 @@ public String lookupName(int id) } @Override - public int lookupId(String name) + public ValueMatcher makeValueMatcher(final String value) + { + return new ValueMatcher() + { + @Override + public boolean matches() + { + return Objects.equals(getValue(), value); + } + }; + } + + @Override + public ValueMatcher makeValueMatcher(final Predicate predicate) + { + return new ValueMatcher() + { + @Override + public boolean matches() + { + return predicate.apply(getValue()); + } + }; + } + + @Override + public boolean nameLookupPossibleInAdvance() + { + return false; + } + + @Nullable + @Override + public IdLookup idLookup() { - throw new UnsupportedOperationException("Cannot lookupId"); + return null; } } diff --git a/processing/src/test/java/io/druid/segment/virtual/ExpressionVirtualColumnTest.java b/processing/src/test/java/io/druid/segment/virtual/ExpressionVirtualColumnTest.java index 663777584dfe..45ac94c8682b 100644 --- a/processing/src/test/java/io/druid/segment/virtual/ExpressionVirtualColumnTest.java +++ b/processing/src/test/java/io/druid/segment/virtual/ExpressionVirtualColumnTest.java @@ -19,12 +19,15 @@ package io.druid.segment.virtual; +import com.google.common.base.Predicates; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import io.druid.data.input.InputRow; import io.druid.data.input.MapBasedInputRow; import io.druid.query.dimension.DefaultDimensionSpec; import io.druid.query.dimension.ExtractionDimensionSpec; import io.druid.query.extraction.BucketExtractionFn; +import io.druid.query.filter.ValueMatcher; import io.druid.query.groupby.epinephelinae.TestColumnSelectorFactory; import io.druid.segment.DimensionSelector; import io.druid.segment.FloatColumnSelector; @@ -35,65 +38,132 @@ public class ExpressionVirtualColumnTest { + private static final InputRow ROW0 = new MapBasedInputRow( + 0, + ImmutableList.of(), + ImmutableMap.of() + ); + + private static final InputRow ROW1 = new MapBasedInputRow( + 0, + ImmutableList.of(), + ImmutableMap.of("x", 4) + ); + + private static final InputRow ROW2 = new MapBasedInputRow( + 0, + ImmutableList.of(), + ImmutableMap.of("x", 2.1, "y", 3L) + ); + + private static final ExpressionVirtualColumn XPLUSY = new ExpressionVirtualColumn("expr", "x + y"); + private static final TestColumnSelectorFactory COLUMN_SELECTOR_FACTORY = new TestColumnSelectorFactory(); + @Test - public void testMakeSelectors() + public void testObjectSelector() { - final TestColumnSelectorFactory columnSelectorFactory = new TestColumnSelectorFactory(); - final ExpressionVirtualColumn virtualColumn = new ExpressionVirtualColumn("expr", "x + y"); + final ObjectColumnSelector selector = XPLUSY.makeObjectColumnSelector("expr", COLUMN_SELECTOR_FACTORY); + + COLUMN_SELECTOR_FACTORY.setRow(ROW0); + Assert.assertEquals(null, selector.get()); + + COLUMN_SELECTOR_FACTORY.setRow(ROW1); + Assert.assertEquals(null, selector.get()); + + COLUMN_SELECTOR_FACTORY.setRow(ROW2); + Assert.assertEquals(5.1d, selector.get()); + } + + @Test + public void testLongSelector() + { + final LongColumnSelector selector = XPLUSY.makeLongColumnSelector("expr", COLUMN_SELECTOR_FACTORY); + + COLUMN_SELECTOR_FACTORY.setRow(ROW0); + Assert.assertEquals(0L, selector.get()); + + COLUMN_SELECTOR_FACTORY.setRow(ROW1); + Assert.assertEquals(0L, selector.get()); + + COLUMN_SELECTOR_FACTORY.setRow(ROW2); + Assert.assertEquals(5L, selector.get()); + } + + @Test + public void testFloatSelector() + { + final FloatColumnSelector selector = XPLUSY.makeFloatColumnSelector("expr", COLUMN_SELECTOR_FACTORY); - final ObjectColumnSelector objectSelector = virtualColumn.makeObjectColumnSelector("expr", columnSelectorFactory); - final DimensionSelector dimensionSelector = virtualColumn.makeDimensionSelector( + COLUMN_SELECTOR_FACTORY.setRow(ROW0); + Assert.assertEquals(0.0f, selector.get(), 0.0f); + + COLUMN_SELECTOR_FACTORY.setRow(ROW1); + Assert.assertEquals(0.0f, selector.get(), 0.0f); + + COLUMN_SELECTOR_FACTORY.setRow(ROW2); + Assert.assertEquals(5.1f, selector.get(), 0.0f); + } + + @Test + public void testDimensionSelector() + { + final DimensionSelector selector = XPLUSY.makeDimensionSelector( new DefaultDimensionSpec("expr", "x"), - columnSelectorFactory + COLUMN_SELECTOR_FACTORY ); - final DimensionSelector extractionDimensionSelector = virtualColumn.makeDimensionSelector( + + final ValueMatcher nullMatcher = selector.makeValueMatcher((String) null); + final ValueMatcher fiveMatcher = selector.makeValueMatcher("5"); + final ValueMatcher nonNullMatcher = selector.makeValueMatcher(Predicates.notNull()); + + COLUMN_SELECTOR_FACTORY.setRow(ROW0); + Assert.assertEquals(true, nullMatcher.matches()); + Assert.assertEquals(false, fiveMatcher.matches()); + Assert.assertEquals(false, nonNullMatcher.matches()); + Assert.assertEquals(null, selector.lookupName(selector.getRow().get(0))); + + COLUMN_SELECTOR_FACTORY.setRow(ROW1); + Assert.assertEquals(true, nullMatcher.matches()); + Assert.assertEquals(false, fiveMatcher.matches()); + Assert.assertEquals(false, nonNullMatcher.matches()); + Assert.assertEquals(null, selector.lookupName(selector.getRow().get(0))); + + COLUMN_SELECTOR_FACTORY.setRow(ROW2); + Assert.assertEquals(false, nullMatcher.matches()); + Assert.assertEquals(false, fiveMatcher.matches()); + Assert.assertEquals(true, nonNullMatcher.matches()); + Assert.assertEquals("5.1", selector.lookupName(selector.getRow().get(0))); + } + + @Test + public void testDimensionSelectorWithExtraction() + { + final DimensionSelector selector = XPLUSY.makeDimensionSelector( new ExtractionDimensionSpec("expr", "x", new BucketExtractionFn(1.0, 0.0)), - columnSelectorFactory - ); - final FloatColumnSelector floatSelector = virtualColumn.makeFloatColumnSelector("expr", columnSelectorFactory); - final LongColumnSelector longSelector = virtualColumn.makeLongColumnSelector("expr", columnSelectorFactory); - - columnSelectorFactory.setRow( - new MapBasedInputRow( - 0, - ImmutableList.of(), - ImmutableMap.of() - ) + COLUMN_SELECTOR_FACTORY ); - Assert.assertEquals(null, objectSelector.get()); - Assert.assertEquals(null, dimensionSelector.lookupName(dimensionSelector.getRow().get(0))); - Assert.assertEquals(null, extractionDimensionSelector.lookupName(extractionDimensionSelector.getRow().get(0))); - Assert.assertEquals(0.0f, floatSelector.get(), 0.0f); - Assert.assertEquals(0L, longSelector.get()); - - columnSelectorFactory.setRow( - new MapBasedInputRow( - 0, - ImmutableList.of(), - ImmutableMap.of("x", 4) - ) - ); + final ValueMatcher nullMatcher = selector.makeValueMatcher((String) null); + final ValueMatcher fiveMatcher = selector.makeValueMatcher("5"); + final ValueMatcher nonNullMatcher = selector.makeValueMatcher(Predicates.notNull()); - Assert.assertEquals(null, objectSelector.get()); - Assert.assertEquals(null, dimensionSelector.lookupName(dimensionSelector.getRow().get(0))); - Assert.assertEquals(null, extractionDimensionSelector.lookupName(extractionDimensionSelector.getRow().get(0))); - Assert.assertEquals(0.0f, floatSelector.get(), 0.0f); - Assert.assertEquals(0L, longSelector.get()); - - columnSelectorFactory.setRow( - new MapBasedInputRow( - 0, - ImmutableList.of(), - ImmutableMap.of("x", 2.1, "y", 3L) - ) - ); + COLUMN_SELECTOR_FACTORY.setRow(ROW0); + Assert.assertEquals(true, nullMatcher.matches()); + Assert.assertEquals(false, fiveMatcher.matches()); + Assert.assertEquals(false, nonNullMatcher.matches()); + Assert.assertEquals(null, selector.lookupName(selector.getRow().get(0))); + + COLUMN_SELECTOR_FACTORY.setRow(ROW1); + Assert.assertEquals(true, nullMatcher.matches()); + Assert.assertEquals(false, fiveMatcher.matches()); + Assert.assertEquals(false, nonNullMatcher.matches()); + Assert.assertEquals(null, selector.lookupName(selector.getRow().get(0))); - Assert.assertEquals(5.1d, objectSelector.get()); - Assert.assertEquals("5.1", dimensionSelector.lookupName(dimensionSelector.getRow().get(0))); - Assert.assertEquals("5", extractionDimensionSelector.lookupName(extractionDimensionSelector.getRow().get(0))); - Assert.assertEquals(5.1f, floatSelector.get(), 0.0f); - Assert.assertEquals(5L, longSelector.get()); + COLUMN_SELECTOR_FACTORY.setRow(ROW2); + Assert.assertEquals(false, nullMatcher.matches()); + Assert.assertEquals(true, fiveMatcher.matches()); + Assert.assertEquals(true, nonNullMatcher.matches()); + Assert.assertEquals("5", selector.lookupName(selector.getRow().get(0))); } @Test diff --git a/processing/src/test/java/io/druid/segment/virtual/VirtualColumnsTest.java b/processing/src/test/java/io/druid/segment/virtual/VirtualColumnsTest.java index 5d62faa911b1..f37d34940bfc 100644 --- a/processing/src/test/java/io/druid/segment/virtual/VirtualColumnsTest.java +++ b/processing/src/test/java/io/druid/segment/virtual/VirtualColumnsTest.java @@ -20,6 +20,7 @@ package io.druid.segment.virtual; import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.base.Predicate; import com.google.common.collect.ImmutableList; import com.google.common.primitives.Longs; import io.druid.jackson.DefaultObjectMapper; @@ -28,9 +29,12 @@ import io.druid.query.dimension.ExtractionDimensionSpec; import io.druid.query.extraction.BucketExtractionFn; import io.druid.query.extraction.ExtractionFn; +import io.druid.query.filter.ValueMatcher; import io.druid.segment.ColumnSelectorFactory; import io.druid.segment.DimensionSelector; +import io.druid.segment.DimensionSelectorUtils; import io.druid.segment.FloatColumnSelector; +import io.druid.segment.IdLookup; import io.druid.segment.LongColumnSelector; import io.druid.segment.ObjectColumnSelector; import io.druid.segment.VirtualColumn; @@ -46,6 +50,7 @@ import org.junit.Test; import org.junit.rules.ExpectedException; +import javax.annotation.Nullable; import java.io.IOException; import java.util.Arrays; import java.util.List; @@ -320,9 +325,35 @@ public String lookupName(int id) } @Override - public int lookupId(String name) + public ValueMatcher makeValueMatcher(final String value) { - return 0; + return DimensionSelectorUtils.makeValueMatcherGeneric(this, value); + } + + @Override + public ValueMatcher makeValueMatcher(final Predicate predicate) + { + return DimensionSelectorUtils.makeValueMatcherGeneric(this, predicate); + } + + @Override + public boolean nameLookupPossibleInAdvance() + { + return false; + } + + @Nullable + @Override + public IdLookup idLookup() + { + return new IdLookup() + { + @Override + public int lookupId(final String name) + { + return 0; + } + }; } };