diff --git a/common/src/main/java/io/druid/math/expr/Expr.java b/common/src/main/java/io/druid/math/expr/Expr.java index f24b4a4d7628..9c986084f0be 100644 --- a/common/src/main/java/io/druid/math/expr/Expr.java +++ b/common/src/main/java/io/druid/math/expr/Expr.java @@ -115,7 +115,7 @@ public Number eval(ObjectBinding bindings) { Number val = bindings.get(value); if (val == null) { - throw new RuntimeException("No binding found for " + value); + return null; } else { return val instanceof Long ? val : val.doubleValue(); } @@ -238,6 +238,11 @@ public BinaryOpExprBase(String op, Expr left, Expr right) this.right = right; } + protected boolean isNull(Number left, Number right) + { + return left == null || right == null; + } + protected boolean isLong(Number left, Number right) { return left instanceof Long && right instanceof Long; @@ -271,7 +276,9 @@ public Number eval(ObjectBinding bindings) { Number leftVal = left.eval(bindings); Number rightVal = right.eval(bindings); - if (isLong(leftVal, rightVal)) { + if (isNull(leftVal, rightVal)) { + return null; + } else if (isLong(leftVal, rightVal)) { return leftVal.longValue() - rightVal.longValue(); } else { return leftVal.doubleValue() - rightVal.doubleValue(); @@ -292,7 +299,9 @@ public Number eval(ObjectBinding bindings) { Number leftVal = left.eval(bindings); Number rightVal = right.eval(bindings); - if (isLong(leftVal, rightVal)) { + if (isNull(leftVal, rightVal)) { + return null; + } else if (isLong(leftVal, rightVal)) { return LongMath.pow(leftVal.longValue(), rightVal.intValue()); } else { return Math.pow(leftVal.doubleValue(), rightVal.doubleValue()); @@ -313,7 +322,9 @@ public Number eval(ObjectBinding bindings) { Number leftVal = left.eval(bindings); Number rightVal = right.eval(bindings); - if (isLong(leftVal, rightVal)) { + if (isNull(leftVal, rightVal)) { + return null; + } else if (isLong(leftVal, rightVal)) { return leftVal.longValue() * rightVal.longValue(); } else { return leftVal.doubleValue() * rightVal.doubleValue(); @@ -355,7 +366,9 @@ public Number eval(ObjectBinding bindings) { Number leftVal = left.eval(bindings); Number rightVal = right.eval(bindings); - if (isLong(leftVal, rightVal)) { + if (isNull(leftVal, rightVal)) { + return null; + } else if (isLong(leftVal, rightVal)) { return leftVal.longValue() % rightVal.longValue(); } else { return leftVal.doubleValue() % rightVal.doubleValue(); @@ -376,7 +389,9 @@ public Number eval(ObjectBinding bindings) { Number leftVal = left.eval(bindings); Number rightVal = right.eval(bindings); - if (isLong(leftVal, rightVal)) { + if (isNull(leftVal, rightVal)) { + return null; + } else if (isLong(leftVal, rightVal)) { return leftVal.longValue() + rightVal.longValue(); } else { return leftVal.doubleValue() + rightVal.doubleValue(); @@ -397,7 +412,9 @@ public Number eval(ObjectBinding bindings) { Number leftVal = left.eval(bindings); Number rightVal = right.eval(bindings); - if (isLong(leftVal, rightVal)) { + if (isNull(leftVal, rightVal)) { + return null; + } else if (isLong(leftVal, rightVal)) { return leftVal.longValue() < rightVal.longValue() ? 1 : 0; } else { return leftVal.doubleValue() < rightVal.doubleValue() ? 1.0d : 0.0d; @@ -418,7 +435,9 @@ public Number eval(ObjectBinding bindings) { Number leftVal = left.eval(bindings); Number rightVal = right.eval(bindings); - if (isLong(leftVal, rightVal)) { + if (isNull(leftVal, rightVal)) { + return null; + } else if (isLong(leftVal, rightVal)) { return leftVal.longValue() <= rightVal.longValue() ? 1 : 0; } else { return leftVal.doubleValue() <= rightVal.doubleValue() ? 1.0d : 0.0d; @@ -439,7 +458,9 @@ public Number eval(ObjectBinding bindings) { Number leftVal = left.eval(bindings); Number rightVal = right.eval(bindings); - if (isLong(leftVal, rightVal)) { + if (isNull(leftVal, rightVal)) { + return null; + } else if (isLong(leftVal, rightVal)) { return leftVal.longValue() > rightVal.longValue() ? 1 : 0; } else { return leftVal.doubleValue() > rightVal.doubleValue() ? 1.0d : 0.0d; @@ -460,7 +481,9 @@ public Number eval(ObjectBinding bindings) { Number leftVal = left.eval(bindings); Number rightVal = right.eval(bindings); - if (isLong(leftVal, rightVal)) { + if (isNull(leftVal, rightVal)) { + return null; + } else if (isLong(leftVal, rightVal)) { return leftVal.longValue() >= rightVal.longValue() ? 1 : 0; } else { return leftVal.doubleValue() >= rightVal.doubleValue() ? 1.0d : 0.0d; @@ -481,7 +504,9 @@ public Number eval(ObjectBinding bindings) { Number leftVal = left.eval(bindings); Number rightVal = right.eval(bindings); - if (isLong(leftVal, rightVal)) { + if (isNull(leftVal, rightVal)) { + return null; + } else if (isLong(leftVal, rightVal)) { return leftVal.longValue() == rightVal.longValue() ? 1 : 0; } else { return leftVal.doubleValue() == rightVal.doubleValue() ? 1.0d : 0.0d; @@ -502,7 +527,9 @@ public Number eval(ObjectBinding bindings) { Number leftVal = left.eval(bindings); Number rightVal = right.eval(bindings); - if (isLong(leftVal, rightVal)) { + if (isNull(leftVal, rightVal)) { + return null; + } else if (isLong(leftVal, rightVal)) { return leftVal.longValue() != rightVal.longValue() ? 1 : 0; } else { return leftVal.doubleValue() != rightVal.doubleValue() ? 1.0d : 0.0d; @@ -523,7 +550,9 @@ public Number eval(ObjectBinding bindings) { Number leftVal = left.eval(bindings); Number rightVal = right.eval(bindings); - if (isLong(leftVal, rightVal)) { + if (isNull(leftVal, rightVal)) { + return null; + } else if (isLong(leftVal, rightVal)) { long lval = leftVal.longValue(); if (lval > 0) { long rval = rightVal.longValue(); @@ -556,7 +585,9 @@ public Number eval(ObjectBinding bindings) { Number leftVal = left.eval(bindings); Number rightVal = right.eval(bindings); - if (isLong(leftVal, rightVal)) { + if (isNull(leftVal, rightVal)) { + return null; + } else if (isLong(leftVal, rightVal)) { long lval = leftVal.longValue(); if (lval > 0) { return 1; diff --git a/processing/src/main/java/io/druid/query/aggregation/AggregatorUtil.java b/processing/src/main/java/io/druid/query/aggregation/AggregatorUtil.java index 11c3007d4732..e106ef828c59 100644 --- a/processing/src/main/java/io/druid/query/aggregation/AggregatorUtil.java +++ b/processing/src/main/java/io/druid/query/aggregation/AggregatorUtil.java @@ -90,9 +90,10 @@ public static Pair, List> condensedAggre } public static FloatColumnSelector getFloatColumnSelector( - ColumnSelectorFactory metricFactory, - String fieldName, - String fieldExpression + final ColumnSelectorFactory metricFactory, + final String fieldName, + final String fieldExpression, + final float nullValue ) { if (fieldName != null && fieldExpression == null) { @@ -105,7 +106,8 @@ public static FloatColumnSelector getFloatColumnSelector( @Override public float get() { - return numeric.get().floatValue(); + final Number number = numeric.get(); + return number == null ? nullValue : number.floatValue(); } }; } @@ -113,9 +115,10 @@ public float get() } public static LongColumnSelector getLongColumnSelector( - ColumnSelectorFactory metricFactory, - String fieldName, - String fieldExpression + final ColumnSelectorFactory metricFactory, + final String fieldName, + final String fieldExpression, + final long nullValue ) { if (fieldName != null && fieldExpression == null) { @@ -128,7 +131,8 @@ public static LongColumnSelector getLongColumnSelector( @Override public long get() { - return numeric.get().longValue(); + final Number number = numeric.get(); + return number == null ? nullValue : number.longValue(); } }; } diff --git a/processing/src/main/java/io/druid/query/aggregation/DoubleMaxAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/DoubleMaxAggregatorFactory.java index e4cdccc4556e..36c3073e2a1c 100644 --- a/processing/src/main/java/io/druid/query/aggregation/DoubleMaxAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/DoubleMaxAggregatorFactory.java @@ -81,7 +81,7 @@ public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) private FloatColumnSelector getFloatColumnSelector(ColumnSelectorFactory metricFactory) { - return AggregatorUtil.getFloatColumnSelector(metricFactory, fieldName, expression); + return AggregatorUtil.getFloatColumnSelector(metricFactory, fieldName, expression, Float.MIN_VALUE); } @Override diff --git a/processing/src/main/java/io/druid/query/aggregation/DoubleMinAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/DoubleMinAggregatorFactory.java index 36c92262914c..bec58003a329 100644 --- a/processing/src/main/java/io/druid/query/aggregation/DoubleMinAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/DoubleMinAggregatorFactory.java @@ -81,7 +81,7 @@ public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) private FloatColumnSelector getFloatColumnSelector(ColumnSelectorFactory metricFactory) { - return AggregatorUtil.getFloatColumnSelector(metricFactory, fieldName, expression); + return AggregatorUtil.getFloatColumnSelector(metricFactory, fieldName, expression, Float.MAX_VALUE); } @Override diff --git a/processing/src/main/java/io/druid/query/aggregation/DoubleSumAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/DoubleSumAggregatorFactory.java index 0032c0712322..fa2b172fea83 100644 --- a/processing/src/main/java/io/druid/query/aggregation/DoubleSumAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/DoubleSumAggregatorFactory.java @@ -81,7 +81,7 @@ public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) private FloatColumnSelector getFloatColumnSelector(ColumnSelectorFactory metricFactory) { - return AggregatorUtil.getFloatColumnSelector(metricFactory, fieldName, expression); + return AggregatorUtil.getFloatColumnSelector(metricFactory, fieldName, expression, 0f); } @Override diff --git a/processing/src/main/java/io/druid/query/aggregation/LongMaxAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/LongMaxAggregatorFactory.java index de95e16b3d9c..e777de612617 100644 --- a/processing/src/main/java/io/druid/query/aggregation/LongMaxAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/LongMaxAggregatorFactory.java @@ -81,7 +81,7 @@ public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) private LongColumnSelector getLongColumnSelector(ColumnSelectorFactory metricFactory) { - return AggregatorUtil.getLongColumnSelector(metricFactory, fieldName, expression); + return AggregatorUtil.getLongColumnSelector(metricFactory, fieldName, expression, Long.MIN_VALUE); } @Override diff --git a/processing/src/main/java/io/druid/query/aggregation/LongMinAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/LongMinAggregatorFactory.java index 07830de70314..52317f4bd2f3 100644 --- a/processing/src/main/java/io/druid/query/aggregation/LongMinAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/LongMinAggregatorFactory.java @@ -81,7 +81,7 @@ public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) private LongColumnSelector getLongColumnSelector(ColumnSelectorFactory metricFactory) { - return AggregatorUtil.getLongColumnSelector(metricFactory, fieldName, expression); + return AggregatorUtil.getLongColumnSelector(metricFactory, fieldName, expression, Long.MAX_VALUE); } @Override diff --git a/processing/src/main/java/io/druid/query/aggregation/LongSumAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/LongSumAggregatorFactory.java index 8984719865bc..6c5679af332b 100644 --- a/processing/src/main/java/io/druid/query/aggregation/LongSumAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/LongSumAggregatorFactory.java @@ -81,7 +81,7 @@ public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) private LongColumnSelector getLongColumnSelector(ColumnSelectorFactory metricFactory) { - return AggregatorUtil.getLongColumnSelector(metricFactory, fieldName, expression); + return AggregatorUtil.getLongColumnSelector(metricFactory, fieldName, expression, 0L); } @Override diff --git a/processing/src/test/java/io/druid/query/QueryRunnerTestHelper.java b/processing/src/test/java/io/druid/query/QueryRunnerTestHelper.java index f375e2033758..8cfe775c897e 100644 --- a/processing/src/test/java/io/druid/query/QueryRunnerTestHelper.java +++ b/processing/src/test/java/io/druid/query/QueryRunnerTestHelper.java @@ -48,6 +48,10 @@ import io.druid.query.spec.MultipleIntervalSegmentSpec; import io.druid.query.spec.QuerySegmentSpec; import io.druid.query.spec.SpecificSegmentSpec; +import io.druid.query.timeseries.TimeseriesQuery; +import io.druid.query.timeseries.TimeseriesQueryEngine; +import io.druid.query.timeseries.TimeseriesQueryQueryToolChest; +import io.druid.query.timeseries.TimeseriesQueryRunnerFactory; import io.druid.segment.IncrementalIndexSegment; import io.druid.segment.QueryableIndex; import io.druid.segment.QueryableIndexSegment; @@ -185,7 +189,10 @@ public TableDataSource apply(@Nullable String input) public static ArithmeticPostAggregator hyperUniqueFinalizingPostAgg = new ArithmeticPostAggregator( hyperUniqueFinalizingPostAggMetric, "+", - Lists.newArrayList(new HyperUniqueFinalizingPostAggregator(uniqueMetric, uniqueMetric), new ConstantPostAggregator(null, 1)) + Lists.newArrayList( + new HyperUniqueFinalizingPostAggregator(uniqueMetric, uniqueMetric), + new ConstantPostAggregator(null, 1) + ) ); public static final List commonAggregators = Arrays.asList( @@ -335,7 +342,11 @@ public static > List> makeQueryRunn makeQueryRunner(factory, new IncrementalIndexSegment(rtIndex, segmentId), "rtIndex"), makeQueryRunner(factory, new IncrementalIndexSegment(noRollupRtIndex, segmentId), "noRollupRtIndex"), makeQueryRunner(factory, new QueryableIndexSegment(segmentId, mMappedTestIndex), "mMappedTestIndex"), - makeQueryRunner(factory, new QueryableIndexSegment(segmentId, noRollupMMappedTestIndex), "noRollupMMappedTestIndex"), + makeQueryRunner( + factory, + new QueryableIndexSegment(segmentId, noRollupMMappedTestIndex), + "noRollupMMappedTestIndex" + ), makeQueryRunner(factory, new QueryableIndexSegment(segmentId, mergedRealtimeIndex), "mergedRealtimeIndex") ); } @@ -361,9 +372,12 @@ public static Collection makeUnionQueryRunners( ) ); } + /** * Iterate through the iterables in a synchronous manner and return each step as an Object[] + * * @param in The iterables to step through. (effectively columns) + * * @return An iterable of Object[] containing the "rows" of the input (effectively rows) */ public static Iterable transformToConstructionFeeder(Iterable... in) @@ -498,7 +512,9 @@ public String toString() public static QueryRunner makeFilteringQueryRunner( final VersionedIntervalTimeline timeline, - final QueryRunnerFactory> factory) { + final QueryRunnerFactory> factory + ) + { final QueryToolChest> toolChest = factory.getToolchest(); return new FluentQueryRunnerBuilder(toolChest) @@ -537,11 +553,16 @@ public Sequence run(Query query, Map responseContext) public static IntervalChunkingQueryRunnerDecorator NoopIntervalChunkingQueryRunnerDecorator() { - return new IntervalChunkingQueryRunnerDecorator(null, null, null) { + return new IntervalChunkingQueryRunnerDecorator(null, null, null) + { @Override - public QueryRunner decorate(final QueryRunner delegate, - QueryToolChest> toolChest) { - return new QueryRunner() { + public QueryRunner decorate( + final QueryRunner delegate, + QueryToolChest> toolChest + ) + { + return new QueryRunner() + { @Override public Sequence run(Query query, Map responseContext) { @@ -560,4 +581,22 @@ public static Map of(Object... keyvalues) } return builder.build(); } + + public static QueryRunnerFactoryConglomerate newConglomerate() + { + return new DefaultQueryRunnerFactoryConglomerate( + ImmutableMap., QueryRunnerFactory>builder() + .put(TimeseriesQuery.class, newTimeseriesQueryRunnerFactory()) + .build() + ); + } + + public static TimeseriesQueryRunnerFactory newTimeseriesQueryRunnerFactory() + { + return new TimeseriesQueryRunnerFactory( + new TimeseriesQueryQueryToolChest(NoopIntervalChunkingQueryRunnerDecorator()), + new TimeseriesQueryEngine(), + QueryRunnerTestHelper.NOOP_QUERYWATCHER + ); + } } diff --git a/processing/src/test/java/io/druid/query/SchemaEvolutionTest.java b/processing/src/test/java/io/druid/query/SchemaEvolutionTest.java new file mode 100644 index 000000000000..8aa5997c109a --- /dev/null +++ b/processing/src/test/java/io/druid/query/SchemaEvolutionTest.java @@ -0,0 +1,372 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.query; + +import com.google.common.base.Function; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import com.google.common.io.Closeables; +import com.google.common.util.concurrent.MoreExecutors; +import io.druid.data.input.InputRow; +import io.druid.data.input.impl.DimensionsSpec; +import io.druid.data.input.impl.MapInputRowParser; +import io.druid.data.input.impl.TimeAndDimsParseSpec; +import io.druid.data.input.impl.TimestampSpec; +import io.druid.java.util.common.ISE; +import io.druid.java.util.common.guava.FunctionalIterable; +import io.druid.java.util.common.guava.Sequence; +import io.druid.java.util.common.guava.Sequences; +import io.druid.query.aggregation.AggregatorFactory; +import io.druid.query.aggregation.CountAggregatorFactory; +import io.druid.query.aggregation.DoubleSumAggregatorFactory; +import io.druid.query.aggregation.LongSumAggregatorFactory; +import io.druid.query.aggregation.hyperloglog.HyperUniquesAggregatorFactory; +import io.druid.query.filter.BoundDimFilter; +import io.druid.query.ordering.StringComparators; +import io.druid.query.timeseries.TimeseriesQuery; +import io.druid.query.timeseries.TimeseriesQueryRunnerFactory; +import io.druid.query.timeseries.TimeseriesResultValue; +import io.druid.segment.IndexBuilder; +import io.druid.segment.QueryableIndex; +import io.druid.segment.QueryableIndexSegment; +import io.druid.segment.incremental.IncrementalIndexSchema; +import org.joda.time.DateTime; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +/** + * Tests designed to exercise changing column types, adding columns, removing columns, etc. + */ +public class SchemaEvolutionTest +{ + private static final String DATA_SOURCE = "foo"; + private static final String TIMESTAMP_COLUMN = "t"; + private static final double THIRTY_ONE_POINT_ONE = 31.100000381469727d; + + public static List> timeseriesResult(final Map map) + { + return ImmutableList.of(new Result<>(new DateTime("2000"), new TimeseriesResultValue((Map) map))); + } + + public static List inputRowsWithDimensions(final List dimensions) + { + final MapInputRowParser parser = new MapInputRowParser( + new TimeAndDimsParseSpec( + new TimestampSpec(TIMESTAMP_COLUMN, "iso", null), + new DimensionsSpec( + DimensionsSpec.getDefaultSchemas(dimensions), + dimensions.isEmpty() ? ImmutableList.of("t", "c1", "c2") : null, + null + ) + ) + ); + return ImmutableList.of( + parser.parse(ImmutableMap.of("t", "2000-01-01", "c1", "9", "c2", ImmutableList.of("a"))), + parser.parse(ImmutableMap.of("t", "2000-01-02", "c1", "10.1", "c2", ImmutableList.of())), + parser.parse(ImmutableMap.of("t", "2000-01-03", "c1", "2", "c2", ImmutableList.of(""))), + parser.parse(ImmutableMap.of("t", "2001-01-01", "c1", "1", "c2", ImmutableList.of("a", "c"))), + parser.parse(ImmutableMap.of("t", "2001-01-02", "c1", "4", "c2", ImmutableList.of("abc"))), + parser.parse(ImmutableMap.of("t", "2001-01-03", "c1", "5")) + ); + } + + public static > List runQuery( + final QueryType query, + final QueryRunnerFactory factory, + final List indexes + ) + { + final Sequence results = new FinalizeResultsQueryRunner<>( + factory.getToolchest().mergeResults( + factory.mergeRunners( + MoreExecutors.sameThreadExecutor(), + FunctionalIterable + .create(indexes) + .transform( + new Function>() + { + @Override + public QueryRunner apply(final QueryableIndex index) + { + return factory.createRunner(new QueryableIndexSegment("xxx", index)); + } + } + ) + ) + ), + (QueryToolChest>) factory.getToolchest() + ).run(query, Maps.newHashMap()); + return Sequences.toList(results, Lists.newArrayList()); + } + + @Rule + public final TemporaryFolder temporaryFolder = new TemporaryFolder(); + + // Index1: c1 is a string, c2 nonexistent, "uniques" nonexistent + private QueryableIndex index1 = null; + + // Index2: c1 is a long, c2 is a string, "uniques" is uniques on c2 + private QueryableIndex index2 = null; + + // Index3: c1 is a float, c2 is a string, "uniques" is uniques on c2 + private QueryableIndex index3 = null; + + // Index4: c1 is nonexistent, c2 is uniques on c2 + private QueryableIndex index4 = null; + + @Before + public void setUp() throws IOException + { + // Index1: c1 is a string, c2 nonexistent, "uniques" nonexistent + index1 = IndexBuilder.create() + .tmpDir(temporaryFolder.newFolder()) + .schema( + new IncrementalIndexSchema.Builder() + .withMetrics(new AggregatorFactory[]{new CountAggregatorFactory("cnt")}) + .withRollup(false) + .build() + ) + .rows(inputRowsWithDimensions(ImmutableList.of("c1"))) + .buildMMappedIndex(); + + // Index2: c1 is a long, c2 is a string, "uniques" is uniques on c2 + index2 = IndexBuilder.create() + .tmpDir(temporaryFolder.newFolder()) + .schema( + new IncrementalIndexSchema.Builder() + .withMetrics(new AggregatorFactory[]{ + new CountAggregatorFactory("cnt"), + new LongSumAggregatorFactory("c1", "c1"), + new HyperUniquesAggregatorFactory("uniques", "c2") + }) + .withRollup(false) + .build() + ) + .rows(inputRowsWithDimensions(ImmutableList.of("c2"))) + .buildMMappedIndex(); + + // Index3: c1 is a float, c2 is a string, "uniques" is uniques on c2 + index3 = IndexBuilder.create() + .tmpDir(temporaryFolder.newFolder()) + .schema( + new IncrementalIndexSchema.Builder() + .withMetrics(new AggregatorFactory[]{ + new CountAggregatorFactory("cnt"), + new DoubleSumAggregatorFactory("c1", "c1"), + new HyperUniquesAggregatorFactory("uniques", "c2") + }) + .withRollup(false) + .build() + ) + .rows(inputRowsWithDimensions(ImmutableList.of("c2"))) + .buildMMappedIndex(); + + // Index4: c1 is nonexistent, c2 is uniques on c2 + index4 = IndexBuilder.create() + .tmpDir(temporaryFolder.newFolder()) + .schema( + new IncrementalIndexSchema.Builder() + .withMetrics(new AggregatorFactory[]{ + new HyperUniquesAggregatorFactory("c2", "c2") + }) + .withRollup(false) + .build() + ) + .rows(inputRowsWithDimensions(ImmutableList.of())) + .buildMMappedIndex(); + + if (index4.getAvailableDimensions().size() != 0) { + // Just double-checking that the exclusions are working properly + throw new ISE("WTF?! Expected no dimensions in index4"); + } + } + + @After + public void tearDown() throws IOException + { + Closeables.close(index1, false); + Closeables.close(index2, false); + Closeables.close(index3, false); + Closeables.close(index4, false); + } + + @Test + public void testHyperUniqueEvolutionTimeseries() + { + final TimeseriesQueryRunnerFactory factory = QueryRunnerTestHelper.newTimeseriesQueryRunnerFactory(); + + final TimeseriesQuery query = Druids + .newTimeseriesQueryBuilder() + .dataSource(DATA_SOURCE) + .intervals("1000/3000") + .aggregators( + ImmutableList.of( + new HyperUniquesAggregatorFactory("uniques", "uniques") + ) + ) + .build(); + + // index1 has no "uniques" column + Assert.assertEquals( + timeseriesResult(ImmutableMap.of("uniques", 0)), + runQuery(query, factory, ImmutableList.of(index1)) + ); + + // index1 (no uniques) + index2 and index3 (yes uniques); we should be able to combine + Assert.assertEquals( + timeseriesResult(ImmutableMap.of("uniques", 4.003911343725148d)), + runQuery(query, factory, ImmutableList.of(index1, index2, index3)) + ); + } + + @Test + public void testNumericEvolutionTimeseriesAggregation() + { + final TimeseriesQueryRunnerFactory factory = QueryRunnerTestHelper.newTimeseriesQueryRunnerFactory(); + + // "c1" changes from string(1) -> long(2) -> float(3) -> nonexistent(4) + // test behavior of longSum/doubleSum with/without expressions + final TimeseriesQuery query = Druids + .newTimeseriesQueryBuilder() + .dataSource(DATA_SOURCE) + .intervals("1000/3000") + .aggregators( + ImmutableList.of( + new LongSumAggregatorFactory("a", "c1"), + new DoubleSumAggregatorFactory("b", "c1"), + new LongSumAggregatorFactory("c", null, "c1 * 1"), + new DoubleSumAggregatorFactory("d", null, "c1 * 1") + ) + ) + .build(); + + // Only string(1) + Assert.assertEquals( + timeseriesResult(ImmutableMap.of("a", 0L, "b", 0.0, "c", 0L, "d", 0.0)), + runQuery(query, factory, ImmutableList.of(index1)) + ); + + // Only long(2) + Assert.assertEquals( + timeseriesResult(ImmutableMap.of("a", 31L, "b", 31.0, "c", 31L, "d", 31.0)), + runQuery(query, factory, ImmutableList.of(index2)) + ); + + // Only float(3) + Assert.assertEquals( + timeseriesResult(ImmutableMap.of("a", 31L, "b", THIRTY_ONE_POINT_ONE, "c", 31L, "d", THIRTY_ONE_POINT_ONE)), + runQuery(query, factory, ImmutableList.of(index3)) + ); + + // Only nonexistent(4) + Assert.assertEquals( + timeseriesResult(ImmutableMap.of("a", 0L, "b", 0.0, "c", 0L, "d", 0.0)), + runQuery(query, factory, ImmutableList.of(index4)) + ); + + // string(1) + long(2) + float(3) + nonexistent(4) + Assert.assertEquals( + timeseriesResult(ImmutableMap.of( + "a", 31L * 2, + "b", THIRTY_ONE_POINT_ONE + 31, + "c", 31L * 2, + "d", THIRTY_ONE_POINT_ONE + 31 + )), + runQuery(query, factory, ImmutableList.of(index1, index2, index3, index4)) + ); + + // long(2) + float(3) + nonexistent(4) + Assert.assertEquals( + timeseriesResult(ImmutableMap.of( + "a", 31L * 2, + "b", THIRTY_ONE_POINT_ONE + 31, + "c", 31L * 2, + "d", THIRTY_ONE_POINT_ONE + 31 + )), + runQuery(query, factory, ImmutableList.of(index2, index3, index4)) + ); + } + + @Test + public void testNumericEvolutionFiltering() + { + final TimeseriesQueryRunnerFactory factory = QueryRunnerTestHelper.newTimeseriesQueryRunnerFactory(); + + // "c1" changes from string(1) -> long(2) -> float(3) -> nonexistent(4) + // test behavior of filtering + final TimeseriesQuery query = Druids + .newTimeseriesQueryBuilder() + .dataSource(DATA_SOURCE) + .intervals("1000/3000") + .filters(new BoundDimFilter("c1", "9", "11", false, false, null, null, StringComparators.NUMERIC)) + .aggregators( + ImmutableList.of( + new LongSumAggregatorFactory("a", "c1"), + new DoubleSumAggregatorFactory("b", "c1"), + new CountAggregatorFactory("c") + ) + ) + .build(); + + // Only string(1) -- which we can filter but not aggregate + Assert.assertEquals( + timeseriesResult(ImmutableMap.of("a", 0L, "b", 0.0, "c", 2L)), + runQuery(query, factory, ImmutableList.of(index1)) + ); + + // Only long(2) -- which we can filter and aggregate + Assert.assertEquals( + timeseriesResult(ImmutableMap.of("a", 19L, "b", 19.0, "c", 2L)), + runQuery(query, factory, ImmutableList.of(index2)) + ); + + // Only float(3) -- which we can't filter, but can aggregate + Assert.assertEquals( + timeseriesResult(ImmutableMap.of("a", 0L, "b", 0.0, "c", 0L)), + runQuery(query, factory, ImmutableList.of(index3)) + ); + + // Only nonexistent(4) + Assert.assertEquals( + timeseriesResult(ImmutableMap.of("a", 0L, "b", 0.0, "c", 0L)), + runQuery(query, factory, ImmutableList.of(index4)) + ); + + // string(1) + long(2) + float(3) + nonexistent(4) + Assert.assertEquals( + timeseriesResult(ImmutableMap.of( + "a", 19L, + "b", 19.0, + "c", 4L + )), + runQuery(query, factory, ImmutableList.of(index1, index2, index3, index4)) + ); + } +} diff --git a/processing/src/test/java/io/druid/segment/IndexBuilder.java b/processing/src/test/java/io/druid/segment/IndexBuilder.java index 2fdfb02f53ea..5626f30b2b17 100644 --- a/processing/src/test/java/io/druid/segment/IndexBuilder.java +++ b/processing/src/test/java/io/druid/segment/IndexBuilder.java @@ -91,12 +91,6 @@ public IndexBuilder tmpDir(File tmpDir) return this; } - public IndexBuilder maxRows(int maxRows) - { - this.maxRows = maxRows; - return this; - } - public IndexBuilder add(InputRow... rows) { return add(Arrays.asList(rows));