diff --git a/benchmarks/src/main/java/io/druid/benchmark/IndexedIntsWrappingBenchmark.java b/benchmarks/src/main/java/io/druid/benchmark/IndexedIntsWrappingBenchmark.java new file mode 100644 index 000000000000..59880df1c1a1 --- /dev/null +++ b/benchmarks/src/main/java/io/druid/benchmark/IndexedIntsWrappingBenchmark.java @@ -0,0 +1,402 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.benchmark; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Lists; +import com.google.common.primitives.Ints; +import com.metamx.common.guava.Sequence; +import com.metamx.common.guava.Sequences; +import io.druid.data.input.InputRow; +import io.druid.data.input.MapBasedInputRow; +import io.druid.granularity.QueryGranularity; +import io.druid.jackson.DefaultObjectMapper; +import io.druid.query.aggregation.AggregatorFactory; +import io.druid.query.aggregation.CountAggregatorFactory; +import io.druid.query.aggregation.DoubleSumAggregatorFactory; +import io.druid.query.aggregation.LongSumAggregatorFactory; +import io.druid.query.dimension.DefaultDimensionSpec; +import io.druid.segment.Cursor; +import io.druid.segment.DimensionSelector; +import io.druid.segment.IndexIO; +import io.druid.segment.IndexMerger; +import io.druid.segment.IndexMergerV9; +import io.druid.segment.IndexSpec; +import io.druid.segment.QueryableIndex; +import io.druid.segment.QueryableIndexStorageAdapter; +import io.druid.segment.StorageAdapter; +import io.druid.segment.column.ColumnConfig; +import io.druid.segment.data.IndexedInts; +import io.druid.segment.data.VSizeIndexedInts; +import io.druid.segment.incremental.IncrementalIndex; +import io.druid.segment.incremental.IncrementalIndexStorageAdapter; +import io.druid.segment.incremental.OnheapIncrementalIndex; +import org.joda.time.Interval; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Level; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OperationsPerInvocation; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.TearDown; +import org.openjdk.jmh.infra.Blackhole; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.Random; +import java.util.concurrent.TimeUnit; + +@State(Scope.Benchmark) +public class IndexedIntsWrappingBenchmark +{ + private class IndexedIntsDimensionSelectorRow implements Iterable + { + private final IndexedInts delegate; + + public IndexedIntsDimensionSelectorRow(IndexedInts delegate) + { + this.delegate = delegate; + } + + public Iterator iterator() + { + return delegate.iterator(); + } + + public int size() + { + return delegate.size(); + } + + public Comparable get(int index) + { + return delegate.get(index); + } + + public IndexedInts getDelegate() + { + return delegate; + } + } + + static final int ROW_SIZE = 64; + static final int MAX_ROWS = 250000; + static final int MAX_VALUE = 2000000000; + static final boolean USE_VSIZE = false; + + private static final IndexMergerV9 INDEX_MERGER_V9; + private static final IndexIO INDEX_IO; + public static final ObjectMapper JSON_MAPPER; + + private IncrementalIndex incIndex; + private QueryableIndex qIndex; + + private static AggregatorFactory[] AGGS; + + private ArrayList rows = new ArrayList<>(); + private ArrayList wrappedRows = new ArrayList<>(); + private ArrayList incSelectors = new ArrayList<>(); + private ArrayList qSelectors = new ArrayList<>(); + private Cursor incCursor; + private Cursor qCursor; + + private Random rng; + private Random rng2; + private File tmpFile; + + private StorageAdapter incAdapter; + private StorageAdapter qAdapter; + + static { + JSON_MAPPER = new DefaultObjectMapper(); + INDEX_IO = new IndexIO( + JSON_MAPPER, + new ColumnConfig() + { + @Override + public int columnCacheSizeBytes() + { + return 0; + } + } + ); + INDEX_MERGER_V9 = new IndexMergerV9(JSON_MAPPER, INDEX_IO); + } + + static { + final ArrayList ingestAggregatorFactories = new ArrayList<>(ROW_SIZE + 1); + ingestAggregatorFactories.add(new CountAggregatorFactory("rows")); + for (int i = 0; i < ROW_SIZE; ++i) { + if (i % 2 != 0) { + continue; + } + + ingestAggregatorFactories.add( + new LongSumAggregatorFactory( + String.format("sumResult%s", i), + String.format("Dim_%s", i) + ) + ); + ingestAggregatorFactories.add( + new DoubleSumAggregatorFactory( + String.format("doubleSumResult%s", i), + String.format("Dim_%s", i) + ) + ); + } + AGGS = ingestAggregatorFactories.toArray(new AggregatorFactory[0]); + } + + @Setup + public void setupTrial() throws IOException + { + rng = new Random(9999); + rng2 = new Random(9999); + + rows = new ArrayList<>(); + wrappedRows = new ArrayList<>(); + for (int i = 0; i < MAX_ROWS; i++) { + IndexedInts row = getIndexedInts(USE_VSIZE); + IndexedIntsDimensionSelectorRow wrappedRow = wrapIndexedInts(row); + rows.add(row); + wrappedRows.add(wrappedRow); + } + + if (incIndex != null) { + incIndex.close(); + } + incIndex = makeIncIndex(); + incAdapter = new IncrementalIndexStorageAdapter(incIndex); + } + + @Setup(Level.Iteration) + public void setupIteration() throws IOException + { + Sequence incCursors = incAdapter.makeCursors(null, new Interval(0L, 10L), QueryGranularity.NONE, false); + incCursor = Sequences.toList(Sequences.limit(incCursors, 1), Lists.newArrayList()).get(0); + incCursor.reset(); + incSelectors = new ArrayList<>(); + for (int i = 0; i < ROW_SIZE; i++) { + String dimName = String.format("Dim_%s", i); + incSelectors.add(incCursor.makeDimensionSelector(new DefaultDimensionSpec(dimName, dimName))); + } + } + + private IndexedInts getIndexedInts(boolean useVSize) + { + final int[] values = new int[ROW_SIZE]; + for(int i = 0; i < values.length; i++) { + values[i] = rng.nextInt(MAX_VALUE); + } + + if(useVSize) { + return VSizeIndexedInts.fromArray(values); + } else { + IndexedInts retVal = new IndexedInts() + { + @Override + public int size() + { + return values.length; + } + + @Override + public int get(int index) + { + return values[index]; + } + + @Override + public Iterator iterator() + { + return Ints.asList(values).iterator(); + } + + @Override + public void fill(int index, int[] toFill) + { + throw new UnsupportedOperationException("fill not supported"); + } + + @Override + public void close() throws IOException + { + + } + }; + return retVal; + } + } + + private IndexedIntsDimensionSelectorRow wrapIndexedInts(IndexedInts values) + { + return new IndexedIntsDimensionSelectorRow(values); + } + + private IncrementalIndex makeIncIndex() throws IOException + { + IncrementalIndex index = new OnheapIncrementalIndex( + 0, + QueryGranularity.NONE, + AGGS, + false, + true, + true, + MAX_ROWS + ); + + for (int i = 0; i < MAX_ROWS; i++) { + InputRow row = getStringRow(5L, i, ROW_SIZE); + index.add(row); + } + + return index; + } + + private MapBasedInputRow getStringRow(long timestamp, int rowID, int dimensionCount) + { + List dimensionList = new ArrayList(dimensionCount); + ImmutableMap.Builder builder = ImmutableMap.builder(); + for (int i = 0; i < dimensionCount; i++) { + String dimName = String.format("Dim_%d", i); + dimensionList.add(dimName); + builder.put(dimName, String.valueOf(rng2.nextInt(1000))); + } + return new MapBasedInputRow(timestamp, dimensionList, builder.build()); + } + + @Benchmark + @BenchmarkMode(Mode.AverageTime) + @OutputTimeUnit(TimeUnit.MICROSECONDS) + @OperationsPerInvocation(MAX_ROWS) + public void unwrapped(Blackhole blackhole) throws Exception + { + int sum = 0; + for (int i = 0; i < MAX_ROWS; i++) { + IndexedInts row = rows.get(i); + for (int val : row) { + sum += val; + blackhole.consume(val); + } + } + blackhole.consume(sum); + } + + @Benchmark + @BenchmarkMode(Mode.AverageTime) + @OutputTimeUnit(TimeUnit.MICROSECONDS) + @OperationsPerInvocation(MAX_ROWS) + public void unwrappedCast(Blackhole blackhole) throws Exception + { + int sum = 0; + for (int i = 0; i < MAX_ROWS; i++) { + IndexedInts row = rows.get(i); + for (int val : row) { + sum += (Integer) val; + blackhole.consume((Integer) val); + } + } + blackhole.consume(sum); + } + + @Benchmark + @BenchmarkMode(Mode.AverageTime) + @OutputTimeUnit(TimeUnit.MICROSECONDS) + @OperationsPerInvocation(MAX_ROWS) + public void unwrappedCast2(Blackhole blackhole) throws Exception + { + int sum = 0; + for (int i = 0; i < MAX_ROWS; i++) { + IndexedInts row = rows.get(i); + for (int val : row) { + Integer iVal = (Integer) val; + sum += iVal; + blackhole.consume(iVal); + } + } + blackhole.consume(sum); + } + + @Benchmark + @BenchmarkMode(Mode.AverageTime) + @OutputTimeUnit(TimeUnit.MICROSECONDS) + @OperationsPerInvocation(MAX_ROWS) + public void wrapped(Blackhole blackhole) throws Exception + { + int sum = 0; + for (int i = 0; i < MAX_ROWS; i++) { + IndexedInts row = rows.get(i); + IndexedIntsDimensionSelectorRow wrow = wrapIndexedInts(row); + for (Comparable val : wrow) { + sum += (Integer) val; + blackhole.consume(val); + } + } + blackhole.consume(sum); + } + + @Benchmark + @BenchmarkMode(Mode.AverageTime) + @OutputTimeUnit(TimeUnit.MICROSECONDS) + @OperationsPerInvocation(MAX_ROWS) + public void readIncIndex(Blackhole blackhole) throws Exception + { + int sum = 0; + for (int i = 0; i < MAX_ROWS; i++) { + for (int j = 0; j < ROW_SIZE; j++) { + IndexedInts row = incSelectors.get(j).getRow(); + for (int val : row) { + sum += val; + blackhole.consume(val); + } + } + incCursor.advance(); + } + blackhole.consume(sum); + } + + @Benchmark + @BenchmarkMode(Mode.AverageTime) + @OutputTimeUnit(TimeUnit.MICROSECONDS) + @OperationsPerInvocation(MAX_ROWS) + public void readIncIndexCast(Blackhole blackhole) throws Exception + { + int sum = 0; + for (int i = 0; i < MAX_ROWS; i++) { + for (int j = 0; j < ROW_SIZE; j++) { + IndexedInts row = incSelectors.get(j).getRow(); + IndexedIntsDimensionSelectorRow dsRow = new IndexedIntsDimensionSelectorRow(row); + for (Comparable val : dsRow) { + sum += (Integer) val; + blackhole.consume(val); + } + } + incCursor.advance(); + } + blackhole.consume(sum); + } +} diff --git a/benchmarks/src/main/java/io/druid/benchmark/QueryableIndexLoadingBenchmark.java b/benchmarks/src/main/java/io/druid/benchmark/QueryableIndexLoadingBenchmark.java new file mode 100644 index 000000000000..3f486977ce44 --- /dev/null +++ b/benchmarks/src/main/java/io/druid/benchmark/QueryableIndexLoadingBenchmark.java @@ -0,0 +1,311 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.benchmark; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Lists; +import com.google.common.primitives.Ints; +import com.metamx.common.guava.Sequence; +import com.metamx.common.guava.Sequences; +import io.druid.data.input.InputRow; +import io.druid.data.input.MapBasedInputRow; +import io.druid.granularity.QueryGranularity; +import io.druid.jackson.DefaultObjectMapper; +import io.druid.query.aggregation.AggregatorFactory; +import io.druid.query.aggregation.CountAggregatorFactory; +import io.druid.query.aggregation.DoubleSumAggregatorFactory; +import io.druid.query.aggregation.LongSumAggregatorFactory; +import io.druid.query.dimension.DefaultDimensionSpec; +import io.druid.segment.Cursor; +import io.druid.segment.DimensionSelector; +import io.druid.segment.IndexIO; +import io.druid.segment.IndexMergerV9; +import io.druid.segment.IndexSpec; +import io.druid.segment.QueryableIndex; +import io.druid.segment.QueryableIndexStorageAdapter; +import io.druid.segment.StorageAdapter; +import io.druid.segment.column.ColumnConfig; +import io.druid.segment.data.IndexedInts; +import io.druid.segment.incremental.IncrementalIndex; +import io.druid.segment.incremental.OnheapIncrementalIndex; +import org.joda.time.Interval; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Level; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OperationsPerInvocation; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.TearDown; +import org.openjdk.jmh.infra.Blackhole; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.Random; +import java.util.concurrent.TimeUnit; + +@State(Scope.Thread) +public class QueryableIndexLoadingBenchmark +{ + private class IndexedIntsDimensionSelectorRow implements Iterable + { + private final IndexedInts delegate; + + public IndexedIntsDimensionSelectorRow(IndexedInts delegate) + { + this.delegate = delegate; + } + + public Iterator iterator() + { + return delegate.iterator(); + } + + public int size() + { + return delegate.size(); + } + + public Comparable get(int index) + { + return delegate.get(index); + } + + public IndexedInts getDelegate() + { + return delegate; + } + } + + static final int ROW_SIZE = 64; + static final int MAX_ROWS = 250000; + static final int MAX_VALUE = 20000; + + private static final IndexMergerV9 INDEX_MERGER_V9; + private static final IndexIO INDEX_IO; + public static final ObjectMapper JSON_MAPPER; + + private QueryableIndex qIndex; + + private static AggregatorFactory[] AGGS; + + private Random rng; + private Random rng2; + private File tmpFile; + + private StorageAdapter qAdapter; + + private boolean initialized = false; + + static { + JSON_MAPPER = new DefaultObjectMapper(); + INDEX_IO = new IndexIO( + JSON_MAPPER, + new ColumnConfig() + { + @Override + public int columnCacheSizeBytes() + { + return 0; + } + } + ); + INDEX_MERGER_V9 = new IndexMergerV9(JSON_MAPPER, INDEX_IO); + } + + static { + final ArrayList ingestAggregatorFactories = new ArrayList<>(ROW_SIZE + 1); + ingestAggregatorFactories.add(new CountAggregatorFactory("rows")); + for (int i = 0; i < ROW_SIZE; ++i) { + if (i % 2 != 0) { + continue; + } + + ingestAggregatorFactories.add( + new LongSumAggregatorFactory( + String.format("sumResult%s", i), + String.format("Dim_%s", i) + ) + ); + ingestAggregatorFactories.add( + new DoubleSumAggregatorFactory( + String.format("doubleSumResult%s", i), + String.format("Dim_%s", i) + ) + ); + } + AGGS = ingestAggregatorFactories.toArray(new AggregatorFactory[0]); + } + + @Setup + public void setup() throws IOException + { + if(initialized) { + return; + } + + rng = new Random(9999); + rng2 = new Random(9999); + + IncrementalIndex index = makeIncIndex(); + + File tmpFile = File.createTempFile("IndexedIntsWrappingBenchmark-INDEX-", String.valueOf(rng.nextInt(10))); + tmpFile.delete(); + tmpFile.mkdirs(); + System.out.println(tmpFile.getAbsolutePath() + " isFile: " + tmpFile.isFile() + " isDir:" + tmpFile.isDirectory()); + tmpFile.deleteOnExit(); + File indexFile = INDEX_MERGER_V9.persist( + index, + tmpFile, + new IndexSpec() + ); + + index.close(); + index = null; + + // Uncomment this to use a fixed location. + //File indexFile = new File("/Users/user/bench_index/64dim_card20000"); + + if (qIndex != null) { + qIndex.close(); + } + qIndex = makeQueryableIndexFromFile(indexFile); + qAdapter = new QueryableIndexStorageAdapter(qIndex); + initialized = true; + } + + private IncrementalIndex makeIncIndex() throws IOException + { + IncrementalIndex index = new OnheapIncrementalIndex( + 0, + QueryGranularity.NONE, + AGGS, + false, + true, + true, + MAX_ROWS + ); + + for (int i = 0; i < MAX_ROWS; i++) { + InputRow row = getStringRow(5L, i, ROW_SIZE); + index.add(row); + } + + return index; + } + + private QueryableIndex makeQueryableIndex(IncrementalIndex incIndex) throws IOException + { + QueryableIndex index = + INDEX_IO.loadIndex( + INDEX_MERGER_V9.persist( + incIndex, + tmpFile, + new IndexSpec() + ) + ); + return index; + } + + private QueryableIndex makeQueryableIndexFromFile(File file) throws IOException + { + QueryableIndex index = + INDEX_IO.loadIndex(file); + return index; + } + + private MapBasedInputRow getStringRow(long timestamp, int rowID, int dimensionCount) + { + List dimensionList = new ArrayList(dimensionCount); + ImmutableMap.Builder builder = ImmutableMap.builder(); + for (int i = 0; i < dimensionCount; i++) { + String dimName = String.format("Dim_%d", i); + dimensionList.add(dimName); + builder.put(dimName, String.valueOf(rng2.nextInt(MAX_VALUE))); + } + return new MapBasedInputRow(timestamp, dimensionList, builder.build()); + } + + @Benchmark + @BenchmarkMode(Mode.AverageTime) + @OutputTimeUnit(TimeUnit.MICROSECONDS) + @OperationsPerInvocation(MAX_ROWS) + public void AloadQIndex(Blackhole blackhole) throws Exception + { + int sum = 0; + Sequence qCursors = qAdapter.makeCursors(null, new Interval(0L, 10L), QueryGranularity.NONE, false); + Cursor qCursor = Sequences.toList(Sequences.limit(qCursors, 1), Lists.newArrayList()).get(0); + qCursor.reset(); + List qSelectors = new ArrayList<>(); + for (int i = 0; i < ROW_SIZE; i++) { + String dimName = String.format("Dim_%s", i); + qSelectors.add(qCursor.makeDimensionSelector(new DefaultDimensionSpec(dimName, dimName))); + } + + for (int i = 0; i < MAX_ROWS; i++) { + for (int j = 0; j < ROW_SIZE; j++) { + IndexedInts row = qSelectors.get(j).getRow(); + for (int val : row) { + sum += val; + blackhole.consume(val); + } + } + qCursor.advance(); + } + blackhole.consume(sum); + } + + @Benchmark + @BenchmarkMode(Mode.AverageTime) + @OutputTimeUnit(TimeUnit.MICROSECONDS) + @OperationsPerInvocation(MAX_ROWS) + public void loadQIndexWrap(Blackhole blackhole) throws Exception + { + int sum = 0; + Sequence qCursors = qAdapter.makeCursors(null, new Interval(0L, 10L), QueryGranularity.NONE, false); + Cursor qCursor = Sequences.toList(Sequences.limit(qCursors, 1), Lists.newArrayList()).get(0); + qCursor.reset(); + List qSelectors = new ArrayList<>(); + for (int i = 0; i < ROW_SIZE; i++) { + String dimName = String.format("Dim_%s", i); + qSelectors.add(qCursor.makeDimensionSelector(new DefaultDimensionSpec(dimName, dimName))); + } + + for (int i = 0; i < MAX_ROWS; i++) { + for (int j = 0; j < ROW_SIZE; j++) { + IndexedInts row = qSelectors.get(j).getRow(); + IndexedIntsDimensionSelectorRow dsRow = new IndexedIntsDimensionSelectorRow(row); + for (Comparable val : dsRow) { + sum += (Integer) val; + blackhole.consume(val); + } + } + qCursor.advance(); + } + blackhole.consume(sum); + } + +} diff --git a/docs/content/design/index.md b/docs/content/design/index.md index fb48c6dcf6b8..fc70f55a860c 100644 --- a/docs/content/design/index.md +++ b/docs/content/design/index.md @@ -24,7 +24,7 @@ This data set is composed of three distinct components. If you are acquainted wi * **Timestamp column**: We treat timestamp separately because all of our queries center around the time axis. -* **Dimension columns**: Dimensions are string attributes of an event, and the columns most commonly used in filtering the data. +* **Dimension columns**: Dimensions are attributes of an event, the columns most commonly used in filtering the data. We have four dimensions in our example data set: publisher, advertiser, gender, and country. They each represent an axis of the data that we’ve chosen to slice across. diff --git a/docs/content/design/segments.md b/docs/content/design/segments.md index 7d0b1258ae43..121e8a3c3462 100644 --- a/docs/content/design/segments.md +++ b/docs/content/design/segments.md @@ -30,16 +30,15 @@ image below: ![Druid column types](../../img/druid-column-types.png "Druid Column Types") -The timestamp and metric columns are simple: behind the scenes each of +The timestamp, numeric dimension (i.e. Long and Float), and metric columns are simple: behind the scenes each of these is an array of integer or floating point values compressed with LZ4. Once a query knows which rows it needs to select, it simply decompresses these, pulls out the relevant rows, and applies the desired aggregation operator. As with all columns, if a query doesn’t require a column, then that column’s data is just skipped over. -Dimensions columns are different because they support filter and -group-by operations, so each dimension requires the following -three data structures: +String-typed dimension columns have additional data structures for supporting filter and +group-by operations: 1. A dictionary that maps values (which are always treated as strings) to integer IDs, 2. A list of the column’s values, encoded using the dictionary in 1, and @@ -88,6 +87,9 @@ columns will have extremely sparse, and therefore highly compressible, bitmaps. Druid exploits this using compression algorithms that are specially suited for bitmaps, such as roaring bitmap compression. +Numeric dimensions do not currently have any supporting index structures, so filtering operations +on such dimensions require a full scan. + ### Multi-value columns If a data source makes use of multi-value columns, then the data diff --git a/docs/content/development/geo.md b/docs/content/development/geo.md index f85fccf35315..bd80a85ff23d 100644 --- a/docs/content/development/geo.md +++ b/docs/content/development/geo.md @@ -5,6 +5,10 @@ layout: doc_page Druid supports filtering specially spatially indexed columns based on an origin and a bound. # Spatial Indexing +
+@Deprecated. +In the dataSpec, the spatial dimensions should be specified within the "dimensions" list instead. +
In any of the data specs, there is the option of providing spatial dimensions. For example, for a JSON data spec, spatial dimensions can be specified as follows: ```json @@ -21,6 +25,8 @@ In any of the data specs, there is the option of providing spatial dimensions. F } ``` + + |property|description|required?| |--------|-----------|---------| |dimName|The name of the spatial dimension. A spatial dimension may be constructed from multiple other dimensions or it may already exist as part of an event. If a spatial dimension already exists, it must be an array of coordinate values.|yes| diff --git a/docs/content/ingestion/faq.md b/docs/content/ingestion/faq.md index 2289f308be58..b8eb4a9533ba 100644 --- a/docs/content/ingestion/faq.md +++ b/docs/content/ingestion/faq.md @@ -16,7 +16,7 @@ If you are trying to batch load historical data but no events are being loaded, ## What types of data does Druid support? -Druid can ingest JSON, CSV, TSV and other delimited data out of the box. Druid supports single dimension values, or multiple dimension values (an array of strings). Druid supports long and float numeric columns. +Druid can ingest JSON, CSV, TSV and other delimited data out of the box. Druid supports single dimension values, or multiple dimension values (an array of values). Druid supports long and float numeric columns. ## Not all of my events were ingested diff --git a/docs/content/ingestion/index.md b/docs/content/ingestion/index.md index 63f2a62bf656..c97a1793b731 100644 --- a/docs/content/ingestion/index.md +++ b/docs/content/ingestion/index.md @@ -36,7 +36,20 @@ An example dataSchema is shown below: "format" : "auto" }, "dimensionsSpec" : { - "dimensions": ["page","language","user","unpatrolled","newPage","robot","anonymous","namespace","continent","country","region","city"], + "dimensions": ["page","language","user","unpatrolled","newPage","robot","anonymous", + "namespace","continent","country","region", + {"name": "city", + "type": "string" + } + {"name": "user_id", + "type": "long" + }, + {"name": "coordinates", + "type": "spatial" + }, + {"name": "weighting_factor", + "type": "float" + }], "dimensionExclusions" : [], "spatialDimensions" : [] } @@ -169,9 +182,35 @@ handle all formatting decisions on their own, without using the ParseSpec. | Field | Type | Description | Required | |-------|------|-------------|----------| -| dimensions | JSON String array | The names of the dimensions. If this is an empty array, Druid will treat all columns that are not timestamp or metric columns as dimension columns. | yes | +| dimensions | JSON String/Object array | An array of Strings and/or DimensionSchema objects. If this is an empty array, Druid will treat all columns that are not timestamp or metric columns as dimension columns. | yes | | dimensionExclusions | JSON String array | The names of dimensions to exclude from ingestion. | no (default == [] | -| spatialDimensions | JSON Object array | An array of [spatial dimensions](../development/geo.html) | no (default == [] | +| spatialDimensions | JSON Object array | @Deprecated, use DimensionSchema objects instead. An array of [spatial dimensions](../development/geo.html) | no (default == [] | + +#### DimensionSchema + +A DimensionSchema defines the name, type, and additional properties of a dimension to be ingested. + +There are several supported types: "string", "long", "float", and "spatial". + +If "type" is not specified, or a String is provided in the dimensions array instead of a DimensionSchema object, a default String DimensionSchema will be used for that dimension. + +##### String/Long/Float DimensionSchema + +| Field | Type | Description | Required | +|-------|------|-------------|----------| +| name | string | The name of this dimension. | yes | +| type | string | The type of dimension | no (default == 'string') | + +##### Spatial DimensionSchema +See [spatial dimensions](../development/geo.html) + +A spatial dimension may be constructed from multiple other dimensions or it may already exist as part of an event. If a spatial dimension already exists, it must be an array of coordinate values. + +| Field | Type | Description | Required | +|-------|------|-------------|----------| +| name | string | The name of this dimension. | yes | +| type | string | The type of dimension | no (default == 'string') | +| dims | string array | A list of dimension names that comprise a spatial dimension.|no| ## GranularitySpec diff --git a/docs/content/ingestion/schema-changes.md b/docs/content/ingestion/schema-changes.md index 23c83f2bbf85..5f90bd1ec473 100644 --- a/docs/content/ingestion/schema-changes.md +++ b/docs/content/ingestion/schema-changes.md @@ -55,7 +55,7 @@ In this case, queries may hit a mixture of `v1` and `v2` segments. ## Different Schemas Among Segments -Druid segments for the same datasource may have different schemas. If a string column (dimension) exists in one segment but not +Druid segments for the same datasource may have different schemas. If a dimension exists in one segment but not another, queries that involve both segments still work. Queries for the segment missing the dimension will behave as if the dimension has only null values. -Similarly, if one segment has a numeric column (metric) but another does not, queries on the segment missing the +Similarly, if one segment has a metric but another does not, queries on the segment missing the metric will generally "do the right thing". Aggregations over this missing metric behave as if the metric were missing. diff --git a/docs/content/ingestion/schema-design.md b/docs/content/ingestion/schema-design.md index ecf1ec64fad7..9eb81c63c662 100644 --- a/docs/content/ingestion/schema-design.md +++ b/docs/content/ingestion/schema-design.md @@ -12,7 +12,7 @@ of OLAP data. For more detailed information: * Every row in Druid must have a timestamp. Data is always partitioned by time, and every query has a time filter. Query results can also be broken down by time buckets like minutes, hours, days, and so on. -* Dimensions are fields that can be filtered on or grouped by. They are always either single Strings or arrays of Strings. +* Dimensions are fields that can be filtered on or grouped by. Dimensions can have Long, Float, or String types. Each String dimension within a row can hold a single value or an array of values. Long and Float dimensions can only hold single values. * Metrics are fields that can be aggregated. They are often stored as numbers (integers or floats) but can also be stored as complex objects like HyperLogLog sketches or approximate histogram sketches. Typical production tables (or datasources as they are known in Druid) have fewer than 100 dimensions and fewer @@ -77,6 +77,8 @@ a dimension that has been excluded, or a metric column as a dimension. It should these segments will be slightly larger than if the list of dimensions was explicitly specified in lexicographic order. This limitation does not impact query correctness- just storage requirements. +All dimensions discovered through schema-less ingestion will have String type. Long and Float dimensions are not currently supported with schema-less ingestion. + ## Including the same column as a dimension and a metric One workflow with unique IDs is to be able to filter on a particular ID, while still being able to do fast unique counts on the ID column. diff --git a/docs/content/querying/aggregations.md b/docs/content/querying/aggregations.md index ec2b64a46735..a7692f198cd2 100644 --- a/docs/content/querying/aggregations.md +++ b/docs/content/querying/aggregations.md @@ -120,7 +120,7 @@ use than a native Java aggregator. Computes the cardinality of a set of Druid dimensions, using HyperLogLog to estimate the cardinality. Please note that this aggregator will be much slower than indexing a column with the hyperUnique aggregator. This aggregator also runs over a dimension column, which -means the string dimension cannot be removed from the dataset to improve rollup. In general, we strongly recommend using the hyperUnique aggregator +means the dimension cannot be removed from the dataset to improve rollup. In general, we strongly recommend using the hyperUnique aggregator instead of the cardinality aggregator if you do not care about the individual values of a dimension. ```json diff --git a/docs/content/querying/dimensionspecs.md b/docs/content/querying/dimensionspecs.md index 2150c01745cd..7e9208071818 100644 --- a/docs/content/querying/dimensionspecs.md +++ b/docs/content/querying/dimensionspecs.md @@ -35,13 +35,14 @@ Returns dimension values transformed using the given [extraction function](#extr Extraction functions define the transformation applied to each dimension value. -Transformations can be applied to both regular (string) dimensions, as well +Transformations can be applied to both regular (Long, Float, String) dimensions, as well as the special `__time` dimension, which represents the current time bucket according to the query [aggregation granularity](../querying/granularities.html). **Note**: for functions taking string values (such as regular expressions), `__time` dimension values will be formatted in [ISO-8601 format](https://en.wikipedia.org/wiki/ISO_8601) -before getting passed to the extraction function. +before getting passed to the extraction function. Long and Float dimension values will be converted to Strings +for these extraction functions. ### Regular Expression Extraction Function diff --git a/docs/content/querying/filters.md b/docs/content/querying/filters.md index 629151069f49..38bb2c345aeb 100644 --- a/docs/content/querying/filters.md +++ b/docs/content/querying/filters.md @@ -245,7 +245,6 @@ If dimension value starts with digit and contains a none digits comparing will b For instance suppose lower bound is `100` and value is `10K` the filter will match (`100 < 10K` returns `true`) since `K` is greater than any digit Now suppose that the lower bound is `110` the filter will not match (`110 < 10K` returns `false`) - #### Search Query Spec ##### Insensitive Contains diff --git a/docs/content/querying/segmentmetadataquery.md b/docs/content/querying/segmentmetadataquery.md index 89e8d183be2b..2e266bc129b2 100644 --- a/docs/content/querying/segmentmetadataquery.md +++ b/docs/content/querying/segmentmetadataquery.md @@ -54,14 +54,14 @@ The format of the result is: } ] ``` -Dimension columns will have type `STRING`. +Dimension columns will have type `LONG`, `FLOAT`, or `STRING`. Metric columns will have type `FLOAT` or `LONG` or name of the underlying complex type such as `hyperUnique` in case of COMPLEX metric. Timestamp column will have type `LONG`. If the `errorMessage` field is non-null, you should not trust the other fields in the response. Their contents are undefined. -Only columns which are dimensions (ie, have type `STRING`) will have any cardinality. Rest of the columns (timestamp and metric columns) will show cardinality as `null`. +Only columns which are dimensions with type `STRING` will have any cardinality. Rest of the columns (timestamp, numeric dimension, and metric columns) will show cardinality as `null`. ### intervals diff --git a/indexing-hadoop/src/main/java/io/druid/indexer/IndexGeneratorJob.java b/indexing-hadoop/src/main/java/io/druid/indexer/IndexGeneratorJob.java index 414a9a4e3cc8..bf9911046f65 100644 --- a/indexing-hadoop/src/main/java/io/druid/indexer/IndexGeneratorJob.java +++ b/indexing-hadoop/src/main/java/io/druid/indexer/IndexGeneratorJob.java @@ -46,6 +46,8 @@ import io.druid.segment.BaseProgressIndicator; import io.druid.segment.ProgressIndicator; import io.druid.segment.QueryableIndex; +import io.druid.segment.column.ColumnCapabilities; +import io.druid.segment.column.ColumnCapabilitiesImpl; import io.druid.segment.incremental.IncrementalIndex; import io.druid.segment.incremental.IncrementalIndexSchema; import io.druid.segment.incremental.OnheapIncrementalIndex; @@ -77,6 +79,7 @@ import java.util.Iterator; import java.util.LinkedHashSet; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.concurrent.BlockingQueue; import java.util.concurrent.ExecutionException; @@ -216,7 +219,8 @@ private static IncrementalIndex makeIncrementalIndex( Bucket theBucket, AggregatorFactory[] aggs, HadoopDruidIndexerConfig config, - Iterable oldDimOrder + Iterable oldDimOrder, + Map oldCapabilities ) { final HadoopTuningConfig tuningConfig = config.getSchema().getTuningConfig(); @@ -234,7 +238,7 @@ private static IncrementalIndex makeIncrementalIndex( ); if (oldDimOrder != null && !indexSchema.getDimensionsSpec().hasCustomDimensions()) { - newIndex.loadDimensionIterable(oldDimOrder); + newIndex.loadDimensionIterable(oldDimOrder, oldCapabilities); } return newIndex; @@ -337,7 +341,7 @@ protected void reduce( LinkedHashSet dimOrder = Sets.newLinkedHashSet(); SortableBytes keyBytes = SortableBytes.fromBytesWritable(key); Bucket bucket = Bucket.fromGroupKey(keyBytes.getGroupKey()).lhs; - IncrementalIndex index = makeIncrementalIndex(bucket, combiningAggs, config, null); + IncrementalIndex index = makeIncrementalIndex(bucket, combiningAggs, config, null, null); index.add(InputRowSerde.fromBytes(first.getBytes(), aggregators)); while (iter.hasNext()) { @@ -348,7 +352,7 @@ protected void reduce( dimOrder.addAll(index.getDimensionOrder()); log.info("current index full due to [%s]. creating new index.", index.getOutOfRowsReason()); flushIndexToContextAndClose(key, index, context); - index = makeIncrementalIndex(bucket, combiningAggs, config, dimOrder); + index = makeIncrementalIndex(bucket, combiningAggs, config, dimOrder, index.getColumnCapabilities()); } index.add(value); @@ -552,6 +556,7 @@ protected void reduce( bucket, combiningAggs, config, + null, null ); try { @@ -649,7 +654,8 @@ public void doRun() bucket, combiningAggs, config, - allDimensionNames + allDimensionNames, + persistIndex.getColumnCapabilities() ); startTime = System.currentTimeMillis(); ++indexCount; diff --git a/indexing-hadoop/src/main/java/io/druid/indexer/InputRowSerde.java b/indexing-hadoop/src/main/java/io/druid/indexer/InputRowSerde.java index b179faefb730..447f721f6342 100644 --- a/indexing-hadoop/src/main/java/io/druid/indexer/InputRowSerde.java +++ b/indexing-hadoop/src/main/java/io/druid/indexer/InputRowSerde.java @@ -88,7 +88,8 @@ public InputRow get() IncrementalIndex.makeColumnSelectorFactory( aggFactory, supplier, - true + true, + null ) ); agg.aggregate(); diff --git a/indexing-service/src/test/java/io/druid/indexing/firehose/IngestSegmentFirehoseFactoryTest.java b/indexing-service/src/test/java/io/druid/indexing/firehose/IngestSegmentFirehoseFactoryTest.java index ae94b5376d7f..ed4228c4a593 100644 --- a/indexing-service/src/test/java/io/druid/indexing/firehose/IngestSegmentFirehoseFactoryTest.java +++ b/indexing-service/src/test/java/io/druid/indexing/firehose/IngestSegmentFirehoseFactoryTest.java @@ -38,6 +38,7 @@ import com.metamx.emitter.service.ServiceEventBuilder; import io.druid.common.utils.JodaUtils; import io.druid.data.input.InputRow; +import io.druid.data.input.impl.DimensionSchema; import io.druid.data.input.impl.DimensionsSpec; import io.druid.data.input.impl.InputRowParser; import io.druid.data.input.impl.JSONParseSpec; diff --git a/processing/src/main/java/io/druid/query/BaseQuery.java b/processing/src/main/java/io/druid/query/BaseQuery.java index 69f297a48618..b395ca092068 100644 --- a/processing/src/main/java/io/druid/query/BaseQuery.java +++ b/processing/src/main/java/io/druid/query/BaseQuery.java @@ -27,6 +27,7 @@ import com.metamx.common.ISE; import com.metamx.common.guava.Sequence; import io.druid.query.spec.QuerySegmentSpec; +import io.druid.segment.column.ValueType; import org.joda.time.Duration; import org.joda.time.Interval; @@ -67,6 +68,11 @@ public static int getContextUncoveredIntervalsLimit(Query query, int defa return parseInt(query, "uncoveredIntervalsLimit", defaultValue); } + public static Map getContextTypeHints(Query query) + { + return query.getContextValue("typeHints"); + } + private static int parseInt(Query query, String key, int defaultValue) { Object val = query.getContextValue(key); diff --git a/processing/src/main/java/io/druid/query/aggregation/FilteredAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/FilteredAggregatorFactory.java index 30bfb9d72d2e..907ea171b44b 100644 --- a/processing/src/main/java/io/druid/query/aggregation/FilteredAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/FilteredAggregatorFactory.java @@ -28,7 +28,9 @@ import io.druid.query.filter.ValueMatcherFactory; import io.druid.segment.ColumnSelectorFactory; import io.druid.segment.DimensionSelector; +import io.druid.segment.data.IndexedFloats; import io.druid.segment.data.IndexedInts; +import io.druid.segment.data.IndexedLongs; import io.druid.segment.filter.BooleanValueMatcher; import io.druid.segment.filter.Filters; @@ -232,23 +234,49 @@ public ValueMatcher makeValueMatcher(final String dimension, final Comparable va if (selector == null) { return new BooleanValueMatcher(isNullOrEmpty); } - - final int valueId = selector.lookupId(valueString); - return new ValueMatcher() - { - @Override - public boolean matches() - { - final IndexedInts row = selector.getRow(); - final int size = row.size(); - for (int i = 0; i < size; ++i) { - if (row.get(i) == valueId) { - return true; + switch (selector.getDimCapabilities().getType()) { + case STRING: + final int valueId = selector.lookupId(valueString); + return new ValueMatcher() + { + @Override + public boolean matches() + { + final IndexedInts row = selector.getRow(); + final int size = row.size(); + for (int i = 0; i < size; ++i) { + if (row.get(i) == valueId) { + return true; + } + } + return false; } - } - return false; - } - }; + }; + case LONG: + final long parsedLong = Long.parseLong(valueString); + return new ValueMatcher() + { + @Override + public boolean matches() + { + final IndexedLongs row = selector.getLongRow(); + return row.get(0) == parsedLong; + } + }; + case FLOAT: + final float parsedFloat = Float.parseFloat(valueString); + return new ValueMatcher() + { + @Override + public boolean matches() + { + final IndexedFloats row = selector.getFloatRow(); + return row.get(0) == parsedFloat; + } + }; + default: + throw new UnsupportedOperationException("Invalid type: " + selector.getDimCapabilities().getType()); + } } @Override @@ -261,31 +289,67 @@ public ValueMatcher makeValueMatcher(final String dimension, final Predicate pre if (selector == null) { return new BooleanValueMatcher(predicate.apply(null)); } - - // Check every value in the dimension, as a String. - final int cardinality = selector.getValueCardinality(); - final BitSet valueIds = new BitSet(cardinality); - for (int i = 0; i < cardinality; i++) { - if (predicate.apply(selector.lookupName(i))) { - valueIds.set(i); - } - } - - return new ValueMatcher() - { - @Override - public boolean matches() - { - final IndexedInts row = selector.getRow(); - final int size = row.size(); - for (int i = 0; i < size; ++i) { - if (valueIds.get(row.get(i))) { - return true; + switch (selector.getDimCapabilities().getType()) { + case STRING: + // Check every value in the dimension, as a String. + final int cardinality = selector.getValueCardinality(); + final BitSet valueIds = new BitSet(cardinality); + for (int i = 0; i < cardinality; i++) { + if (predicate.apply(selector.lookupName(i))) { + valueIds.set(i); } } - return false; - } - }; + + return new ValueMatcher() + { + @Override + public boolean matches() + { + final IndexedInts row = selector.getRow(); + final int size = row.size(); + for (int i = 0; i < size; ++i) { + if (valueIds.get(row.get(i))) { + return true; + } + } + return false; + } + }; + case LONG: + return new ValueMatcher() + { + @Override + public boolean matches() + { + final IndexedLongs row = selector.getLongRow(); + final int size = row.size(); + for (int i = 0; i < size; ++i) { + if (predicate.apply(row.get(i))) { + return true; + } + } + return false; + } + }; + case FLOAT: + return new ValueMatcher() + { + @Override + public boolean matches() + { + final IndexedFloats row = selector.getFloatRow(); + final int size = row.size(); + for (int i = 0; i < size; ++i) { + if (predicate.apply(row.get(i))) { + return true; + } + } + return false; + } + }; + default: + throw new UnsupportedOperationException("Invalid type: " + selector.getDimCapabilities().getType()); + } } } } diff --git a/processing/src/main/java/io/druid/query/aggregation/cardinality/CardinalityAggregator.java b/processing/src/main/java/io/druid/query/aggregation/cardinality/CardinalityAggregator.java index 3ba2c15f3621..456100c2bb60 100644 --- a/processing/src/main/java/io/druid/query/aggregation/cardinality/CardinalityAggregator.java +++ b/processing/src/main/java/io/druid/query/aggregation/cardinality/CardinalityAggregator.java @@ -22,10 +22,17 @@ import com.google.common.hash.HashFunction; import com.google.common.hash.Hasher; import com.google.common.hash.Hashing; +import com.google.common.primitives.Floats; +import com.google.common.primitives.Ints; +import com.google.common.primitives.Longs; +import com.metamx.common.IAE; import io.druid.query.aggregation.Aggregator; import io.druid.query.aggregation.hyperloglog.HyperLogLogCollector; import io.druid.segment.DimensionSelector; +import io.druid.segment.column.ValueType; +import io.druid.segment.data.IndexedFloats; import io.druid.segment.data.IndexedInts; +import io.druid.segment.data.IndexedLongs; import java.util.Arrays; import java.util.List; @@ -49,26 +56,56 @@ protected static void hashRow(List selectorList, HyperLogLogC hasher.putByte((byte) 0); } final DimensionSelector selector = selectorList.get(k); - final IndexedInts row = selector.getRow(); - final int size = row.size(); - // nothing to add to hasher if size == 0, only handle size == 1 and size != 0 cases. - if (size == 1) { - final String value = selector.lookupName(row.get(0)); - hasher.putUnencodedChars(value != null ? value : NULL_STRING); - } else if (size != 0) { - final String[] values = new String[size]; - for (int i = 0; i < size; ++i) { - final String value = selector.lookupName(row.get(i)); - values[i] = value != null ? value : NULL_STRING; - } - // Values need to be sorted to ensure consistent multi-value ordering across different segments - Arrays.sort(values); - for (int i = 0; i < size; ++i) { - if (i != 0) { - hasher.putChar(SEPARATOR); + int rowSize = 0; + IndexedInts intVals = null; + IndexedLongs longVals = null; + IndexedFloats floatVals = null; + Comparable objVal = null; + ValueType type = selector.getDimCapabilities().getType(); + + switch (type) { + case STRING: + intVals = selector.getRow(); + rowSize = intVals.size(); + if (rowSize == 1) { + final String value = selector.lookupName(intVals.get(0)); + hasher.putUnencodedChars(value != null ? value : NULL_STRING); + } else if (rowSize != 0) { + final String[] values = new String[rowSize]; + for (int i = 0; i < rowSize; ++i) { + final String value = selector.lookupName(intVals.get(i)); + values[i] = value != null ? value : NULL_STRING; + } + // Values need to be sorted to ensure consistent multi-value ordering across different segments + Arrays.sort(values); + for (int i = 0; i < rowSize; ++i) { + if (i != 0) { + hasher.putChar(SEPARATOR); + } + hasher.putUnencodedChars(values[i]); + } } - hasher.putUnencodedChars(values[i]); - } + break; + case LONG: + longVals = selector.getLongRow(); + rowSize = longVals.size(); + if (rowSize > 0) { + hasher.putLong(longVals.get(0)); + } + break; + case FLOAT: + floatVals = selector.getFloatRow(); + rowSize = floatVals.size(); + if (rowSize > 0) { + hasher.putFloat(floatVals.get(0)); + } + break; + case COMPLEX: + objVal = selector.getComparableRow(); + hasher.putUnencodedChars(objVal != null ? objVal.toString() : NULL_STRING); + break; + default: + throw new IAE("Invalid type: " + selector.getDimCapabilities().getType()); } } collector.add(hasher.hash().asBytes()); @@ -77,9 +114,26 @@ protected static void hashRow(List selectorList, HyperLogLogC protected static void hashValues(final List selectors, HyperLogLogCollector collector) { for (final DimensionSelector selector : selectors) { - for (final Integer index : selector.getRow()) { - final String value = selector.lookupName(index); - collector.add(hashFn.hashUnencodedChars(value == null ? NULL_STRING : value).asBytes()); + switch(selector.getDimCapabilities().getType()) { + case STRING: + for (final Integer index : selector.getRow()) { + final String value = selector.lookupName(index); + collector.add(hashFn.hashUnencodedChars(value == null ? NULL_STRING : value).asBytes()); + } + break; + case LONG: + long longVal = selector.getLongRow().get(0); + collector.add(hashFn.hashBytes(Longs.toByteArray(longVal)).asBytes()); + break; + case FLOAT: + float floatVal = selector.getFloatRow().get(0); + collector.add(hashFn.hashBytes(Ints.toByteArray(Float.floatToIntBits(floatVal))).asBytes()); + break; + case COMPLEX: + //TODO: pass in handler and call get Bytes + break; + default: + break; } } } diff --git a/processing/src/main/java/io/druid/query/aggregation/hyperloglog/HyperUniquesAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/hyperloglog/HyperUniquesAggregatorFactory.java index cc8ff7ba431c..ba9610ad4f2e 100644 --- a/processing/src/main/java/io/druid/query/aggregation/hyperloglog/HyperUniquesAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/hyperloglog/HyperUniquesAggregatorFactory.java @@ -112,7 +112,7 @@ public Comparator getComparator() public int compare(HyperLogLogCollector lhs, HyperLogLogCollector rhs) { if(lhs == null) { - return -1; + return rhs == null ? 0 : -1; } if(rhs == null) { return 1; diff --git a/processing/src/main/java/io/druid/query/dimension/BaseFilteredDimensionSpec.java b/processing/src/main/java/io/druid/query/dimension/BaseFilteredDimensionSpec.java index eb3a8010481c..539092faf204 100644 --- a/processing/src/main/java/io/druid/query/dimension/BaseFilteredDimensionSpec.java +++ b/processing/src/main/java/io/druid/query/dimension/BaseFilteredDimensionSpec.java @@ -21,9 +21,13 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Preconditions; +import com.google.common.base.Predicate; import io.druid.query.extraction.ExtractionFn; import io.druid.segment.DimensionSelector; +import io.druid.segment.column.ColumnCapabilities; +import io.druid.segment.data.IndexedFloats; import io.druid.segment.data.IndexedInts; +import io.druid.segment.data.IndexedLongs; import io.druid.segment.data.ListBasedIndexedInts; import java.util.ArrayList; @@ -117,6 +121,50 @@ public int lookupId(String name) { return forwardMapping.get(selector.lookupId(name)); } + + @Override + public IndexedLongs getLongRow() + { + throw new UnsupportedOperationException("getLongRow() is not supported."); + } + + @Override + public Comparable getExtractedValueLong(long val) + { + throw new UnsupportedOperationException("getExtractedValueLong() is not supported."); + } + + @Override + public IndexedFloats getFloatRow() + { + throw new UnsupportedOperationException("getFloatRow() is not supported."); + } + + @Override + public Comparable getExtractedValueFloat(float val) + { + throw new UnsupportedOperationException("getExtractedValueFloat() is not supported."); + } + + @Override + public Comparable getComparableRow() + { + throw new UnsupportedOperationException("getComparableRow() is not supported."); + } + + @Override + public Comparable getExtractedValueComparable(Comparable val) + { + throw new UnsupportedOperationException("getExtractedValueComparable() is not supported."); + + } + + @Override + public ColumnCapabilities getDimCapabilities() + { + return selector.getDimCapabilities(); + } + }; } } diff --git a/processing/src/main/java/io/druid/query/dimension/ListFilteredDimensionSpec.java b/processing/src/main/java/io/druid/query/dimension/ListFilteredDimensionSpec.java index 1f303bb7a055..a6610cc51781 100644 --- a/processing/src/main/java/io/druid/query/dimension/ListFilteredDimensionSpec.java +++ b/processing/src/main/java/io/druid/query/dimension/ListFilteredDimensionSpec.java @@ -74,6 +74,11 @@ public DimensionSelector decorate(final DimensionSelector selector) return selector; } + if (!selector.getDimCapabilities().isDictionaryEncoded()) { + String errmsg = "ListFilteredDimensionSpec is not supported for non-dictionary encoded columns."; + throw new UnsupportedOperationException(errmsg); + } + int selectorCardinality = selector.getValueCardinality(); int cardinality = isWhitelist ? values.size() : selectorCardinality - values.size(); diff --git a/processing/src/main/java/io/druid/query/dimension/RegexFilteredDimensionSpec.java b/processing/src/main/java/io/druid/query/dimension/RegexFilteredDimensionSpec.java index db3548b76f64..9ffbd57c1847 100644 --- a/processing/src/main/java/io/druid/query/dimension/RegexFilteredDimensionSpec.java +++ b/processing/src/main/java/io/druid/query/dimension/RegexFilteredDimensionSpec.java @@ -65,6 +65,11 @@ public DimensionSelector decorate(final DimensionSelector selector) return selector; } + if (!selector.getDimCapabilities().isDictionaryEncoded()) { + String errmsg = "RegexFilteredDimensionSpec is not supported for non-dictionary encoded columns."; + throw new UnsupportedOperationException(errmsg); + } + int count = 0; final Map forwardMapping = new HashMap<>(); diff --git a/processing/src/main/java/io/druid/query/filter/BitmapIndexSelector.java b/processing/src/main/java/io/druid/query/filter/BitmapIndexSelector.java index 61fe6c59bafa..3f5274d15dce 100644 --- a/processing/src/main/java/io/druid/query/filter/BitmapIndexSelector.java +++ b/processing/src/main/java/io/druid/query/filter/BitmapIndexSelector.java @@ -19,18 +19,30 @@ package io.druid.query.filter; +import com.google.common.base.Predicate; import com.metamx.collections.bitmap.BitmapFactory; import com.metamx.collections.bitmap.ImmutableBitmap; import com.metamx.collections.spatial.ImmutableRTree; +import com.metamx.collections.spatial.search.Bound; +import io.druid.segment.column.ValueType; import io.druid.segment.data.Indexed; /** */ public interface BitmapIndexSelector { + public boolean hasBitmapIndexes(String dimension); + public ValueType getDimensionType(String dimension); + + // for scans on dims that don't have bitmap indexes + public ImmutableBitmap getBitmapIndexFromColumnScan(String dimension, Predicate predicate); + + // for scans with bitmap indexes public Indexed getDimensionValues(String dimension); public int getNumRows(); public BitmapFactory getBitmapFactory(); public ImmutableBitmap getBitmapIndex(String dimension, String value); - public ImmutableRTree getSpatialIndex(String dimension); + + // for scans with spatial indexes + public ImmutableBitmap getBitmapIndex(String dimension, Bound bound); } diff --git a/processing/src/main/java/io/druid/query/groupby/GroupByQueryEngine.java b/processing/src/main/java/io/druid/query/groupby/GroupByQueryEngine.java index 110d8bef6d7c..96183af26099 100644 --- a/processing/src/main/java/io/druid/query/groupby/GroupByQueryEngine.java +++ b/processing/src/main/java/io/druid/query/groupby/GroupByQueryEngine.java @@ -26,6 +26,7 @@ import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.primitives.Ints; +import com.google.common.primitives.Longs; import com.google.inject.Inject; import com.metamx.common.IAE; import com.metamx.common.ISE; @@ -45,9 +46,15 @@ import io.druid.query.aggregation.PostAggregator; import io.druid.query.dimension.DimensionSpec; import io.druid.segment.Cursor; +import io.druid.segment.DimensionHandler; import io.druid.segment.DimensionSelector; import io.druid.segment.StorageAdapter; +import io.druid.segment.column.Column; +import io.druid.segment.column.ColumnCapabilities; +import io.druid.segment.column.ValueType; +import io.druid.segment.data.IndexedFloats; import io.druid.segment.data.IndexedInts; +import io.druid.segment.data.IndexedLongs; import io.druid.segment.filter.Filters; import org.joda.time.DateTime; import org.joda.time.Interval; @@ -57,6 +64,7 @@ import java.io.IOException; import java.nio.ByteBuffer; import java.util.ArrayList; +import java.util.Comparator; import java.util.Iterator; import java.util.List; import java.util.Map; @@ -71,6 +79,62 @@ public class GroupByQueryEngine private final Supplier config; private final StupidPool intermediateResultsBufferPool; + private static void getBytesFromBuffer(ByteBuffer src, byte[] dst, int srcOffset, int readLen) + { + for (int i = 0; i < readLen; i++) { + dst[i] = src.get(srcOffset + i); + } + } + + private static final Comparator makeKeyComparator(final List dimHandlers) + { + Comparator comp = new Comparator() + { + @Override + public int compare(ByteBuffer o1, ByteBuffer o2) + { + int pos = 0; + int limit = o1.limit(); + int ret = 0; + int dimIndex = 0; + int maxDimIndex = dimHandlers.size(); + + while (pos < limit && dimIndex < maxDimIndex) { + DimensionHandler handler = dimHandlers.get(dimIndex); + if (handler == null) { + // Special time column handling + int int1 = o1.getInt(pos); + int int2 = o2.getInt(pos); + pos += Ints.BYTES; + ret = Integer.compare(int1, int2); + } else { + int valLen = handler.getEncodedValueSize(); + byte[] bytes1 = new byte[valLen]; + byte[] bytes2 = new byte[valLen]; + + getBytesFromBuffer(o1, bytes1, pos, valLen); + getBytesFromBuffer(o2, bytes2, pos, valLen); + + pos += valLen; + + Comparator encodedComparator = handler.getEncodedComparator(); + Comparable val1 = handler.getRowValueFromBytes(bytes1); + Comparable val2 = handler.getRowValueFromBytes(bytes2); + ret = encodedComparator.compare(val1, val2); + } + + if (ret != 0) { + return ret; + } + + dimIndex++; + } + return ret; + } + }; + return comp; + } + @Inject public GroupByQueryEngine( Supplier config, @@ -101,6 +165,8 @@ public Sequence process(final GroupByQuery query, final StorageAdapter stor false ); + final Map dimHandlers = storageAdapter.getDimensionHandlers(); + final ResourceHolder bufferHolder = intermediateResultsBufferPool.take(); return Sequences.concat( @@ -118,7 +184,7 @@ public Sequence apply(final Cursor cursor) @Override public RowIterator make() { - return new RowIterator(query, cursor, bufferHolder.get(), config.get()); + return new RowIterator(query, cursor, bufferHolder.get(), config.get(), storageAdapter); } @Override @@ -148,20 +214,25 @@ private static class RowUpdater private final ByteBuffer metricValues; private final BufferAggregator[] aggregators; private final PositionMaintainer positionMaintainer; + private final StorageAdapter adapter; - private final Map positions = Maps.newTreeMap(); + private final Map positions; // GroupBy queries tend to do a lot of reads from this. We co-store a hash map to make those reads go faster. private final Map positionsHash = Maps.newHashMap(); public RowUpdater( ByteBuffer metricValues, BufferAggregator[] aggregators, - PositionMaintainer positionMaintainer + PositionMaintainer positionMaintainer, + StorageAdapter adapter, + List dimHandlers ) { this.metricValues = metricValues; this.aggregators = aggregators; this.positionMaintainer = positionMaintainer; + this.adapter = adapter; + this.positions = Maps.newTreeMap(makeKeyComparator(dimHandlers)); } public int getNumRows() @@ -176,7 +247,9 @@ public Map getPositions() private List updateValues( ByteBuffer key, - List dims + List dims, + final List dimNames, + final List dimHandlers ) { if (dims.size() > 0) { @@ -184,18 +257,128 @@ private List updateValues( List unaggregatedBuffers = null; final DimensionSelector dimSelector = dims.get(0); - final IndexedInts row = dimSelector.getRow(); - if (row == null || row.size() == 0) { - ByteBuffer newKey = key.duplicate(); - newKey.putInt(dimSelector.getValueCardinality()); - unaggregatedBuffers = updateValues(newKey, dims.subList(1, dims.size())); + final DimensionHandler dimHandler = dimHandlers.get(0); + final ColumnCapabilities capabilities = adapter.getColumnCapabilities(dimNames.get(0)); + int rowSize = 0; + IndexedInts intVals = null; + IndexedLongs longVals = null; + IndexedFloats floatVals = null; + Comparable objVal = null; + ValueType type = null; + + if (dimHandler != null) { + type = capabilities.getType(); + switch (type) { + case STRING: + intVals = dimSelector.getRow(); + rowSize = intVals == null ? 0 : intVals.size(); + break; + case LONG: + longVals = dimSelector.getLongRow(); + rowSize = longVals.size(); + break; + case FLOAT: + floatVals = dimSelector.getFloatRow(); + rowSize = floatVals.size(); + break; + case COMPLEX: + objVal = dimSelector.getComparableRow(); + rowSize = 1; + break; + default: + break; + } } else { - for (Integer dimValue : row) { + // Dimension does not exist, dimSelector is a NullDimensionSelector. + // The null value WILL appear in the result rows. + // tests in GroupByQueryRunnerTest expect this behavior. (e.g., testGroupByWithMetricColumnDisappears()) + type = ValueType.STRING; + intVals = dimSelector.getRow(); + rowSize = intVals.size(); + } + + if(dimNames.get(0).equals(Column.TIME_COLUMN_NAME)) { + type = ValueType.STRING; + intVals = dimSelector.getRow(); + rowSize = intVals.size(); + } + + switch (type) { + case STRING: + if (rowSize == 0) { + ByteBuffer newKey = key.duplicate(); + newKey.putInt(dimSelector.getValueCardinality()); + unaggregatedBuffers = updateValues(newKey, + dims.subList(1, dims.size()), + dimNames.subList(1, dimNames.size()), + dimHandlers.subList(1, dimHandlers.size()) + ); + } + + for (int i = 0; i < rowSize; i++) { + int dimValue = intVals.get(i); + ByteBuffer newKey = key.duplicate(); + byte[] valBytes = Ints.toByteArray(dimValue); + for (byte byt : valBytes) { + newKey.put(byt); + } + unaggregatedBuffers = updateValues( + newKey, + dims.subList(1, dims.size()), + dimNames.subList(1, dimNames.size()), + dimHandlers.subList(1, dimHandlers.size()) + ); + } + break; + case LONG: + for (int i = 0; i < rowSize; i++) { + long longVal = longVals.get(i); + ByteBuffer newKey = key.duplicate(); + byte[] valBytes = Longs.toByteArray(longVal); + for (byte byt : valBytes) { + newKey.put(byt); + } + unaggregatedBuffers = updateValues( + newKey, + dims.subList(1, dims.size()), + dimNames.subList(1, dimNames.size()), + dimHandlers.subList(1, dimHandlers.size()) + ); + } + break; + case FLOAT: + for (int i = 0; i < rowSize; i++) { + float floatVal = floatVals.get(i); + ByteBuffer newKey = key.duplicate(); + byte[] valBytes = Ints.toByteArray(Float.floatToIntBits(floatVal)); + for (byte byt : valBytes) { + newKey.put(byt); + } + unaggregatedBuffers = updateValues( + newKey, + dims.subList(1, dims.size()), + dimNames.subList(1, dimNames.size()), + dimHandlers.subList(1, dimHandlers.size()) + ); + } + break; + case COMPLEX: ByteBuffer newKey = key.duplicate(); - newKey.putInt(dimValue); - unaggregatedBuffers = updateValues(newKey, dims.subList(1, dims.size())); - } + byte[] valBytes = dimHandler.getBytesFromRowValue(objVal); + for (byte byt : valBytes) { + newKey.put(byt); + } + unaggregatedBuffers = updateValues( + newKey, + dims.subList(1, dims.size()), + dimNames.subList(1, dimNames.size()), + dimHandlers.subList(1, dimHandlers.size()) + ); + break; + default: + throw new IAE("Invalid type: " + type); } + if (unaggregatedBuffers != null) { if (retVal == null) { retVal = Lists.newArrayList(); @@ -296,19 +479,24 @@ private static class RowIterator implements CloseableIterator private final List dimensionSpecs; private final List dimensions; private final ArrayList dimNames; + private final ArrayList originalDimNames; private final List aggregatorSpecs; private final BufferAggregator[] aggregators; private final String[] metricNames; private final int[] sizesRequired; + private final StorageAdapter adapter; + private final List dimHandlers; private List unprocessedKeys; private Iterator delegate; + private int keyBufferSize; - public RowIterator(GroupByQuery query, final Cursor cursor, ByteBuffer metricsBuffer, GroupByQueryConfig config) + public RowIterator(GroupByQuery query, final Cursor cursor, ByteBuffer metricsBuffer, GroupByQueryConfig config, StorageAdapter adapter) { this.query = query; this.cursor = cursor; this.metricsBuffer = metricsBuffer; + this.adapter = adapter; this.maxIntermediateRows = Math.min( query.getContextValue( @@ -322,13 +510,26 @@ public RowIterator(GroupByQuery query, final Cursor cursor, ByteBuffer metricsBu dimensionSpecs = query.getDimensions(); dimensions = Lists.newArrayListWithExpectedSize(dimensionSpecs.size()); dimNames = Lists.newArrayListWithExpectedSize(dimensionSpecs.size()); + originalDimNames = Lists.newArrayListWithExpectedSize(dimensionSpecs.size()); + dimHandlers = Lists.newArrayListWithExpectedSize(dimensionSpecs.size()); + + Map handlerMap = adapter.getDimensionHandlers(); for (int i = 0; i < dimensionSpecs.size(); ++i) { final DimensionSpec dimSpec = dimensionSpecs.get(i); final DimensionSelector selector = cursor.makeDimensionSelector(dimSpec); if (selector != null) { + String dimName = dimSpec.getDimension(); + DimensionHandler handler = handlerMap.get(dimName); dimensions.add(selector); dimNames.add(dimSpec.getOutputName()); + originalDimNames.add(dimName); + dimHandlers.add(handlerMap.get(dimName)); + if (dimName.equals(Column.TIME_COLUMN_NAME) || handler == null) { + keyBufferSize += Ints.BYTES; + } else { + keyBufferSize += handler.getEncodedValueSize(); + } } } @@ -342,6 +543,7 @@ public RowIterator(GroupByQuery query, final Cursor cursor, ByteBuffer metricsBu metricNames[i] = aggregatorSpec.getName(); sizesRequired[i] = aggregatorSpec.getMaxIntermediateSize(); } + } @Override @@ -362,10 +564,14 @@ public Row next() } final PositionMaintainer positionMaintainer = new PositionMaintainer(0, sizesRequired, metricsBuffer.remaining()); - final RowUpdater rowUpdater = new RowUpdater(metricsBuffer, aggregators, positionMaintainer); + final RowUpdater rowUpdater = new RowUpdater(metricsBuffer, aggregators, positionMaintainer, adapter, dimHandlers); if (unprocessedKeys != null) { for (ByteBuffer key : unprocessedKeys) { - final List unprocUnproc = rowUpdater.updateValues(key, ImmutableList.of()); + final List unprocUnproc = rowUpdater.updateValues(key, + ImmutableList.of(), + ImmutableList.of(), + ImmutableList.of() + ); if (unprocUnproc != null) { throw new ISE("Not enough memory to process the request."); } @@ -373,13 +579,11 @@ public Row next() cursor.advance(); } while (!cursor.isDone() && rowUpdater.getNumRows() < maxIntermediateRows) { - ByteBuffer key = ByteBuffer.allocate(dimensions.size() * Ints.BYTES); - - unprocessedKeys = rowUpdater.updateValues(key, dimensions); + ByteBuffer key = ByteBuffer.allocate(keyBufferSize); + unprocessedKeys = rowUpdater.updateValues(key, dimensions, originalDimNames, dimHandlers); if (unprocessedKeys != null) { break; } - cursor.advance(); } @@ -404,11 +608,47 @@ public Row apply(@Nullable Map.Entry input) Map theEvent = Maps.newLinkedHashMap(); ByteBuffer keyBuffer = input.getKey().duplicate(); + int pos = 0; for (int i = 0; i < dimensions.size(); ++i) { + final ColumnCapabilities capabilities = adapter.getColumnCapabilities(originalDimNames.get(i)); + final DimensionHandler dimHandler = dimHandlers.get(i); final DimensionSelector dimSelector = dimensions.get(i); - final int dimVal = keyBuffer.getInt(); - if (dimSelector.getValueCardinality() != dimVal) { - theEvent.put(dimNames.get(i), dimSelector.lookupName(dimVal)); + + ValueType type; + int valLen; + if (originalDimNames.get(i).equals(Column.TIME_COLUMN_NAME) || dimHandler == null) { + type = ValueType.STRING; + valLen = Ints.BYTES; + } else { + type = capabilities.getType(); + valLen = dimHandler.getEncodedValueSize(); + } + + byte[] valBytes = new byte[valLen]; + getBytesFromBuffer(keyBuffer, valBytes, pos, valLen); + pos += valLen; + + switch (type) { + case STRING: + int dimVal = Ints.fromByteArray(valBytes); + if (dimVal != dimSelector.getValueCardinality()) { + theEvent.put(dimNames.get(i), dimSelector.lookupName(dimVal)); + } + break; + case LONG: + long longVal = Longs.fromByteArray(valBytes); + theEvent.put(dimNames.get(i), dimSelector.getExtractedValueLong(longVal)); + break; + case FLOAT: + float floatVal = Float.intBitsToFloat(Ints.fromByteArray(valBytes)); + theEvent.put(dimNames.get(i), dimSelector.getExtractedValueFloat(floatVal)); + break; + case COMPLEX: + Comparable objVal = dimHandler.getRowValueFromBytes(valBytes); + theEvent.put(dimNames.get(i), dimSelector.getExtractedValueComparable(objVal)); + break; + default: + throw new IAE("Invalid type: " + dimSelector.getDimCapabilities().getType()); } } diff --git a/processing/src/main/java/io/druid/query/groupby/GroupByQueryHelper.java b/processing/src/main/java/io/druid/query/groupby/GroupByQueryHelper.java index 637dce3f077d..0574faabfdd8 100644 --- a/processing/src/main/java/io/druid/query/groupby/GroupByQueryHelper.java +++ b/processing/src/main/java/io/druid/query/groupby/GroupByQueryHelper.java @@ -28,8 +28,10 @@ import io.druid.data.input.MapBasedInputRow; import io.druid.data.input.MapBasedRow; import io.druid.granularity.QueryGranularity; +import io.druid.query.BaseQuery; import io.druid.query.aggregation.AggregatorFactory; import io.druid.query.dimension.DimensionSpec; +import io.druid.segment.column.ValueType; import io.druid.segment.incremental.IncrementalIndex; import io.druid.segment.incremental.IndexSizeExceededException; import io.druid.segment.incremental.OffheapIncrementalIndex; @@ -37,6 +39,7 @@ import java.nio.ByteBuffer; import java.util.List; +import java.util.Map; import java.util.Queue; import java.util.concurrent.ConcurrentLinkedQueue; @@ -111,6 +114,13 @@ public String apply(DimensionSpec input) ); } + Map typeHints = BaseQuery.getContextTypeHints(query); + if (typeHints != null) { + for (String dimName : typeHints.keySet()) { + index.loadDimensionCapability(dimName, typeHints.get(dimName)); + } + } + Accumulator accumulator = new Accumulator() { @Override diff --git a/processing/src/main/java/io/druid/query/ordering/StringComparators.java b/processing/src/main/java/io/druid/query/ordering/StringComparators.java index ecd31bc884f0..848e18c857f5 100644 --- a/processing/src/main/java/io/druid/query/ordering/StringComparators.java +++ b/processing/src/main/java/io/druid/query/ordering/StringComparators.java @@ -34,23 +34,23 @@ public class StringComparators { public static final String LEXICOGRAPHIC_NAME = "lexicographic"; public static final String ALPHANUMERIC_NAME = "alphanumeric"; - + public static final LexicographicComparator LEXICOGRAPHIC = new LexicographicComparator(); public static final AlphanumericComparator ALPHANUMERIC = new AlphanumericComparator(); - + @JsonTypeInfo(use=Id.NAME, include=As.PROPERTY, property="type", defaultImpl = LexicographicComparator.class) @JsonSubTypes(value = { @JsonSubTypes.Type(name = StringComparators.LEXICOGRAPHIC_NAME, value = LexicographicComparator.class), @JsonSubTypes.Type(name = StringComparators.ALPHANUMERIC_NAME, value = AlphanumericComparator.class) }) - public static interface StringComparator extends Comparator + public static interface StringComparator extends Comparator { } - + public static class LexicographicComparator implements StringComparator { @Override - public int compare(String s, String s2) + public int compare(Comparable s, Comparable s2) { // Avoid conversion to bytes for equal references if(s == s2){ @@ -65,11 +65,11 @@ public int compare(String s, String s2) } return UnsignedBytes.lexicographicalComparator().compare( - StringUtils.toUtf8(s), - StringUtils.toUtf8(s2) + StringUtils.toUtf8(s.toString()), + StringUtils.toUtf8(s2.toString()) ); } - + @Override public boolean equals(Object o) { @@ -79,22 +79,42 @@ public boolean equals(Object o) if (o == null || getClass() != o.getClass()) { return false; } - + return true; } - + @Override public String toString() { return StringComparators.LEXICOGRAPHIC_NAME; } } - + public static class AlphanumericComparator implements StringComparator { + + public int compare(Comparable o1, Comparable o2) + { + if (o1 == null) { + if (o2 == null) { + return 0; + } else { + return -1; + } + } + if (o2 == null) { + return 1; + } + if (o1 instanceof String) { + return compareString((String) o1, (String) o2); + } else { + return o1.compareTo(o2); + } + } + // This code is based on https://github.com/amjjd/java-alphanum, see // NOTICE file for more information - public int compare(String str1, String str2) + private int compareString(String str1, String str2) { int[] pos = { 0, 0 }; diff --git a/processing/src/main/java/io/druid/query/search/SearchQueryRunner.java b/processing/src/main/java/io/druid/query/search/SearchQueryRunner.java index e6fb921ff024..b844f72a0899 100644 --- a/processing/src/main/java/io/druid/query/search/SearchQueryRunner.java +++ b/processing/src/main/java/io/druid/query/search/SearchQueryRunner.java @@ -27,6 +27,7 @@ import com.google.common.collect.Maps; import com.metamx.collections.bitmap.BitmapFactory; import com.metamx.collections.bitmap.ImmutableBitmap; +import com.metamx.common.IAE; import com.metamx.common.ISE; import com.metamx.common.guava.Accumulator; import com.metamx.common.guava.FunctionalIterable; @@ -47,13 +48,16 @@ import io.druid.query.search.search.SearchQuerySpec; import io.druid.segment.ColumnSelectorBitmapIndexSelector; import io.druid.segment.Cursor; +import io.druid.segment.DimensionColumnReader; import io.druid.segment.DimensionSelector; import io.druid.segment.QueryableIndex; import io.druid.segment.Segment; import io.druid.segment.StorageAdapter; import io.druid.segment.column.BitmapIndex; import io.druid.segment.column.Column; +import io.druid.segment.data.IndexedFloats; import io.druid.segment.data.IndexedInts; +import io.druid.segment.data.IndexedLongs; import io.druid.segment.filter.Filters; import org.apache.commons.lang.mutable.MutableInt; @@ -104,47 +108,63 @@ public Sequence> run( dimsToSearch = dimensions; } - final BitmapFactory bitmapFactory = index.getBitmapFactoryForDimensions(); - - final ImmutableBitmap baseFilter = - filter == null ? null : filter.getBitmapIndex(new ColumnSelectorBitmapIndexSelector(bitmapFactory, index)); - + boolean useStorageAdapter = false; for (DimensionSpec dimension : dimsToSearch) { final Column column = index.getColumn(dimension.getDimension()); if (column == null) { continue; } - - final BitmapIndex bitmapIndex = column.getBitmapIndex(); - ExtractionFn extractionFn = dimension.getExtractionFn(); - if (extractionFn == null) { - extractionFn = IdentityExtractionFn.getInstance(); + if (!column.getCapabilities().hasBitmapIndexes()) { + // If one of the columns lacks bitmap indexes, fallback to using StorageAdapter and do a per-row scan. + // Could optimize this by using bitmaps when available, full scan otherwise + useStorageAdapter = true; + break; } - if (bitmapIndex != null) { - for (int i = 0; i < bitmapIndex.getCardinality(); ++i) { - String dimVal = Strings.nullToEmpty(extractionFn.apply(bitmapIndex.getValue(i))); - if (!searchQuerySpec.accept(dimVal)) { - continue; - } - ImmutableBitmap bitmap = bitmapIndex.getBitmap(i); - if (baseFilter != null) { - bitmap = bitmapFactory.intersection(Arrays.asList(baseFilter, bitmap)); - } - if (bitmap.size() > 0) { - MutableInt counter = new MutableInt(bitmap.size()); - MutableInt prev = retVal.put(new SearchHit(dimension.getOutputName(), dimVal), counter); - if (prev != null) { - counter.add(prev.intValue()); + } + + if (!useStorageAdapter) { + final BitmapFactory bitmapFactory = index.getBitmapFactoryForDimensions(); + final Map readers = index.getDimensionReaders(); + final ImmutableBitmap baseFilter = + filter == null + ? null + : filter.getBitmapIndex(new ColumnSelectorBitmapIndexSelector(bitmapFactory, index, readers)); + + for (DimensionSpec dimension : dimsToSearch) { + final Column column = index.getColumn(dimension.getDimension()); + if (column == null) { + continue; + } + final BitmapIndex bitmapIndex = column.getBitmapIndex(); + ExtractionFn extractionFn = dimension.getExtractionFn(); + if (extractionFn == null) { + extractionFn = IdentityExtractionFn.getInstance(); + } + if (bitmapIndex != null) { + for (int i = 0; i < bitmapIndex.getCardinality(); ++i) { + String dimVal = Strings.nullToEmpty(extractionFn.apply(bitmapIndex.getValue(i))); + if (!searchQuerySpec.accept(dimVal)) { + continue; } - if (retVal.size() >= limit) { - return makeReturnResult(limit, retVal); + ImmutableBitmap bitmap = bitmapIndex.getBitmap(i); + if (baseFilter != null) { + bitmap = bitmapFactory.intersection(Arrays.asList(baseFilter, bitmap)); + } + if (bitmap.size() > 0) { + MutableInt counter = new MutableInt(bitmap.size()); + MutableInt prev = retVal.put(new SearchHit(dimension.getOutputName(), dimVal), counter); + if (prev != null) { + counter.add(prev.intValue()); + } + if (retVal.size() >= limit) { + return makeReturnResult(limit, retVal); + } } } } } + return makeReturnResult(limit, retVal); } - - return makeReturnResult(limit, retVal); } final StorageAdapter adapter = segment.asStorageAdapter(); @@ -191,19 +211,59 @@ public TreeMap accumulate(TreeMap final DimensionSelector selector = entry.getValue(); if (selector != null) { - final IndexedInts vals = selector.getRow(); - for (int i = 0; i < vals.size(); ++i) { - final String dimVal = selector.lookupName(vals.get(i)); - if (searchQuerySpec.accept(dimVal)) { - MutableInt counter = new MutableInt(1); - MutableInt prev = set.put(new SearchHit(entry.getKey(), dimVal), counter); - if (prev != null) { - counter.add(prev.intValue()); + Comparable extractedVal; + String rowValStr; + switch(selector.getDimCapabilities().getType()) { + case STRING: + final IndexedInts vals = selector.getRow(); + for (int i = 0; i < vals.size(); ++i) { + final String dimVal = selector.lookupName(vals.get(i)); + if (searchQuerySpec.accept(dimVal)) { + MutableInt counter = new MutableInt(1); + MutableInt prev = set.put(new SearchHit(entry.getKey(), dimVal), counter); + if (prev != null) { + counter.add(prev.intValue()); + } + if (set.size() >= limit) { + return set; + } + } + } + break; + case LONG: + final IndexedLongs longVals = selector.getLongRow(); + extractedVal = selector.getExtractedValueLong(longVals.get(0)); + rowValStr = extractedVal == null ? null : extractedVal.toString(); + if (searchQuerySpec.accept(rowValStr)) { + MutableInt counter = new MutableInt(1); + MutableInt prev = set.put(new SearchHit(entry.getKey(), rowValStr), counter); + if (prev != null) { + counter.add(prev.intValue()); + } + if (set.size() >= limit) { + return set; + } } - if (set.size() >= limit) { - return set; + break; + case FLOAT: + final IndexedFloats floatVals = selector.getFloatRow(); + extractedVal = selector.getExtractedValueFloat(floatVals.get(0)); + rowValStr = extractedVal == null ? null : extractedVal.toString(); + if (searchQuerySpec.accept(rowValStr)) { + MutableInt counter = new MutableInt(1); + MutableInt prev = set.put(new SearchHit(entry.getKey(), rowValStr), counter); + if (prev != null) { + counter.add(prev.intValue()); + } + if (set.size() >= limit) { + return set; + } } - } + break; + case COMPLEX: + throw new UnsupportedOperationException("SearchQuery not supported for Complex types"); + default: + throw new IAE("Invalid type: " + selector.getDimCapabilities().getType()); } } } diff --git a/processing/src/main/java/io/druid/query/select/SelectQueryEngine.java b/processing/src/main/java/io/druid/query/select/SelectQueryEngine.java index 21037a5150a0..170a82b5c8c3 100644 --- a/processing/src/main/java/io/druid/query/select/SelectQueryEngine.java +++ b/processing/src/main/java/io/druid/query/select/SelectQueryEngine.java @@ -23,6 +23,7 @@ import com.google.common.base.Preconditions; import com.google.common.collect.Lists; import com.google.common.collect.Maps; +import com.metamx.common.IAE; import com.metamx.common.ISE; import com.metamx.common.guava.Sequence; import io.druid.query.QueryRunnerHelper; @@ -37,7 +38,11 @@ import io.druid.segment.SegmentDesc; import io.druid.segment.StorageAdapter; import io.druid.segment.column.Column; +import io.druid.segment.column.ColumnCapabilities; +import io.druid.segment.column.ValueType; +import io.druid.segment.data.IndexedFloats; import io.druid.segment.data.IndexedInts; +import io.druid.segment.data.IndexedLongs; import io.druid.segment.filter.Filters; import org.joda.time.DateTime; import org.joda.time.Interval; @@ -121,22 +126,43 @@ public Result apply(Cursor cursor) for (Map.Entry dimSelector : dimSelectors.entrySet()) { final String dim = dimSelector.getKey(); final DimensionSelector selector = dimSelector.getValue(); + final ColumnCapabilities capabilities = adapter.getColumnCapabilities(dim); if (selector == null) { theEvent.put(dim, null); } else { - final IndexedInts vals = selector.getRow(); - - if (vals.size() == 1) { - final String dimVal = selector.lookupName(vals.get(0)); - theEvent.put(dim, dimVal); - } else { - List dimVals = Lists.newArrayList(); - for (int i = 0; i < vals.size(); ++i) { - dimVals.add(selector.lookupName(vals.get(i))); - } - theEvent.put(dim, dimVals); + Object dimVals; + ValueType type = capabilities == null ? ValueType.STRING : capabilities.getType(); + switch(type) { + case STRING: + final IndexedInts vals = selector.getRow(); + if (vals.size() == 1) { + dimVals = selector.lookupName(vals.get(0)); + } else { + List strVals = Lists.newArrayList(); + for (int i = 0; i < vals.size(); ++i) { + strVals.add(selector.lookupName(vals.get(i))); + } + dimVals = strVals; + } + break; + // These types only support single values. + case LONG: + final IndexedLongs longVals = selector.getLongRow(); + dimVals = selector.getExtractedValueLong(longVals.get(0)); + break; + case FLOAT: + final IndexedFloats floatVals = selector.getFloatRow(); + dimVals = selector.getExtractedValueFloat(floatVals.get(0)); + break; + case COMPLEX: + final Comparable objVal = selector.getComparableRow(); + dimVals = selector.getExtractedValueComparable(objVal); + break; + default: + throw new IAE("Invalid type: " + capabilities.getType()); } + theEvent.put(dim, dimVals); } } diff --git a/processing/src/main/java/io/druid/query/topn/AlphaNumericTopNMetricSpec.java b/processing/src/main/java/io/druid/query/topn/AlphaNumericTopNMetricSpec.java index c31b5e043940..34391f2a2339 100644 --- a/processing/src/main/java/io/druid/query/topn/AlphaNumericTopNMetricSpec.java +++ b/processing/src/main/java/io/druid/query/topn/AlphaNumericTopNMetricSpec.java @@ -34,7 +34,7 @@ public class AlphaNumericTopNMetricSpec extends LexicographicTopNMetricSpec { private static final byte CACHE_TYPE_ID = 0x2; - protected static Comparator comparator = StringComparators.ALPHANUMERIC; + protected static Comparator comparator = StringComparators.ALPHANUMERIC; @JsonCreator public AlphaNumericTopNMetricSpec( diff --git a/processing/src/main/java/io/druid/query/topn/BaseTopNAlgorithm.java b/processing/src/main/java/io/druid/query/topn/BaseTopNAlgorithm.java index 170c349514e0..af7a8213cd54 100644 --- a/processing/src/main/java/io/druid/query/topn/BaseTopNAlgorithm.java +++ b/processing/src/main/java/io/druid/query/topn/BaseTopNAlgorithm.java @@ -26,10 +26,12 @@ import io.druid.segment.Capabilities; import io.druid.segment.Cursor; import io.druid.segment.DimensionSelector; +import io.druid.segment.column.ValueType; import java.util.Arrays; import java.util.Comparator; import java.util.List; +import java.util.Map; /** */ @@ -80,14 +82,18 @@ public void run( final int numToProcess; int maxNumToProcess = Math.min(params.getNumValuesPerPass(), cardinality - numProcessed); - DimValSelector theDimValSelector; - if (!hasDimValSelector) { - numToProcess = maxNumToProcess; - theDimValSelector = makeDimValSelector(params, numProcessed, numToProcess); + DimValSelector theDimValSelector = null; + if (supportsDimValSelector()) { + if (!hasDimValSelector) { + numToProcess = maxNumToProcess; + theDimValSelector = makeDimValSelector(params, numProcessed, numToProcess); + } else { + //skip invalid, calculate length to have enough valid value to process or hit the end. + numToProcess = computeNewLength(dimValSelector, numProcessed, maxNumToProcess); + theDimValSelector = updateDimValSelector(dimValSelector, numProcessed, numToProcess); + } } else { - //skip invalid, calculate length to have enough valid value to process or hit the end. - numToProcess = computeNewLength(dimValSelector, numProcessed, maxNumToProcess); - theDimValSelector = updateDimValSelector(dimValSelector, numProcessed, numToProcess); + numToProcess = maxNumToProcess; } DimValAggregateStore aggregatesStore = makeDimValAggregateStore(params); @@ -145,6 +151,8 @@ protected abstract void closeAggregators( DimValAggregateStore dimValAggregateStore ); + protected abstract boolean supportsDimValSelector(); + protected class AggregatorArrayProvider extends BaseArrayProvider { Aggregator[][] expansionAggs; @@ -260,7 +268,8 @@ public static TopNResultBuilder makeResultBuilder(TopNParams params, TopNQuery q query.getThreshold(), comparator, query.getAggregatorSpecs(), - query.getPostAggregatorSpecs() + query.getPostAggregatorSpecs(), + (Map) query.getContextValue("typeHints") ); } } diff --git a/processing/src/main/java/io/druid/query/topn/DimExtractionTopNAlgorithm.java b/processing/src/main/java/io/druid/query/topn/DimExtractionTopNAlgorithm.java index edceeb9f2e83..f21fc002a0da 100644 --- a/processing/src/main/java/io/druid/query/topn/DimExtractionTopNAlgorithm.java +++ b/processing/src/main/java/io/druid/query/topn/DimExtractionTopNAlgorithm.java @@ -20,28 +20,37 @@ package io.druid.query.topn; import com.google.common.collect.Maps; +import com.metamx.common.IAE; import io.druid.query.aggregation.Aggregator; import io.druid.segment.Capabilities; +import io.druid.segment.ComparableDimensionSelector; import io.druid.segment.Cursor; import io.druid.segment.DimensionSelector; +import io.druid.segment.column.ColumnCapabilities; +import io.druid.segment.data.IndexedFloats; import io.druid.segment.data.IndexedInts; +import io.druid.segment.data.IndexedLongs; +import java.util.List; import java.util.Map; /** * This has to be its own strategy because the pooled topn algorithm assumes each index is unique, and cannot handle multiple index numerals referencing the same dimension value. */ -public class DimExtractionTopNAlgorithm extends BaseTopNAlgorithm, TopNParams> +public class DimExtractionTopNAlgorithm + extends BaseTopNAlgorithm, TopNParams> { private final TopNQuery query; + private final ColumnCapabilities columnCapabilities; public DimExtractionTopNAlgorithm( Capabilities capabilities, - TopNQuery query + TopNQuery query, + ColumnCapabilities columnCapabilities ) { super(capabilities); - + this.columnCapabilities = columnCapabilities; this.query = query; } @@ -81,45 +90,89 @@ protected Aggregator[][] updateDimValSelector(Aggregator[][] aggregators, int nu } @Override - protected Map makeDimValAggregateStore(TopNParams params) + protected Map makeDimValAggregateStore(TopNParams params) { return Maps.newHashMap(); } + private void updateAggsWithoutDictionary( + Cursor cursor, + Map aggregatesStore, + Comparable key + ) + { + Aggregator[] theAggregators = aggregatesStore.get(key); + if (theAggregators == null) { + theAggregators = makeAggregators(cursor, query.getAggregatorSpecs()); + aggregatesStore.put(key, theAggregators); + } + for (Aggregator aggregator : theAggregators) { + aggregator.aggregate(); + } + } + @Override public void scanAndAggregate( TopNParams params, Aggregator[][] rowSelector, - Map aggregatesStore, + Map aggregatesStore, int numProcessed ) { final Cursor cursor = params.getCursor(); final DimensionSelector dimSelector = params.getDimSelector(); - while (!cursor.isDone()) { - final IndexedInts dimValues = dimSelector.getRow(); - - for (int i = 0; i < dimValues.size(); ++i) { - - final int dimIndex = dimValues.get(i); - Aggregator[] theAggregators = rowSelector[dimIndex]; - if (theAggregators == null) { - final String key = dimSelector.lookupName(dimIndex); - theAggregators = aggregatesStore.get(key); - if (theAggregators == null) { - theAggregators = makeAggregators(cursor, query.getAggregatorSpecs()); - aggregatesStore.put(key, theAggregators); + Comparable key; + switch (columnCapabilities.getType()) { + case STRING: + while (!cursor.isDone()) { + final IndexedInts dimValues = dimSelector.getRow(); + for (int i = 0; i < dimValues.size(); ++i) { + final int dimIndex = dimValues.get(i); + Aggregator[] theAggregators = rowSelector[dimIndex]; + if (theAggregators == null) { + key = dimSelector.lookupName(dimIndex); + theAggregators = aggregatesStore.get(key); + if (theAggregators == null) { + theAggregators = makeAggregators(cursor, query.getAggregatorSpecs()); + aggregatesStore.put(key, theAggregators); + } + rowSelector[dimIndex] = theAggregators; + } + + for (Aggregator aggregator : theAggregators) { + aggregator.aggregate(); + } } - rowSelector[dimIndex] = theAggregators; + cursor.advance(); } - - for (Aggregator aggregator : theAggregators) { - aggregator.aggregate(); + break; + case LONG: + while (!cursor.isDone()) { + final IndexedLongs longVals = dimSelector.getLongRow(); + key = dimSelector.getExtractedValueLong(longVals.get(0)); + updateAggsWithoutDictionary(cursor, aggregatesStore, key); + cursor.advance(); } - } - - cursor.advance(); + break; + case FLOAT: + while (!cursor.isDone()) { + final IndexedFloats floatVals = dimSelector.getFloatRow(); + key = dimSelector.getExtractedValueFloat(floatVals.get(0)); + updateAggsWithoutDictionary(cursor, aggregatesStore, key); + cursor.advance(); + } + break; + case COMPLEX: + while (!cursor.isDone()) { + final Comparable objVal = dimSelector.getComparableRow(); + key = dimSelector.getExtractedValueComparable(objVal); + updateAggsWithoutDictionary(cursor, aggregatesStore, key); + cursor.advance(); + } + break; + default: + throw new IAE("Invalid type: " + columnCapabilities.getType()); } } @@ -127,11 +180,11 @@ public void scanAndAggregate( protected void updateResults( TopNParams params, Aggregator[][] rowSelector, - Map aggregatesStore, + Map aggregatesStore, TopNResultBuilder resultBuilder ) { - for (Map.Entry entry : aggregatesStore.entrySet()) { + for (Map.Entry entry : aggregatesStore.entrySet()) { Aggregator[] aggs = entry.getValue(); if (aggs != null && aggs.length > 0) { Object[] vals = new Object[aggs.length]; @@ -149,7 +202,7 @@ protected void updateResults( } @Override - protected void closeAggregators(Map stringMap) + protected void closeAggregators(Map stringMap) { for (Aggregator[] aggregators : stringMap.values()) { for (Aggregator agg : aggregators) { @@ -158,6 +211,12 @@ protected void closeAggregators(Map stringMap) } } + @Override + protected boolean supportsDimValSelector() + { + return columnCapabilities.isDictionaryEncoded(); + } + @Override public void cleanup(TopNParams params) { diff --git a/processing/src/main/java/io/druid/query/topn/DimValHolder.java b/processing/src/main/java/io/druid/query/topn/DimValHolder.java index 2299622cbff1..f983d8960d84 100644 --- a/processing/src/main/java/io/druid/query/topn/DimValHolder.java +++ b/processing/src/main/java/io/druid/query/topn/DimValHolder.java @@ -26,13 +26,13 @@ public class DimValHolder { private final Object topNMetricVal; - private final String dimName; + private final Comparable dimName; private final Object dimValIndex; private final Map metricValues; public DimValHolder( Object topNMetricVal, - String dimName, + Comparable dimName, Object dimValIndex, Map metricValues ) @@ -48,7 +48,7 @@ public Object getTopNMetricVal() return topNMetricVal; } - public String getDimName() + public Comparable getDimName() { return dimName; } @@ -66,7 +66,7 @@ public Map getMetricValues() public static class Builder { private Object topNMetricVal; - private String dimName; + private Comparable dimName; private Object dimValIndex; private Map metricValues; @@ -84,7 +84,7 @@ public Builder withTopNMetricVal(Object topNMetricVal) return this; } - public Builder withDimName(String dimName) + public Builder withDimName(Comparable dimName) { this.dimName = dimName; return this; diff --git a/processing/src/main/java/io/druid/query/topn/InvertedTopNMetricSpec.java b/processing/src/main/java/io/druid/query/topn/InvertedTopNMetricSpec.java index dd2d695b9e24..64aa7b1fe15f 100644 --- a/processing/src/main/java/io/druid/query/topn/InvertedTopNMetricSpec.java +++ b/processing/src/main/java/io/druid/query/topn/InvertedTopNMetricSpec.java @@ -25,11 +25,13 @@ import io.druid.query.aggregation.AggregatorFactory; import io.druid.query.aggregation.PostAggregator; import io.druid.query.dimension.DimensionSpec; +import io.druid.segment.column.ValueType; import org.joda.time.DateTime; import java.nio.ByteBuffer; import java.util.Comparator; import java.util.List; +import java.util.Map; /** */ @@ -93,7 +95,8 @@ public TopNResultBuilder getResultBuilder( int threshold, Comparator comparator, List aggFactories, - List postAggs + List postAggs, + Map typeHints ) { return delegate.getResultBuilder( @@ -102,7 +105,8 @@ public TopNResultBuilder getResultBuilder( threshold, comparator, aggFactories, - postAggs + postAggs, + typeHints ); } diff --git a/processing/src/main/java/io/druid/query/topn/LexicographicTopNMetricSpec.java b/processing/src/main/java/io/druid/query/topn/LexicographicTopNMetricSpec.java index 0ac649555428..c64889cae95c 100644 --- a/processing/src/main/java/io/druid/query/topn/LexicographicTopNMetricSpec.java +++ b/processing/src/main/java/io/druid/query/topn/LexicographicTopNMetricSpec.java @@ -28,11 +28,13 @@ import io.druid.query.dimension.DimensionSpec; import io.druid.query.ordering.StringComparators; +import io.druid.segment.column.ValueType; import org.joda.time.DateTime; import java.nio.ByteBuffer; import java.util.Comparator; import java.util.List; +import java.util.Map; /** */ @@ -40,7 +42,7 @@ public class LexicographicTopNMetricSpec implements TopNMetricSpec { private static final byte CACHE_TYPE_ID = 0x1; - private static Comparator comparator = StringComparators.LEXICOGRAPHIC; + private static Comparator comparator = StringComparators.LEXICOGRAPHIC; private final String previousStop; @@ -77,7 +79,8 @@ public TopNResultBuilder getResultBuilder( int threshold, Comparator comparator, List aggFactories, - List postAggs + List postAggs, + Map typeHints ) { return new TopNLexicographicResultBuilder( diff --git a/processing/src/main/java/io/druid/query/topn/NumericTopNMetricSpec.java b/processing/src/main/java/io/druid/query/topn/NumericTopNMetricSpec.java index 5beda3c2c2d1..d2f7e03e0531 100644 --- a/processing/src/main/java/io/druid/query/topn/NumericTopNMetricSpec.java +++ b/processing/src/main/java/io/druid/query/topn/NumericTopNMetricSpec.java @@ -27,11 +27,13 @@ import io.druid.query.aggregation.AggregatorFactory; import io.druid.query.aggregation.PostAggregator; import io.druid.query.dimension.DimensionSpec; +import io.druid.segment.column.ValueType; import org.joda.time.DateTime; import java.nio.ByteBuffer; import java.util.Comparator; import java.util.List; +import java.util.Map; /** */ @@ -122,10 +124,11 @@ public TopNResultBuilder getResultBuilder( int threshold, Comparator comparator, List aggFactories, - List postAggs + List postAggs, + Map typeHints ) { - return new TopNNumericResultBuilder(timestamp, dimSpec, metric, threshold, comparator, aggFactories, postAggs); + return new TopNNumericResultBuilder(timestamp, dimSpec, metric, threshold, comparator, aggFactories, postAggs, typeHints); } @Override diff --git a/processing/src/main/java/io/druid/query/topn/PooledTopNAlgorithm.java b/processing/src/main/java/io/druid/query/topn/PooledTopNAlgorithm.java index 521d1903e552..ca34c398a1f2 100644 --- a/processing/src/main/java/io/druid/query/topn/PooledTopNAlgorithm.java +++ b/processing/src/main/java/io/druid/query/topn/PooledTopNAlgorithm.java @@ -493,6 +493,12 @@ protected void closeAggregators(BufferAggregator[] bufferAggregators) } } + @Override + protected boolean supportsDimValSelector() + { + return true; + } + @Override public void cleanup(PooledTopNParams params) { diff --git a/processing/src/main/java/io/druid/query/topn/TimeExtractionTopNAlgorithm.java b/processing/src/main/java/io/druid/query/topn/TimeExtractionTopNAlgorithm.java index 504e8d80be1a..7bacb95bd743 100644 --- a/processing/src/main/java/io/druid/query/topn/TimeExtractionTopNAlgorithm.java +++ b/processing/src/main/java/io/druid/query/topn/TimeExtractionTopNAlgorithm.java @@ -128,6 +128,12 @@ protected void closeAggregators(Map stringMap) } } + @Override + protected boolean supportsDimValSelector() + { + return true; + } + @Override public void cleanup(TopNParams params) { diff --git a/processing/src/main/java/io/druid/query/topn/TopNBinaryFn.java b/processing/src/main/java/io/druid/query/topn/TopNBinaryFn.java index 781aa7c3dbba..3e97383a7e6f 100644 --- a/processing/src/main/java/io/druid/query/topn/TopNBinaryFn.java +++ b/processing/src/main/java/io/druid/query/topn/TopNBinaryFn.java @@ -19,6 +19,8 @@ package io.druid.query.topn; +import com.google.common.base.Function; +import com.google.common.collect.Maps; import com.metamx.common.guava.nary.BinaryFn; import io.druid.granularity.AllGranularity; import io.druid.granularity.QueryGranularity; @@ -27,6 +29,10 @@ import io.druid.query.aggregation.AggregatorUtil; import io.druid.query.aggregation.PostAggregator; import io.druid.query.dimension.DimensionSpec; +import io.druid.segment.DimensionHandler; +import io.druid.segment.DimensionHandlerUtil; +import io.druid.segment.StringDimensionHandler; +import io.druid.segment.column.ValueType; import org.joda.time.DateTime; import java.util.Comparator; @@ -47,6 +53,8 @@ public class TopNBinaryFn implements BinaryFn, Result aggregations; private final List postAggregations; private final Comparator comparator; + private final Map typeHints; + private final Function valueTransformer; public TopNBinaryFn( final TopNResultMerger merger, @@ -55,7 +63,8 @@ public TopNBinaryFn( final TopNMetricSpec topNMetricSpec, final int threshold, final List aggregatorSpecs, - final List postAggregatorSpecs + final List postAggregatorSpecs, + final Map typeHints ) { this.merger = merger; @@ -72,6 +81,28 @@ public TopNBinaryFn( this.dimension = dimSpec.getOutputName(); this.comparator = topNMetricSpec.getComparator(aggregatorSpecs, postAggregatorSpecs); + this.typeHints = typeHints; + ValueType dimType = null; + if(this.typeHints != null) { + String dimName = dimSpec.getDimension(); + dimType = this.typeHints.get(dimName); + } + + if (dimType == null) { + valueTransformer = StringDimensionHandler.STRING_TRANSFORMER; + } else { + valueTransformer = DimensionHandlerUtil.getTransformerForType(dimType); + } + } + + private DimensionAndMetricValueExtractor getExtractorWithOverride(String dimension, + Object overrideVal, + DimensionAndMetricValueExtractor originalExtractor) + { + Map newMap = Maps.newHashMap(originalExtractor.getBaseObject()); + newMap.put(dimension, overrideVal); + DimensionAndMetricValueExtractor newExtractor = new DimensionAndMetricValueExtractor(newMap); + return newExtractor; } @Override @@ -84,16 +115,27 @@ public Result apply(Result arg1, Result retVals = new LinkedHashMap<>(); + Map retVals = new LinkedHashMap<>(); TopNResultValue arg1Vals = arg1.getValue(); TopNResultValue arg2Vals = arg2.getValue(); for (DimensionAndMetricValueExtractor arg1Val : arg1Vals) { - retVals.put(arg1Val.getStringDimensionValue(dimension), arg1Val); + Object dimVal1 = arg1Val.getDimensionValue(dimension); + Object transformedDimVal1 = valueTransformer.apply(dimVal1); + if (dimVal1 != transformedDimVal1) { + arg1Val = getExtractorWithOverride(dimension, transformedDimVal1, arg1Val); + dimVal1 = transformedDimVal1; + } + retVals.put(dimVal1, arg1Val); } for (DimensionAndMetricValueExtractor arg2Val : arg2Vals) { - final String dimensionValue = arg2Val.getStringDimensionValue(dimension); + Object dimensionValue = arg2Val.getDimensionValue(dimension); + Object transformedValue = valueTransformer.apply(dimensionValue); + if (dimensionValue != transformedValue) { + arg2Val = getExtractorWithOverride(dimension, transformedValue, arg2Val); + dimensionValue = transformedValue; + } DimensionAndMetricValueExtractor arg1Val = retVals.get(dimensionValue); if (arg1Val != null) { @@ -129,7 +171,8 @@ public Result apply(Result arg1, Result aggFactories, - List postAggs + List postAggs, + Map typeHints ); public byte[] getCacheKey(); diff --git a/processing/src/main/java/io/druid/query/topn/TopNNumericResultBuilder.java b/processing/src/main/java/io/druid/query/topn/TopNNumericResultBuilder.java index e6ebaa9b4da0..12b25f54cd94 100644 --- a/processing/src/main/java/io/druid/query/topn/TopNNumericResultBuilder.java +++ b/processing/src/main/java/io/druid/query/topn/TopNNumericResultBuilder.java @@ -28,6 +28,11 @@ import io.druid.query.aggregation.AggregatorUtil; import io.druid.query.aggregation.PostAggregator; import io.druid.query.dimension.DimensionSpec; +import io.druid.segment.DimensionHandlerUtil; +import io.druid.segment.FloatDimensionHandler; +import io.druid.segment.LongDimensionHandler; +import io.druid.segment.StringDimensionHandler; +import io.druid.segment.column.ValueType; import org.joda.time.DateTime; import java.util.Arrays; @@ -50,10 +55,12 @@ public class TopNNumericResultBuilder implements TopNResultBuilder private final PriorityQueue pQueue; private final Comparator dimValComparator; private final String[] aggFactoryNames; - private static final Comparator dimNameComparator = new Comparator() + //private final Map typeHints; + //private final Function valueTransformer; + private static final Comparator dimNameComparator = new Comparator() { @Override - public int compare(String o1, String o2) + public int compare(Comparable o1, Comparable o2) { int retval; if (null == o1) { @@ -80,7 +87,8 @@ public TopNNumericResultBuilder( int threshold, final Comparator comparator, List aggFactories, - List postAggs + List postAggs, + Map typeHints ) { this.timestamp = timestamp; @@ -90,6 +98,18 @@ public TopNNumericResultBuilder( this.postAggs = AggregatorUtil.pruneDependentPostAgg(postAggs, this.metricName); this.threshold = threshold; + //this.typeHints = typeHints; + + /* + if(this.typeHints != null) { + String dimName = dimSpec.getDimension(); + ValueType dimType = this.typeHints.get(dimName); + valueTransformer = DimensionHandlerUtil.getTransformerForType(dimType); + } else { + valueTransformer = StringDimensionHandler.STRING_TRANSFORMER; + } + */ + this.metricComparator = comparator; this.dimValComparator = new Comparator() { @@ -100,6 +120,10 @@ public int compare(DimValHolder d1, DimValHolder d2) if (retVal == 0) { retVal = dimNameComparator.compare(d1.getDimName(), d2.getDimName()); + /* + retVal = dimNameComparator.compare((Comparable) valueTransformer.apply(d1.getDimName()), + (Comparable) valueTransformer.apply(d2.getDimName())); + */ } return retVal; @@ -114,7 +138,7 @@ public int compare(DimValHolder d1, DimValHolder d2) @Override public TopNNumericResultBuilder addEntry( - String dimName, + Comparable dimName, Object dimValIndex, Object[] metricVals ) @@ -194,9 +218,12 @@ public TopNResultBuilder addEntry(DimensionAndMetricValueExtractor dimensionAndM final Object dimValue = dimensionAndMetricValueExtractor.getDimensionValue(metricName); if (shouldAdd(dimValue)) { + //Comparable transformedDimVal = (Comparable) dimensionAndMetricValueExtractor.getDimensionValue(dimSpec.getOutputName()); + //transformedDimVal = (Comparable) valueTransformer.apply(transformedDimVal); final DimValHolder valHolder = new DimValHolder.Builder() .withTopNMetricVal(dimValue) - .withDimName(dimensionAndMetricValueExtractor.getStringDimensionValue(dimSpec.getOutputName())) + //.withDimName(transformedDimVal) + .withDimName((Comparable) dimensionAndMetricValueExtractor.getDimensionValue(dimSpec.getOutputName())) .withMetricValues(dimensionAndMetricValueExtractor.getBaseObject()) .build(); pQueue.add(valHolder); @@ -228,6 +255,10 @@ public int compare(DimValHolder d1, DimValHolder d2) if (retVal == 0) { retVal = dimNameComparator.compare(d1.getDimName(), d2.getDimName()); + /* + retVal = dimNameComparator.compare((Comparable) valueTransformer.apply(d1.getDimName()), + (Comparable) valueTransformer.apply(d2.getDimName())); + */ } return retVal; diff --git a/processing/src/main/java/io/druid/query/topn/TopNQueryEngine.java b/processing/src/main/java/io/druid/query/topn/TopNQueryEngine.java index 9626eb853b0b..5dc43f85c349 100644 --- a/processing/src/main/java/io/druid/query/topn/TopNQueryEngine.java +++ b/processing/src/main/java/io/druid/query/topn/TopNQueryEngine.java @@ -36,6 +36,7 @@ import io.druid.segment.SegmentMissingException; import io.druid.segment.StorageAdapter; import io.druid.segment.column.Column; +import io.druid.segment.column.ColumnCapabilities; import io.druid.segment.filter.Filters; import org.joda.time.Interval; @@ -104,6 +105,9 @@ private Function> getMapFn(TopNQuery query, fina final TopNAlgorithmSelector selector = new TopNAlgorithmSelector(cardinality, numBytesPerRecord); query.initTopNAlgorithmSelector(selector); + final ColumnCapabilities columnCapabilities = adapter.getColumnCapabilities(dimension); + final boolean isDictionaryEncoded = columnCapabilities == null ? true : columnCapabilities.isDictionaryEncoded(); + final TopNAlgorithm topNAlgorithm; if ( selector.isHasExtractionFn() && @@ -115,8 +119,8 @@ private Function> getMapFn(TopNQuery query, fina // A special TimeExtractionTopNAlgorithm is required, since DimExtractionTopNAlgorithm // currently relies on the dimension cardinality to support lexicographic sorting topNAlgorithm = new TimeExtractionTopNAlgorithm(capabilities, query); - } else if (selector.isHasExtractionFn()) { - topNAlgorithm = new DimExtractionTopNAlgorithm(capabilities, query); + } else if (selector.isHasExtractionFn() || !isDictionaryEncoded) { + topNAlgorithm = new DimExtractionTopNAlgorithm(capabilities, query, columnCapabilities); } else if (selector.isAggregateAllMetrics()) { topNAlgorithm = new PooledTopNAlgorithm(capabilities, query, bufferPool); } else if (selector.isAggregateTopNMetricFirst() || query.getContextBoolean("doAggregateTopNMetricFirst", false)) { diff --git a/processing/src/main/java/io/druid/query/topn/TopNQueryQueryToolChest.java b/processing/src/main/java/io/druid/query/topn/TopNQueryQueryToolChest.java index 20ca60c66128..023209e26967 100644 --- a/processing/src/main/java/io/druid/query/topn/TopNQueryQueryToolChest.java +++ b/processing/src/main/java/io/druid/query/topn/TopNQueryQueryToolChest.java @@ -52,6 +52,7 @@ import io.druid.query.dimension.DefaultDimensionSpec; import io.druid.query.dimension.DimensionSpec; import io.druid.query.filter.DimFilter; +import io.druid.segment.column.ValueType; import org.joda.time.DateTime; import javax.annotation.Nullable; @@ -136,7 +137,8 @@ protected BinaryFn, Result, Result) query.getContextValue("typeHints") ); } }; diff --git a/processing/src/main/java/io/druid/query/topn/TopNResultBuilder.java b/processing/src/main/java/io/druid/query/topn/TopNResultBuilder.java index 7c2b64c37f05..890347448f23 100644 --- a/processing/src/main/java/io/druid/query/topn/TopNResultBuilder.java +++ b/processing/src/main/java/io/druid/query/topn/TopNResultBuilder.java @@ -28,7 +28,7 @@ public interface TopNResultBuilder { public TopNResultBuilder addEntry( - String dimName, + Comparable dimName, Object dimValIndex, Object[] metricVals ); diff --git a/processing/src/main/java/io/druid/segment/ColumnSelectorBitmapIndexSelector.java b/processing/src/main/java/io/druid/segment/ColumnSelectorBitmapIndexSelector.java index 619c5e5f0165..3cc06d9aaf3b 100644 --- a/processing/src/main/java/io/druid/segment/ColumnSelectorBitmapIndexSelector.java +++ b/processing/src/main/java/io/druid/segment/ColumnSelectorBitmapIndexSelector.java @@ -19,19 +19,23 @@ package io.druid.segment; +import com.google.common.base.Predicate; import com.google.common.base.Strings; import com.metamx.collections.bitmap.BitmapFactory; import com.metamx.collections.bitmap.ImmutableBitmap; import com.metamx.collections.spatial.ImmutableRTree; +import com.metamx.collections.spatial.search.Bound; import com.metamx.common.guava.CloseQuietly; import io.druid.query.filter.BitmapIndexSelector; import io.druid.segment.column.Column; import io.druid.segment.column.DictionaryEncodedColumn; import io.druid.segment.column.GenericColumn; +import io.druid.segment.column.ValueType; import io.druid.segment.data.Indexed; import io.druid.segment.data.IndexedIterable; import java.util.Iterator; +import java.util.Map; /** */ @@ -39,14 +43,17 @@ public class ColumnSelectorBitmapIndexSelector implements BitmapIndexSelector { private final BitmapFactory bitmapFactory; private final ColumnSelector index; + private final Map dimReaders; public ColumnSelectorBitmapIndexSelector( final BitmapFactory bitmapFactory, - final ColumnSelector index + final ColumnSelector index, + final Map dimReaders ) { this.bitmapFactory = bitmapFactory; this.index = index; + this.dimReaders = dimReaders; } @Override @@ -122,21 +129,47 @@ public ImmutableBitmap getBitmapIndex(String dimension, String value) } } - if (!column.getCapabilities().hasBitmapIndexes()) { + return dimReaders.get(dimension).getBitmapIndex(value, bitmapFactory, getNumRows()); + } + + @Override + public ImmutableBitmap getBitmapIndex(String dimension, Bound bound) + { + final Column column = index.getColumn(dimension); + if (column == null) { return bitmapFactory.makeEmptyImmutableBitmap(); } - return column.getBitmapIndex().getBitmap(value); + DimensionColumnReader reader = dimReaders.get(dimension); + return reader.getBitmapIndex(bound, bitmapFactory); + } + + @Override + public ImmutableBitmap getBitmapIndexFromColumnScan(String dimension, Predicate predicate) + { + DimensionColumnReader reader = dimReaders.get(dimension); + return reader.getBitmapIndex(predicate, bitmapFactory); } @Override - public ImmutableRTree getSpatialIndex(String dimension) + public ValueType getDimensionType(String dimension) { final Column column = index.getColumn(dimension); - if (column == null || !column.getCapabilities().hasSpatialIndexes()) { - return new ImmutableRTree(); + if (column == null) { + return null; + } else { + return column.getCapabilities().getType(); } + } - return column.getSpatialIndex().getRTree(); + @Override + public boolean hasBitmapIndexes(String dimension) { + final Column column = index.getColumn(dimension); + if (column == null) { + // default to "has bitmap index" path for null columns + return true; + } + return column.getCapabilities().hasBitmapIndexes(); } + } diff --git a/processing/src/main/java/io/druid/segment/ColumnSelectorFactory.java b/processing/src/main/java/io/druid/segment/ColumnSelectorFactory.java index 2aff7f831c19..3102115ef01f 100644 --- a/processing/src/main/java/io/druid/segment/ColumnSelectorFactory.java +++ b/processing/src/main/java/io/druid/segment/ColumnSelectorFactory.java @@ -27,6 +27,7 @@ public interface ColumnSelectorFactory { public DimensionSelector makeDimensionSelector(DimensionSpec dimensionSpec); + public DimensionSelector makeDictEncodedStringDimensionSelector(DimensionSpec dimensionSpec); public FloatColumnSelector makeFloatColumnSelector(String columnName); public LongColumnSelector makeLongColumnSelector(String columnName); public ObjectColumnSelector makeObjectColumnSelector(String columnName); diff --git a/processing/src/main/java/io/druid/segment/ComparableDimensionSelector.java b/processing/src/main/java/io/druid/segment/ComparableDimensionSelector.java new file mode 100644 index 000000000000..947b0be53d60 --- /dev/null +++ b/processing/src/main/java/io/druid/segment/ComparableDimensionSelector.java @@ -0,0 +1,120 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.segment; + +import com.google.common.collect.Maps; +import io.druid.query.extraction.ExtractionFn; +import io.druid.segment.column.ColumnCapabilities; +import io.druid.segment.data.IndexedFloats; +import io.druid.segment.data.IndexedInts; +import io.druid.segment.data.IndexedLongs; + +import java.util.Map; + +public class ComparableDimensionSelector implements DimensionSelector +{ + private final ObjectColumnSelector selector; + private final ExtractionFn extractionFn; + private final ColumnCapabilities capabilities; + private final Map remappedValues = Maps.newHashMap(); + + public ComparableDimensionSelector(Object selector, ExtractionFn extractionFn, ColumnCapabilities capabilities) + { + this.selector = (ObjectColumnSelector) selector; + this.extractionFn = extractionFn; + this.capabilities = capabilities; + } + + @Override + public IndexedInts getRow() + { + throw new UnsupportedOperationException("float column does not support getRow"); + } + + @Override + public int getValueCardinality() + { + throw new UnsupportedOperationException("comparable column does not support getValueCardinality"); + } + + @Override + public int lookupId(String name) + { + throw new UnsupportedOperationException("comparable column does not support lookupId"); + } + + @Override + public String lookupName(int id) + { + throw new UnsupportedOperationException("comparable column does not support lookupName"); + } + + @Override + public IndexedFloats getFloatRow() + { + throw new UnsupportedOperationException("comparable column does not support getFloatRow"); + } + + @Override + public Comparable getExtractedValueFloat(float val) + { + throw new UnsupportedOperationException("comparable column does not support getExtractedValueComparable"); + } + + @Override + public IndexedLongs getLongRow() + { + throw new UnsupportedOperationException("comparable column does not support getFloatRow"); + } + + @Override + public Comparable getExtractedValueLong(long val) + { + throw new UnsupportedOperationException("comparable column does not support getExtractedValueFloat"); + } + + @Override + public Comparable getComparableRow() + { + return (Comparable) selector.get(); + } + + @Override + public Comparable getExtractedValueComparable(Comparable val) + { + if (extractionFn == null) { + return val; + } + + String extractedVal = remappedValues.get(val); + if (extractedVal == null) { + extractedVal = extractionFn.apply(val); + remappedValues.put(val, extractedVal); + } + return extractedVal; + } + + @Override + public ColumnCapabilities getDimCapabilities() + { + return capabilities; + } + +} diff --git a/processing/src/main/java/io/druid/segment/DimensionColumnReader.java b/processing/src/main/java/io/druid/segment/DimensionColumnReader.java new file mode 100644 index 000000000000..e1514d0565a3 --- /dev/null +++ b/processing/src/main/java/io/druid/segment/DimensionColumnReader.java @@ -0,0 +1,105 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.segment; + +import com.google.common.base.Predicate; +import com.metamx.collections.bitmap.BitmapFactory; +import com.metamx.collections.bitmap.ImmutableBitmap; +import com.metamx.collections.spatial.search.Bound; +import io.druid.segment.column.ColumnCapabilities; +import io.druid.segment.data.Indexed; + +/** + * Processing related interface + * + * A DimensionColumnReader is an object that encapsulates Column-reading operations used by + * QueryableIndex and its associated wrappers. + * + * @param class of the encoded values + * @param class of the actual values + */ +public interface DimensionColumnReader, ActualType extends Comparable> +{ + /** + * Filter on a value and return a result bitmap + * + * @param value value to filter on + * @param bitmapFactory bitmap factory + * @param numRows number of rows in the column to be read + * + * @return A bitmap indicating rows that matched the provided value + */ + public ImmutableBitmap getBitmapIndex(String value, BitmapFactory bitmapFactory, int numRows); + + + /** + * Filter on a predicate and return a result bitmap + * + * @param predicate predicate to filter on + * @param bitmapFactory bitmap factory + * + * @return A bitmap indicating rows that matched the provided predicate + */ + public ImmutableBitmap getBitmapIndex(Predicate predicate, BitmapFactory bitmapFactory); + + + /** + * Filter on a Bound and return a result bitmap + * + * @param bound spatial boundary, used for spatial filtering + * @param bitmapFactory bitmap factory + * + * @return A bitmap indicating rows that matched the provided bound + */ + public ImmutableBitmap getBitmapIndex(Bound bound, BitmapFactory bitmapFactory); + + + /** + * Get the minimum value for this dimension. + * + * @return min value + */ + public ActualType getMinValue(); + + + /** + * Get the maximum value for this dimension. + * + * @return max value + */ + public ActualType getMaxValue(); + + + /** + * Returns an indexed structure of this dimension's sorted actual values. + * The integer IDs represent the ordering of the sorted values. + * + * @return Sorted index of actual values + */ + public Indexed getSortedIndexedValues(); + + + /** + * Get the ColumnCapabilites of this dimension. + * + * @return column capabilities + */ + public ColumnCapabilities getCapabilities(); +} diff --git a/processing/src/main/java/io/druid/segment/DimensionHandler.java b/processing/src/main/java/io/druid/segment/DimensionHandler.java new file mode 100644 index 000000000000..8645ab802ba2 --- /dev/null +++ b/processing/src/main/java/io/druid/segment/DimensionHandler.java @@ -0,0 +1,173 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.segment; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.base.Function; +import com.metamx.common.io.smoosh.FileSmoosher; +import io.druid.query.dimension.DimensionSpec; +import io.druid.segment.column.Column; +import io.druid.segment.column.ColumnCapabilities; +import io.druid.segment.data.IOPeon; + +import java.io.File; +import java.util.Comparator; + +/** + * Processing related interface + * + * A DimensionHandler is an object that encapsulates indexing, column merging/building, and querying operations + * for a given dimension type (e.g., dict-encoded String, Long). + * + * These operations are handled by sub-objects created through a DimensionHandler's methods: + * DimensionIndexer, DimensionMerger, and DimensionColumnReader, respectively. + * + * Each DimensionHandler object is associated with a single dimension. + * + * This interface allows type-specific behavior column logic, such as choice of indexing structures and disk formats. + * to be contained within a type-specific set of handler objects, simplifying processing classes + * such as IncrementalIndex and IndexMerger and allowing for abstracted development of additional dimension types. + * + * A dimension may have two representations, an encoded representation and a full representation. + * For example, a value for a String dimension has an integer dictionary encoding, and an actual String representation. + * + * @param class of the encoded values + * @param class of the actual values + */ +public interface DimensionHandler, ActualType extends Comparable> +{ + /** + * Creates a new DimensionIndexer, a per-dimension object responsible for processing ingested rows in-memory, used by the + * IncrementalIndex. See {@link DimensionIndexer} interface for more information. + * + * @param lock An object to synchronize on. The IncrementalIndex will pass in its dimensionsDescs + * Map object, which stores descriptors for each dimension. + * + * @return A new DimensionIndexer object that synchronizes on the provided lock. + */ + public DimensionIndexer makeIndexer(Object lock); + + + /** + * Creates a new DimensionMerger, a per-dimension object responsible for merging indexes/row data across segments + * and building the on-disk representation of a dimension. + * + * See {@link DimensionMerger} interface for more information. + * + * @param indexSpec Specification object for the index merge + * @param outDir Location to store files generated by the merging process + * @param ioPeon ioPeon object passed in by IndexMerger, manages files created by the merging process + * @param v9Smoosher FileSmoosher used for combining files created by the merging process. + * @param mapper Object mapper used for JSON serde + * @param capabilities The ColumnCapabilities of the dimension represented by this DimensionHandler + * @param progress ProgressIndicator used by the merging process + + * @return A new DimensionMerger object. + */ + public DimensionMerger makeMerger( + IndexSpec indexSpec, + File outDir, + IOPeon ioPeon, + FileSmoosher v9Smoosher, + ObjectMapper mapper, + ColumnCapabilities capabilities, + ProgressIndicator progress + ); + + + /** + * Creates a new DimensionColumnReader, a per-dimension object responsible for reading data from an on-disk column. + * Used by QueryableIndex and associated wrapper classes. + * + * See {@link DimensionColumnReader} interface for more information. + * + * @param column The Column object associated with this DimensionHandler's dimension. + + * @return A new DimensionColumnReader object. + */ + public DimensionColumnReader makeColumnReader(Column column); + + + /** + * Creates a DimensionSelector of a type suitable for reading values from this dimension, used for querying and filtering. + * + * @param cursor Cursor object created by a StorageAdapter. + * @param dimSpec Specifies extraction function and output name for a dimension. + * @param capabilities The capabilities associated with this handler's dimension. + * + * @return A new DimensionSelector object linked with the provided cursor and dimSpec + */ + public DimensionSelector getDimensionSelector(ColumnSelectorFactory cursor, DimensionSpec dimSpec, ColumnCapabilities capabilities); + + + /** + * Returns a function that attempts to convert Object values to this dimension's type. + * Needed because the types of arriving row values may not always match this dimension's type during ingestion. + * + * If no transformation was performed, the transformation function must return the input object. + * + * @return A value transformation function. + */ + public Function getValueTypeTransformer(); + + + /** + * Returns a Comparator suitable for comparing values of EncodedType + * + * @return An EncodedType Comparator. + */ + public Comparator getEncodedComparator(); + + + /** + * Returns a Comparator suitable for comparing values of ActualType + * + * @return An ActualType Comparator. + */ + public Comparator getActualComparator(); + + + /** + * Convert a value of this dimension's EncodedType to a byte array representation. + * + * @param rowVal value to convert + * + * @return Converted byte array + */ + public byte[] getBytesFromRowValue(EncodedType rowVal); + + + /** + * Convert a byte array representation to this dimension's EncodedType. + * + * @param bytes the byte array to convert + * + * @return Converted value + */ + public EncodedType getRowValueFromBytes(byte[] bytes); + + + /** + * Get the size of this dimension's EncodedType values + ** + * @return The size in bytes of a single encoded value. + */ + public int getEncodedValueSize(); +} diff --git a/processing/src/main/java/io/druid/segment/DimensionHandlerUtil.java b/processing/src/main/java/io/druid/segment/DimensionHandlerUtil.java new file mode 100644 index 000000000000..2cd571cdecf7 --- /dev/null +++ b/processing/src/main/java/io/druid/segment/DimensionHandlerUtil.java @@ -0,0 +1,77 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.segment; + +import com.google.common.base.Function; +import com.metamx.common.IAE; +import io.druid.segment.column.ColumnCapabilities; +import io.druid.segment.column.ValueType; + +public final class DimensionHandlerUtil +{ + private DimensionHandlerUtil() {} + + public static DimensionHandler getHandlerFromCapabilities(String dimensionName, ColumnCapabilities capabilities) + { + DimensionHandler handler = null; + if (capabilities.getType() == ValueType.STRING) { + if (!capabilities.isDictionaryEncoded() || !capabilities.hasBitmapIndexes()) { + throw new IAE("String column must have dictionary encoding and bitmap index."); + } + handler = new StringDimensionHandler(dimensionName); + } + if (capabilities.getType() == ValueType.LONG) { + if (capabilities.isDictionaryEncoded() || capabilities.hasBitmapIndexes()) { + throw new IAE("Long column does not support dictionary encoding or bitmap index."); + } + handler = new LongDimensionHandler(dimensionName); + } + if (capabilities.getType() == ValueType.FLOAT) { + if (capabilities.isDictionaryEncoded() || capabilities.hasBitmapIndexes()) { + throw new IAE("Float column does not support dictionary encoding or bitmap index."); + } + handler = new FloatDimensionHandler(dimensionName); + } + + if (handler == null) { + throw new IAE("Could not create handler from invalid column type: " + capabilities.getType()); + } + return handler; + } + + public static Function getTransformerForType(ValueType type) + { + Function valueTransformer; + switch (type) { + case LONG: + valueTransformer = LongDimensionHandler.LONG_TRANSFORMER; + break; + case FLOAT: + valueTransformer = FloatDimensionHandler.FLOAT_TRANSFORMER; + break; + case STRING: + valueTransformer = StringDimensionHandler.STRING_TRANSFORMER; + break; + default: + throw new UnsupportedOperationException("Invalid type: " + type); + } + return valueTransformer; + } +} diff --git a/processing/src/main/java/io/druid/segment/DimensionIndexer.java b/processing/src/main/java/io/druid/segment/DimensionIndexer.java new file mode 100644 index 000000000000..921598563bae --- /dev/null +++ b/processing/src/main/java/io/druid/segment/DimensionIndexer.java @@ -0,0 +1,189 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.segment; + +import io.druid.segment.data.Indexed; + + +/** + * Processing related interface + * + * A DimensionIndexer is a per-dimension stateful object that encapsulates type-specific operations and data structures + * used during the in-memory ingestion process (i.e., work done by IncrementalIndex). + * + * Ingested row values are passed to a DimensionIndexer, which will update its internal data structures such as + * a value->ID dictionary as row values are seen. + * + * The DimensionIndexer is also responsible for implementing various value lookup operations, + * such as conversion between an encoded value and its full representation. It maintains knowledge of the + * mappings between encoded values and actual values. + * + * NOTE: + * When encoding is present, there are two relevant orderings for the encoded values. + * + * 1.) Ordering based on encoded value's order of ingestion + * 2.) Ordering based on converted actual value + * + * Suppose we have a new String dimension DimA, which sees the values "Hello", "World", and "Apple", in that order. + * This would correspond to dictionary encodings of "Hello"=0, "World"=1, and "Apple"=2, by the order + * in which these values were first seen during ingestion. + * + * However, some use cases require the encodings to be sorted by their associated actual values. + * In this example, that ordering would be "Apple"=0, "Hello"=1, "World"=2. + * + * @param class of the encoded values + * @param class of the actual values + * + */ +public interface DimensionIndexer, ActualType extends Comparable> +{ + + /** + * Given a list of row values (for multivalued dimensions) or single row value, update any + * internal data structures with the ingested values and return the row values as a Comparable[] + * to be used as a TimeAndDims key. + * + * The values within the Comparable[] return value should be encoded if applicable, i.e. as instances of EncodedType. + * + * NOTE: This function can change the internal state of the DimensionIndexer. + * + * @param dimValues Row value(s) to process + * + * @return A new DimensionIndexer object that synchronizes on the provided lock. + */ + public Comparable[] processRowValsToIndexKey(Object dimValues); + + + /** + * Some rows may not contain a particular dimension. Suppose the schema has DimA and DimB. + * If an event {"dimB"="abcd"} appears, dimA's DimensionIndexer will not see that null value for DimA. + * This function is called by IncrementalIndexAdapter to inform the DimensionIndexer of such unseen null values. + */ + public void addNullLookup(); + + + /** + * Given an encoded value, return its full representation. + * + * Using the example in the class description: + * getActualValue(0, true) would return "Apple" + * getActualValue(0, false) would return "Hello" + * + * @param intermediateValue The encoded value to be converted + * @param idSorted true, if the encoded value was sorted by actual value + * false, if the encoded value was ordered by time of ingestion + * + * @return The actual value associated with the encoded intermediateValue + * + */ + public ActualType getActualValue(EncodedType intermediateValue, boolean idSorted); + + + /** + * Given an actual value, return its encoded form. + * + * Using the example in the class description: + * getEncodedValue("Apple", true) would return 0 + * getEncodedValue("Apple", false) would return 2 + * + * @param fullValue Actual value to be converted. + * @param idSorted true, if return value should be an encoded value ordered by associated actual value + * false, if return value should be an encoded value ordered by time of ingestion + * @return An encoded value associated with fullValue + */ + public EncodedType getEncodedValue(ActualType fullValue, boolean idSorted); + + + /** + * Given an encoded value that was ordered by time of ingestion, return the equivalent + * encoded value ordered by associated actual value. + * + * Using the example in the class description: + * getSortedEncodedValueFromUnsorted(0) would return 2 + * + * @param unsortedIntermediateValue value to convert + * @return converted value + */ + public EncodedType getSortedEncodedValueFromUnsorted(EncodedType unsortedIntermediateValue); + + + /** + * Given an encoded value that was ordered by associated actual value, return the equivalent + * encoded value ordered by time of ingestion. + * + * Using the example in the class description: + * getUnsortedEncodedValueFromSorted(2) would return 0 + * + * @param sortedIntermediateValue value to convert + * @return converted value + */ + public EncodedType getUnsortedEncodedValueFromSorted(EncodedType sortedIntermediateValue); + + + /** + * Returns an indexed structure of this dimension's sorted actual values. + * The integer IDs represent the ordering of the sorted values. + * + * Using the example in the class description: + * "Apple"=0, + * "Hello"=1, + * "World"=2 + * + * @return Sorted index of actual values + */ + public Indexed getSortedIndexedValues(); + + + /** + * Get the minimum dimension value seen by this indexer. + * + * NOTE: + * On an in-memory segment (IncrementaIndex), we can determine min/max values by looking at the stream of + * row values seen in calls to processRowValsToIndexKey(). + * + * However, on a disk-backed segment (QueryableIndex), the numeric dimensions do not currently have any + * supporting index structures that can be used to efficiently determine min/max values. A full column scan would + * be needed, or a change to the segment file that stores the min/max values previously determined + * during ingestion. + * + * To maintain consistency between in-memory/on-disk semantics, getMinValue()/getMaxValue() will return + * null for both cases on numeric dimensions. + * + * @return min value + */ + public ActualType getMinValue(); + + + /** + * Get the maximum dimension value seen by this indexer. + * + * @return max value + */ + public ActualType getMaxValue(); + + + /** + * Get the cardinality of this dimension's values. + * + * @return value cardinality + */ + public int getCardinality(); + +} diff --git a/processing/src/main/java/io/druid/segment/DimensionMerger.java b/processing/src/main/java/io/druid/segment/DimensionMerger.java new file mode 100644 index 000000000000..641afd84e9a6 --- /dev/null +++ b/processing/src/main/java/io/druid/segment/DimensionMerger.java @@ -0,0 +1,165 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.segment; + +import com.google.common.io.OutputSupplier; + +import java.io.FileOutputStream; +import java.io.IOException; +import java.nio.Buffer; +import java.nio.IntBuffer; +import java.util.List; + +/** + * Processing related interface + * + * A DimensionMerger is a per-dimension stateful object that encapsulates type-specific operations and data structures + * used during the segment merging process (i.e., work done by IndexMerger/IndexMergerV9). + * + * This object is responsible for: + * - merging any relevant structures from the segments (e.g., encoding dictionaries) + * - writing the merged column data and any merged indexing structures (e.g., dictionaries, bitmaps) to disk + * + * @param class of the encoded values + * @param class of the actual values + */ +public interface DimensionMerger, ActualType extends Comparable> +{ + /** + * Initialize any writer objects that can be instantiated before data from the segments has been processed/merged. + * @throws IOException + */ + public void initWriters() throws IOException; + + + /** + * Read the segments to be merged and combine their indexes/data structures internally. + * (e.g., merging of dictionaries for String dimensions). + * + * @param adapters List of adapters to be merged. + * @throws IOException + */ + public void mergeAcrossSegments(List adapters) throws IOException; + + + /** + * Update the DimensionMerger's internal structures with row value(s) to be merged. + * + * An example implementation might be to add the values to an open column writer. + * + * @param values The row values to be added. + * @throws IOException + */ + public void addColumnValue(Comparable[] values) throws IOException; + + + /** + * Close any open writers and write merged data to disk. + * + * @param rowNumConversions A list of conversion buffers indexed by adapter number + * The list will have the same order as the list of adapters passed to mergeAcrossSegments() + * This buffer can be used to convert the row numbers from individual adapters. + * @throws IOException + */ + public void closeWriters(List rowNumConversions) throws IOException; + + + /** + * When merging multiple segments, it is sometimes necessary to convert "missing" values to "nulls". + * + * Suppose we have String dimensions DimA and DimB and two segments to be merged. + * + * Segment 1 has rows: + * {dimB = hello} + * + * Segment 2 has rows + * {dimB = hello, dimA=null} + * {dimB = world, dimA=foobar} + * + * When segment 1 and segment 2 are merged, dimA will not be present in segment 1 (no rows had values for dimA, + * so dimA was not written to disk with segment 1). + * + * For segment 2, dimA did have non-null values, so when it was persisted to disk, "null" would be + * assigned an encoded value (i.e., 0) within segment 2. + * + * The first row in both segments where dimB = hello should merge with each other. + * + * However, when rows from segment 1 and segment 2 are merged, the "missing" dimA values in segment 1 appear as + * null values within the merging process; the null dimA values from segment 2 appear as a dictionary encoding of 0. + * + * Thus, the rows from segment 1 and segment 2 where dimA is missing/null will not merge properly. + * + * To correct this, the missing values for dimA in segment 1 will be converted to + * encoded null values so that the rows can equate. + * + * @return true, if this conversion from missing->null is necessary + * false, otherwise. + */ + public boolean needsConvertMissingValues(); + + + /** + * If a dimension has no values after merging across segments, it does not need to be written to disk and can be skipped. + * + * @return true, if this dimension can be skipped + * false, if this dimension must be written + */ + public boolean canSkip(); + + + /** + * When merging values across segments when encoding is present, the encoded values within each segment + * may need to be converted, based on the total ordering of actual values in the merged data set. + * + * This function converts encoded values from a single segment to its equivalent encoded value within the + * final, sorted merged data set. + * + * @param value Value to convert + * @param indexNumber The index number of the segment/adapter where the to-be-converted value originated. + * + * @return converted encoded value + */ + public EncodedType getConvertedEncodedValue(EncodedType value, int indexNumber); + + + /** + * IndexMerger and IndexMergerV9 persist their columns in a different format. + * + * setWriteV8() instructs the DimensionMerger to operate in "V8" mode, i.e. in the manner of IndexMerger, + * instead of IndexMergerV9. + * + * If using V8, this function MUST be called before any other functions on the DimensionMerger. + */ + public void setWriteV8(); + + + /** + * V8 version of closeWriters(). + * + * @param rowNumConversions Same as with closeWriters() + * @param invertedOut Output file for inverted indexes (e.g., bitmaps) + * @param spatialOut Output file for spatial indexes + * @throws IOException + */ + public void closeWritersV8(List rowNumConversions, + OutputSupplier invertedOut, + OutputSupplier spatialOut) throws IOException; + +} diff --git a/processing/src/main/java/io/druid/segment/DimensionSelector.java b/processing/src/main/java/io/druid/segment/DimensionSelector.java index 57445a18a6c4..6d097d4ccf2b 100644 --- a/processing/src/main/java/io/druid/segment/DimensionSelector.java +++ b/processing/src/main/java/io/druid/segment/DimensionSelector.java @@ -17,7 +17,14 @@ * under the License. */ -package io.druid.segment;import io.druid.segment.data.IndexedInts; +package io.druid.segment; + +import io.druid.segment.column.ColumnCapabilities; +import io.druid.segment.data.IndexedFloats; +import io.druid.segment.data.IndexedInts; +import io.druid.segment.data.IndexedLongs; + +import java.util.List; /** */ @@ -79,4 +86,50 @@ public interface DimensionSelector * @return the id for the given field name */ public int lookupId(String name); + + /** + * Get a long row + * @return row value + */ + public IndexedLongs getLongRow(); + + /** + * Apply extraction function to a row value. + * @param val value to be transformed + * @return extracted value if extraction function is defined, return unchanged value otherwise + */ + public Comparable getExtractedValueLong(long val); + + /** + * Get a float row + * @return row value + */ + public IndexedFloats getFloatRow(); + + /** + * Apply extraction function to a row value. + * @param val value to be transformed + * @return extracted value if extraction function is defined, return unchanged value otherwise + */ + public Comparable getExtractedValueFloat(float val); + + /** + * Get a Comparable row (e.g., ComplexColumn.getRowValue()) + * @return row value + */ + public Comparable getComparableRow(); + + /** + * Apply extraction function to a row value. + * @param val value to be transformed + * @return extracted value if extraction function is defined, return unchanged value otherwise + */ + public Comparable getExtractedValueComparable(Comparable val); + + + /** + * Get the capabilities for this selector's dimension. + * @return capabilities + */ + public ColumnCapabilities getDimCapabilities(); } diff --git a/processing/src/main/java/io/druid/segment/FloatDimensionHandler.java b/processing/src/main/java/io/druid/segment/FloatDimensionHandler.java new file mode 100644 index 000000000000..87e87b6d3fa2 --- /dev/null +++ b/processing/src/main/java/io/druid/segment/FloatDimensionHandler.java @@ -0,0 +1,523 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.segment; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.base.Function; +import com.google.common.base.Predicate; +import com.google.common.io.OutputSupplier; +import com.google.common.primitives.Floats; +import com.google.common.primitives.Ints; +import com.metamx.collections.bitmap.BitmapFactory; +import com.metamx.collections.bitmap.ImmutableBitmap; +import com.metamx.collections.bitmap.MutableBitmap; +import com.metamx.collections.spatial.search.Bound; +import com.metamx.common.io.smoosh.FileSmoosher; +import com.metamx.common.io.smoosh.SmooshedWriter; +import com.metamx.common.logger.Logger; +import com.metamx.common.parsers.ParseException; +import io.druid.common.utils.SerializerUtils; +import io.druid.query.dimension.DimensionSpec; +import io.druid.segment.column.Column; +import io.druid.segment.column.ColumnCapabilities; +import io.druid.segment.column.ColumnDescriptor; +import io.druid.segment.column.GenericColumn; +import io.druid.segment.column.ValueType; +import io.druid.segment.data.IOPeon; +import io.druid.segment.data.Indexed; +import io.druid.segment.serde.FloatGenericColumnPartSerde; + +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.FloatBuffer; +import java.nio.IntBuffer; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; + +public class FloatDimensionHandler implements DimensionHandler +{ + private static final Logger log = new Logger(FloatDimensionHandler.class); + + public static final Function FLOAT_TRANSFORMER = new Function() + { + @Override + public Float apply(final Object o) + { + if (o == null) { + return null; + } + + if (o instanceof Float) { + return (Float) o; + } + + if (o instanceof String) { + if (((String) o).length() == 0) { + return 0.0f; + } + try { + return Float.valueOf((String) o); + } + catch (NumberFormatException nfe) { + throw new ParseException(nfe, "Unable to parse value[%s] as float in column: ", o); + } + } + if (o instanceof Number) { + return ((Number) o).floatValue(); + } + return null; + } + }; + + private static final Comparator Float_COMPARATOR = new Comparator() + { + @Override + public int compare(Float o1, Float o2) + { + if (o1 == null) { + return o2 == null ? 0 : -1; + } + if (o2 == null) { + return 1; + } + return Float.compare(o1, o2); + } + }; + + private final String dimensionName; + + public FloatDimensionHandler(String dimensionName) + { + this.dimensionName = dimensionName; + } + + @Override + public DimensionSelector getDimensionSelector( + ColumnSelectorFactory cursor, DimensionSpec dimSpec, ColumnCapabilities capabilities + ) + { + return new FloatDimensionSelector( + cursor.makeFloatColumnSelector(dimensionName), + dimSpec.getExtractionFn(), + capabilities + ); + } + + @Override + public Function getValueTypeTransformer() + { + return FLOAT_TRANSFORMER; + } + + @Override + public Comparator getEncodedComparator() + { + return Float_COMPARATOR; + } + + @Override + public Comparator getActualComparator() + { + return Float_COMPARATOR; + } + + @Override + public byte[] getBytesFromRowValue(Float rowVal) + { + return Ints.toByteArray(Float.floatToIntBits(rowVal)); + } + + @Override + public Float getRowValueFromBytes(byte[] bytes) + { + return Float.intBitsToFloat(Ints.fromByteArray(bytes)); + } + + @Override + public int getEncodedValueSize() + { + return Floats.BYTES; + } + + @Override + public DimensionIndexer makeIndexer(Object lock) + { + return new FloatDimensionIndexer(); + } + + @Override + public DimensionMerger makeMerger( + IndexSpec indexSpec, + File outDir, + IOPeon ioPeon, + FileSmoosher v9Smoosher, + ObjectMapper mapper, + ColumnCapabilities capabilities, + ProgressIndicator progress + ) + { + return new FloatDimensionMerger(indexSpec, outDir, ioPeon, v9Smoosher, mapper, capabilities, progress); + } + + @Override + public DimensionColumnReader makeColumnReader(Column column) + { + return new FloatDimensionColumnReader(column); + } + + + public class FloatDimensionIndexer implements DimensionIndexer + { + @Override + public Comparable[] processRowValsToIndexKey(Object dimValues) + { + final Comparable[] dimensionValues; + Float transformedVal = 0.0f; + try { + if (dimValues == null) { + return null; + } else if (dimValues instanceof List) { + List dimValuesList = (List) dimValues; + + if (dimValuesList.size() > 1) { + throw new UnsupportedOperationException("Multi-value rows are not supported on Float dimensions."); + } + + dimensionValues = new Comparable[dimValuesList.size()]; + for (int i = 0; i < dimValuesList.size(); i++) { + transformedVal = FLOAT_TRANSFORMER.apply(dimValuesList.get(i)); + dimensionValues[i] = transformedVal; + } + } else { + transformedVal = FLOAT_TRANSFORMER.apply(dimValues); + dimensionValues = new Comparable[]{transformedVal}; + } + } + catch (ParseException pe) { + throw new ParseException(pe.getMessage() + dimensionName); + } + return dimensionValues; + } + + @Override + public Float getActualValue(Float intermediateValue, boolean idSorted) + { + return intermediateValue; + } + + @Override + public Float getEncodedValue(Float fullValue, boolean idSorted) + { + return fullValue; + } + + @Override + public Float getSortedEncodedValueFromUnsorted(Float unsortedIntermediateValue) + { + return unsortedIntermediateValue; + } + + @Override + public Float getUnsortedEncodedValueFromSorted(Float sortedIntermediateValue) + { + return sortedIntermediateValue; + } + + @Override + public Indexed getSortedIndexedValues() + { + throw new UnsupportedOperationException("getSortedIndexedValues() is not supported for Float column: " + + dimensionName); + } + + @Override + public Float getMinValue() + { + return null; + } + + @Override + public Float getMaxValue() + { + return null; + } + + @Override + public void addNullLookup() + { + + } + + @Override + public int getCardinality() + { + // DimensionSelector.getValueCardinality() returns Integer.MAX_VALUE for Long and Floats. + // Return the same here for consistency. + return Integer.MAX_VALUE; + } + } + + public class FloatDimensionMerger implements DimensionMerger + { + private final IndexSpec indexSpec; + private final File outDir; + private final IOPeon ioPeon; + private final FileSmoosher v9Smoosher; + private final ObjectMapper mapper; + private final ColumnCapabilities capabilities; + private boolean useV8 = false; + private ProgressIndicator progress; + + private GenericColumnSerializer valueWriter; + + private MetricColumnSerializer valueWriterV8; + + public FloatDimensionMerger( + IndexSpec indexSpec, + File outDir, + IOPeon ioPeon, + FileSmoosher v9Smoosher, + ObjectMapper mapper, + ColumnCapabilities capabilities, + ProgressIndicator progress + ) + { + this.indexSpec = indexSpec; + this.ioPeon = ioPeon; + this.outDir = outDir; + this.v9Smoosher = v9Smoosher; + this.mapper = mapper; + this.capabilities = capabilities; + this.progress = progress; + } + + @Override + public void initWriters() throws IOException + { + if (useV8) { + valueWriterV8 = new FloatMetricColumnSerializer(dimensionName, outDir, ioPeon); + valueWriterV8.open(); + } else { + String filenameBase = String.format("%s.forward_dim", dimensionName); + valueWriter = FloatColumnSerializer.create(ioPeon, filenameBase, indexSpec.getDimensionCompressionStrategy()); + valueWriter.open(); + } + } + + @Override + public void mergeAcrossSegments(List adapters) throws IOException + { + return; + } + + @Override + public void addColumnValue(Comparable[] values) throws IOException + { + // Only singleValue numerics are supported right now. + if (useV8) { + valueWriterV8.serialize(values[0]); + } else { + valueWriter.serialize(values[0]); + } + } + + @Override + public void closeWriters(List rowNumConversions) throws IOException + { + float startTime = System.currentTimeMillis(); + + valueWriter.close(); + + final ColumnDescriptor.Builder builder = ColumnDescriptor.builder() + .setValueType(ValueType.FLOAT); + + builder.addSerde( + FloatGenericColumnPartSerde.serializerBuilder() + .withByteOrder(IndexIO.BYTE_ORDER) + .withDelegate((FloatColumnSerializer) valueWriter) + .build()); + final ColumnDescriptor serdeficator = builder.build(); + + makeColumn(v9Smoosher, dimensionName, serdeficator); + + log.info("Completed dimension column[%s] in %,d millis.", dimensionName, System.currentTimeMillis() - startTime); + } + + @Override + public boolean needsConvertMissingValues() + { + return true; + } + + @Override + public boolean canSkip() + { + return false; + } + + @Override + public Float getConvertedEncodedValue(Float value, int indexNumber) + { + return value; + } + + @Override + public void setWriteV8() + { + useV8 = true; + } + + @Override + public void closeWritersV8( + List rowNumConversions, + OutputSupplier invertedOut, + OutputSupplier spatialOut + ) throws IOException + { + valueWriterV8.closeFile( + IndexIO.makeNumericDimFile(outDir, dimensionName, IndexIO.BYTE_ORDER) + ); + } + + private void makeColumn( + final FileSmoosher v9Smoosher, + final String columnName, + final ColumnDescriptor serdeficator + ) throws IOException + { + final SerializerUtils serializerUtils = new SerializerUtils(); + + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + serializerUtils.writeString(baos, mapper.writeValueAsString(serdeficator)); + byte[] specBytes = baos.toByteArray(); + + final SmooshedWriter channel = v9Smoosher.addWithSmooshedWriter( + columnName, serdeficator.numBytes() + specBytes.length + ); + try { + channel.write(ByteBuffer.wrap(specBytes)); + serdeficator.write(channel); + } + finally { + channel.close(); + } + } + } + + public class FloatDimensionColumnReader implements DimensionColumnReader + { + private final Column column; + private final GenericColumn genericColumn; + + public FloatDimensionColumnReader(Column column) + { + this.column = column; + this.genericColumn = column.getGenericColumn(); + } + + @Override + public ImmutableBitmap getBitmapIndex( + String value, BitmapFactory bitmapFactory, int numRows + ) + { + if (column == null) { + return bitmapFactory.makeEmptyImmutableBitmap(); + } + final float parsedFloat; + try { + parsedFloat = Float.parseFloat(value); + } + catch (NumberFormatException nfe) { + throw new ParseException("Could not parse value[%s] as float!", value); + } + + final MutableBitmap bitmap = bitmapFactory.makeEmptyMutableBitmap(); + for (int rowIdx = 0; rowIdx < column.getLength(); rowIdx++) { + float rowVal = genericColumn.getFloatSingleValueRow(rowIdx); + if (rowVal == parsedFloat) { + bitmap.add(rowIdx); + } + } + + return bitmapFactory.makeImmutableBitmap(bitmap); + } + + @Override + public ImmutableBitmap getBitmapIndex( + Predicate predicate, BitmapFactory bitmapFactory + ) + { + if (column == null) { + return bitmapFactory.makeEmptyImmutableBitmap(); + } + + final MutableBitmap bitmap = bitmapFactory.makeEmptyMutableBitmap(); + + Float floatVal; + for (int rowIdx = 0; rowIdx < column.getLength(); rowIdx++) { + floatVal = genericColumn.getFloatSingleValueRow(rowIdx); + if (predicate.apply(floatVal)) { + bitmap.add(rowIdx); + } + } + return bitmapFactory.makeImmutableBitmap(bitmap); + } + + @Override + public ImmutableBitmap getBitmapIndex( + Bound bound, BitmapFactory bitmapFactory + ) + { + throw new UnsupportedOperationException("Float column does not support spatial filtering: " + dimensionName); + } + + @Override + public Float getMinValue() + { + return null; + } + + @Override + public Float getMaxValue() + { + return null; + } + + @Override + public Indexed getSortedIndexedValues() + { + throw new UnsupportedOperationException("getSortedIndexedValues() is not supported for Float column: " + + dimensionName); + } + + @Override + public ColumnCapabilities getCapabilities() + { + return column.getCapabilities(); + } + + } + + +} diff --git a/processing/src/main/java/io/druid/segment/FloatDimensionSelector.java b/processing/src/main/java/io/druid/segment/FloatDimensionSelector.java new file mode 100644 index 000000000000..e3effdd00a71 --- /dev/null +++ b/processing/src/main/java/io/druid/segment/FloatDimensionSelector.java @@ -0,0 +1,130 @@ +package io.druid.segment; + +import com.google.common.collect.Maps; +import io.druid.query.extraction.ExtractionFn; +import io.druid.segment.column.ColumnCapabilities; +import io.druid.segment.data.IndexedFloats; +import io.druid.segment.data.IndexedInts; +import io.druid.segment.data.IndexedFloats; +import io.druid.segment.data.IndexedLongs; + +import java.io.IOException; +import java.util.Map; + +public class FloatDimensionSelector implements DimensionSelector +{ + private final FloatColumnSelector selector; + private final ExtractionFn extractionFn; + private final ColumnCapabilities capabilities; + private final Map remappedValues = Maps.newHashMap(); + + public FloatDimensionSelector(Object selector, ExtractionFn extractionFn, ColumnCapabilities capabilities) + { + this.selector = (FloatColumnSelector) selector; + this.extractionFn = extractionFn; + this.capabilities = capabilities; + } + + @Override + public IndexedInts getRow() + { + throw new UnsupportedOperationException("float column does not support getRow"); + } + + @Override + public int getValueCardinality() + { + return Integer.MAX_VALUE; + } + + @Override + public int lookupId(String name) + { + throw new UnsupportedOperationException("float column does not support lookupId"); + } + + @Override + public String lookupName(int id) + { + throw new UnsupportedOperationException("float column does not support lookupName"); + } + + @Override + public IndexedFloats getFloatRow() + { + final float val = selector.get(); + return new IndexedFloats() + { + @Override + public int size() + { + // only single-value numerics are supported + return 1; + } + + @Override + public float get(int index) + { + return val; + } + + @Override + public void fill(int index, float[] toFill) + { + throw new UnsupportedOperationException("fill not supported"); + } + + @Override + public void close() throws IOException + { + + } + }; + } + + @Override + public Comparable getExtractedValueFloat(float val) + { + if (extractionFn == null) { + return val; + } + + String extractedVal = remappedValues.get(val); + if (extractedVal == null) { + extractedVal = extractionFn.apply(val); + remappedValues.put(val, extractedVal); + } + return extractedVal; + } + + @Override + public IndexedLongs getLongRow() + { + throw new UnsupportedOperationException("float column does not support getFloatRow"); + } + + @Override + public Comparable getExtractedValueLong(long val) + { + throw new UnsupportedOperationException("float column does not support getExtractedValueFloat"); + } + + @Override + public Comparable getComparableRow() + { + throw new UnsupportedOperationException("float column does not support getComparableRow"); + } + + @Override + public Comparable getExtractedValueComparable(Comparable val) + { + throw new UnsupportedOperationException("float column does not support getExtractedValueComparable"); + } + + @Override + public ColumnCapabilities getDimCapabilities() + { + return capabilities; + } + +} diff --git a/processing/src/main/java/io/druid/segment/FloatMetricColumnSerializer.java b/processing/src/main/java/io/druid/segment/FloatMetricColumnSerializer.java index c7ed37a03880..4f11ccaac48b 100644 --- a/processing/src/main/java/io/druid/segment/FloatMetricColumnSerializer.java +++ b/processing/src/main/java/io/druid/segment/FloatMetricColumnSerializer.java @@ -70,6 +70,12 @@ public void serialize(Object obj) throws IOException public void close() throws IOException { final File outFile = IndexIO.makeMetricFile(outDir, metricName, IndexIO.BYTE_ORDER); + closeFile(outFile); + } + + @Override + public void closeFile(File outFile) throws IOException + { outFile.delete(); MetricHolder.writeFloatMetric( Files.newOutputStreamSupplier(outFile, true), metricName, writer diff --git a/processing/src/main/java/io/druid/segment/IndexIO.java b/processing/src/main/java/io/druid/segment/IndexIO.java index 107df523578d..5d133bbd816b 100644 --- a/processing/src/main/java/io/druid/segment/IndexIO.java +++ b/processing/src/main/java/io/druid/segment/IndexIO.java @@ -51,6 +51,7 @@ import io.druid.common.utils.SerializerUtils; import io.druid.segment.column.Column; import io.druid.segment.column.ColumnBuilder; +import io.druid.segment.column.ColumnCapabilities; import io.druid.segment.column.ColumnConfig; import io.druid.segment.column.ColumnDescriptor; import io.druid.segment.column.ValueType; @@ -65,6 +66,7 @@ import io.druid.segment.data.Indexed; import io.druid.segment.data.IndexedInts; import io.druid.segment.data.IndexedIterable; +import io.druid.segment.data.IndexedLongs; import io.druid.segment.data.IndexedMultivalue; import io.druid.segment.data.IndexedRTree; import io.druid.segment.data.VSizeIndexed; @@ -123,17 +125,17 @@ public IndexIO(ObjectMapper mapper, ColumnConfig columnConfig) this.columnConfig = Preconditions.checkNotNull(columnConfig, "null ColumnConfig"); defaultIndexIOHandler = new DefaultIndexIOHandler(mapper); indexLoaders = ImmutableMap.builder() - .put(0, new LegacyIndexLoader(defaultIndexIOHandler, columnConfig)) - .put(1, new LegacyIndexLoader(defaultIndexIOHandler, columnConfig)) - .put(2, new LegacyIndexLoader(defaultIndexIOHandler, columnConfig)) - .put(3, new LegacyIndexLoader(defaultIndexIOHandler, columnConfig)) - .put(4, new LegacyIndexLoader(defaultIndexIOHandler, columnConfig)) - .put(5, new LegacyIndexLoader(defaultIndexIOHandler, columnConfig)) - .put(6, new LegacyIndexLoader(defaultIndexIOHandler, columnConfig)) - .put(7, new LegacyIndexLoader(defaultIndexIOHandler, columnConfig)) - .put(8, new LegacyIndexLoader(defaultIndexIOHandler, columnConfig)) - .put(9, new V9IndexLoader(columnConfig)) - .build(); + .put(0, new LegacyIndexLoader(defaultIndexIOHandler, columnConfig)) + .put(1, new LegacyIndexLoader(defaultIndexIOHandler, columnConfig)) + .put(2, new LegacyIndexLoader(defaultIndexIOHandler, columnConfig)) + .put(3, new LegacyIndexLoader(defaultIndexIOHandler, columnConfig)) + .put(4, new LegacyIndexLoader(defaultIndexIOHandler, columnConfig)) + .put(5, new LegacyIndexLoader(defaultIndexIOHandler, columnConfig)) + .put(6, new LegacyIndexLoader(defaultIndexIOHandler, columnConfig)) + .put(7, new LegacyIndexLoader(defaultIndexIOHandler, columnConfig)) + .put(8, new LegacyIndexLoader(defaultIndexIOHandler, columnConfig)) + .put(9, new V9IndexLoader(columnConfig)) + .build(); } @@ -320,8 +322,8 @@ public static void validateRowValues( rb1.getTimestamp(), rb2.getTimestamp() ); } - final int[][] dims1 = rb1.getDims(); - final int[][] dims2 = rb2.getDims(); + final Comparable[][] dims1 = rb1.getDims(); + final Comparable[][] dims2 = rb2.getDims(); if (dims1.length != dims2.length) { throw new SegmentValidationException( "Dim lengths not equal %s vs %s", @@ -332,12 +334,23 @@ public static void validateRowValues( final Indexed dim1Names = adapter1.getDimensionNames(); final Indexed dim2Names = adapter2.getDimensionNames(); for (int i = 0; i < dims1.length; ++i) { - final int[] dim1Vals = dims1[i]; - final int[] dim2Vals = dims2[i]; + final Comparable[] dim1Vals = dims1[i]; + final Comparable[] dim2Vals = dims2[i]; final String dim1Name = dim1Names.get(i); final String dim2Name = dim2Names.get(i); - final Indexed dim1ValNames = adapter1.getDimValueLookup(dim1Name); - final Indexed dim2ValNames = adapter2.getDimValueLookup(dim2Name); + + ColumnCapabilities capabilities1 = adapter1.getCapabilities(dim1Name); + ColumnCapabilities capabilities2 = adapter2.getCapabilities(dim2Name); + ValueType dim1Type = capabilities1.getType(); + ValueType dim2Type = capabilities2.getType(); + if (dim1Type != dim2Type) { + throw new SegmentValidationException( + "Dim [%s] types not equal. Expected %d found %d", + dim1Name, + dim1Type, + dim2Type + ); + } if (dim1Vals == null || dim2Vals == null) { if (dim1Vals != dim2Vals) { @@ -350,72 +363,111 @@ public static void validateRowValues( continue; } } + + final Indexed dim1ValNames = adapter1.getDimValueLookup(dim1Name); + final Indexed dim2ValNames = adapter2.getDimValueLookup(dim2Name); + if (dim1Vals.length != dim2Vals.length) { - // Might be OK if one of them has null. This occurs in IndexMakerTest - if (dim1Vals.length == 0 && dim2Vals.length == 1) { - final String dimValName = dim2ValNames.get(dim2Vals[0]); - if (dimValName == null) { - continue; - } else { - throw new SegmentValidationException( - "Dim [%s] value [%s] is not null", - dim2Name, - dimValName - ); - } - } else if (dim2Vals.length == 0 && dim1Vals.length == 1) { - final String dimValName = dim1ValNames.get(dim1Vals[0]); - if (dimValName == null) { - continue; - } else { - throw new SegmentValidationException( - "Dim [%s] value [%s] is not null", - dim1Name, - dimValName - ); + + if (capabilities1.isDictionaryEncoded()) { + // Might be OK if one of them has null. This occurs in IndexMakerTest + if (dim1Vals.length == 0 && dim2Vals.length == 1) { + final String dimValName = dim2ValNames.get((Integer) dim2Vals[0]); + if (dimValName == null) { + continue; + } else { + throw new SegmentValidationException( + "Dim [%s] value [%s] is not null", + dim2Name, + dimValName + ); + } + } else if (dim2Vals.length == 0 && dim1Vals.length == 1) { + final String dimValName = dim1ValNames.get((Integer) dim1Vals[0]); + if (dimValName == null) { + continue; + } else { + throw new SegmentValidationException( + "Dim [%s] value [%s] is not null", + dim1Name, + dimValName + ); + } } - } else { - throw new SegmentValidationException( - "Dim [%s] value lengths not equal. Expected %d found %d", - dim1Name, - dims1.length, - dims2.length - ); } + throw new SegmentValidationException( + "Dim [%s] value lengths not equal. Expected %d found %d", + dim1Name, + dims1.length, + dims2.length + ); } - for (int j = 0; j < Math.max(dim1Vals.length, dim2Vals.length); ++j) { - final int dIdex1 = dim1Vals.length <= j ? -1 : dim1Vals[j]; - final int dIdex2 = dim2Vals.length <= j ? -1 : dim2Vals[j]; + if (capabilities1.isDictionaryEncoded()) { + validateDictionaryEncodedValues(dim1Name, dim1Vals, dim2Vals, dim1ValNames, dim2ValNames); + } else { + validateUnencodedValues(dim1Name, dim1Vals, dim2Vals); + } + } + } - if (dIdex1 == dIdex2) { - continue; - } + private static void validateUnencodedValues( + final String dimName, + final Comparable[] dim1Vals, + final Comparable[] dim2Vals + ) + { + for (int j = 0; j < dim1Vals.length; j++) { + if (dim1Vals[j] != dim2Vals[j]) { + throw new SegmentValidationException( + "Dim [%s] value not equal. Expected [%s] found [%s]", + dimName, + dim1Vals[j], + dim2Vals[j] + ); + } + } + } - final String dim1ValName = dIdex1 < 0 ? null : dim1ValNames.get(dIdex1); - final String dim2ValName = dIdex2 < 0 ? null : dim2ValNames.get(dIdex2); - if ((dim1ValName == null) || (dim2ValName == null)) { - if ((dim1ValName == null) && (dim2ValName == null)) { - continue; - } else { - throw new SegmentValidationException( - "Dim [%s] value not equal. Expected [%s] found [%s]", - dim1Name, - dim1ValName, - dim2ValName - ); - } - } + private static void validateDictionaryEncodedValues( + final String dimName, + final Comparable[] dim1Vals, + final Comparable[] dim2Vals, + final Indexed dim1ValNames, + final Indexed dim2ValNames + ) + { + for (int j = 0; j < Math.max(dim1Vals.length, dim2Vals.length); ++j) { + final Comparable dIdex1 = dim1Vals.length <= j ? -1 : dim1Vals[j]; + final Comparable dIdex2 = dim2Vals.length <= j ? -1 : dim2Vals[j]; + + if (dIdex1 == dIdex2) { + continue; + } - if (!dim1ValName.equals(dim2ValName)) { + final Comparable dim1ValName = (Integer) dIdex1 < 0 ? null : dim1ValNames.get((Integer) dIdex1); + final Comparable dim2ValName = (Integer) dIdex2 < 0 ? null : dim2ValNames.get((Integer) dIdex2); + if ((dim1ValName == null) || (dim2ValName == null)) { + if ((dim1ValName == null) && (dim2ValName == null)) { + continue; + } else { throw new SegmentValidationException( "Dim [%s] value not equal. Expected [%s] found [%s]", - dim1Name, + dimName, dim1ValName, dim2ValName ); } } + + if (!dim1ValName.equals(dim2ValName)) { + throw new SegmentValidationException( + "Dim [%s] value not equal. Expected [%s] found [%s]", + dimName, + dim1ValName, + dim2ValName + ); + } } } @@ -747,7 +799,7 @@ public int size() channel.write(ByteBuffer.wrap(specBytes)); serdeficator.write(channel); channel.close(); - } else if (filename.startsWith("met_")) { + } else if (filename.startsWith("met_") || filename.startsWith("numeric_dim_")) { if (!filename.endsWith(String.format("%s.drd", BYTE_ORDER))) { skippedFiles.add(filename); continue; @@ -755,7 +807,6 @@ public int size() MetricHolder holder = MetricHolder.fromByteBuffer(v8SmooshedFiles.mapFile(filename)); final String metric = holder.getName(); - final ColumnDescriptor.Builder builder = ColumnDescriptor.builder(); switch (holder.getType()) { @@ -1094,6 +1145,11 @@ public static File makeDimFile(File dir, String dimension) return new File(dir, String.format("dim_%s.drd", dimension)); } + public static File makeNumericDimFile(File dir, String dimension, ByteOrder order) + { + return new File(dir, String.format("numeric_dim_%s_%s.drd", dimension, order)); + } + public static File makeTimeFile(File dir, ByteOrder order) { return new File(dir, String.format("time_%s.drd", order)); diff --git a/processing/src/main/java/io/druid/segment/IndexMerger.java b/processing/src/main/java/io/druid/segment/IndexMerger.java index bff448da55b1..939d152884e7 100644 --- a/processing/src/main/java/io/druid/segment/IndexMerger.java +++ b/processing/src/main/java/io/druid/segment/IndexMerger.java @@ -90,10 +90,14 @@ import java.io.File; import java.io.FileOutputStream; import java.io.IOException; +import java.io.OutputStream; +import java.nio.Buffer; import java.nio.ByteBuffer; import java.nio.IntBuffer; import java.nio.MappedByteBuffer; +import java.nio.channels.Channels; import java.nio.channels.FileChannel; +import java.nio.channels.WritableByteChannel; import java.util.ArrayList; import java.util.Arrays; import java.util.Comparator; @@ -582,6 +586,7 @@ public Metadata apply(IndexableAdapter input) final Map valueTypes = Maps.newTreeMap(Ordering.natural().nullsFirst()); final Map metricTypeNames = Maps.newTreeMap(Ordering.natural().nullsFirst()); final Map columnCapabilities = Maps.newHashMap(); + final List dimCapabilities = new ArrayList<>(); for (IndexableAdapter adapter : indexes) { for (String dimension : adapter.getDimensionNames()) { @@ -589,7 +594,7 @@ public Metadata apply(IndexableAdapter input) ColumnCapabilities capabilities = adapter.getCapabilities(dimension); if (mergedCapabilities == null) { mergedCapabilities = new ColumnCapabilitiesImpl(); - mergedCapabilities.setType(ValueType.STRING); + mergedCapabilities.setType(null); } columnCapabilities.put(dimension, mergedCapabilities.merge(capabilities)); } @@ -606,6 +611,10 @@ public Metadata apply(IndexableAdapter input) } } + for (String dimension : mergedDimensions) { + dimCapabilities.add(columnCapabilities.get(dimension)); + } + Closer closer = Closer.create(); final Interval dataInterval; final File v8OutDir = new File(outDir, "v8-tmp"); @@ -659,87 +668,20 @@ public void close() throws IOException progress.progress(); startTime = System.currentTimeMillis(); - ArrayList dimOuts = Lists.newArrayListWithCapacity(mergedDimensions.size()); - Map dimensionCardinalities = Maps.newHashMap(); - ArrayList> dimConversions = Lists.newArrayListWithCapacity(indexes.size()); - final ArrayList convertMissingDimsFlags = Lists.newArrayListWithCapacity(mergedDimensions.size()); - - for (int i = 0; i < indexes.size(); ++i) { - dimConversions.add(Maps.newHashMap()); + final DimensionHandler[] handlers = makeDimensionHandlers(mergedDimensions, dimCapabilities); + final List mergers = new ArrayList<>(); + for (int i = 0; i < mergedDimensions.size(); i++) { + DimensionMerger merger = handlers[i].makeMerger(indexSpec, v8OutDir, ioPeon, null, mapper, dimCapabilities.get(i), progress); + merger.setWriteV8(); + merger.initWriters(); + mergers.add(merger); } - for (String dimension : mergedDimensions) { - final GenericIndexedWriter writer = new GenericIndexedWriter( - ioPeon, dimension, GenericIndexed.STRING_STRATEGY - ); - writer.open(); - - boolean dimHasNull = false; - boolean dimHasValues = false; - boolean dimAbsentFromSomeIndex = false; - - int numMergeIndex = 0; - Indexed dimValueLookup = null; - Indexed[] dimValueLookups = new Indexed[indexes.size() + 1]; - for (int i = 0; i < indexes.size(); i++) { - Indexed dimValues = indexes.get(i).getDimValueLookup(dimension); - if (!isNullColumn(dimValues)) { - dimHasValues = true; - dimHasNull |= dimValues.indexOf(null) >= 0; - dimValueLookups[i] = dimValueLookup = dimValues; - numMergeIndex++; - } else { - dimAbsentFromSomeIndex = true; - } - } - - boolean convertMissingDims = dimHasValues && dimAbsentFromSomeIndex; - convertMissingDimsFlags.add(convertMissingDims); - - /* - * Ensure the empty str is always in the dictionary if the dimension was missing from one index but - * has non-null values in another index. - * This is done so that MMappedIndexRowIterable can convert null columns to empty strings - * later on, to allow rows from indexes without a particular dimension to merge correctly with - * rows from indexes with null/empty str values for that dimension. - */ - if (convertMissingDims && !dimHasNull) { - dimValueLookups[indexes.size()] = dimValueLookup = EMPTY_STR_DIM_VAL; - numMergeIndex++; - } - - int cardinality = 0; - if (numMergeIndex > 1) { - DictionaryMergeIterator iterator = new DictionaryMergeIterator(dimValueLookups, true); - - while (iterator.hasNext()) { - writer.write(iterator.next()); - } - - for (int i = 0; i < indexes.size(); i++) { - if (dimValueLookups[i] != null && iterator.needConversion(i)) { - dimConversions.get(i).put(dimension, iterator.conversions[i]); - } - } - cardinality = iterator.counter; - } else if (numMergeIndex == 1) { - for (String value : dimValueLookup) { - writer.write(value); - } - cardinality = dimValueLookup.size(); - } - - dimensionCardinalities.put(dimension, cardinality); - - FileOutputSupplier dimOut = new FileOutputSupplier(IndexIO.makeDimFile(v8OutDir, dimension), true); - dimOuts.add(dimOut); - - writer.close(); - serializerUtils.writeString(dimOut, dimension); - ByteStreams.copy(writer.combineStreams(), dimOut); - - ioPeon.cleanup(); + for (int dimIndex = 0; dimIndex < mergedDimensions.size(); ++dimIndex) { + long dimStartTime = System.currentTimeMillis(); + mergers.get(dimIndex).mergeAcrossSegments(indexes); } + log.info("outDir[%s] completed dim conversions in %,d millis.", v8OutDir, System.currentTimeMillis() - startTime); /************* Walk through data sets and merge them *************/ @@ -750,9 +692,10 @@ public void close() throws IOException indexes, mergedDimensions, mergedMetrics, - dimConversions, - convertMissingDimsFlags, - rowMergerFn + rowMergerFn, + dimCapabilities, + handlers, + mergers ); CompressedLongsSupplierSerializer timeWriter = CompressedLongsSupplierSerializer.create( @@ -761,13 +704,6 @@ public void close() throws IOException timeWriter.open(); - ArrayList forwardDimWriters = Lists.newArrayListWithCapacity(mergedDimensions.size()); - for (String dimension : mergedDimensions) { - VSizeIndexedWriter writer = new VSizeIndexedWriter(ioPeon, dimension, dimensionCardinalities.get(dimension)); - writer.open(); - forwardDimWriters.add(writer); - } - ArrayList metWriters = Lists.newArrayListWithCapacity(mergedMetrics.size()); for (String metric : mergedMetrics) { ValueType type = valueTypes.get(metric); @@ -815,12 +751,10 @@ public void close() throws IOException metWriters.get(i).serialize(metrics[i]); } - int[][] dims = theRow.getDims(); + Comparable[][] dims = theRow.getDims(); for (int i = 0; i < dims.length; ++i) { - List listToWrite = (i >= dims.length || dims[i] == null) - ? null - : Ints.asList(dims[i]); - forwardDimWriters.get(i).write(listToWrite); + String dimName = mergedDimensions.get(i); + mergers.get(i).addColumnValue(dims[i]); } for (Map.Entry> comprisedRow : theRow.getComprisedRows().entrySet()) { @@ -852,16 +786,10 @@ public void close() throws IOException timeWriter.closeAndConsolidate(out); IndexIO.checkFileSize(timeFile); - for (int i = 0; i < mergedDimensions.size(); ++i) { - forwardDimWriters.get(i).close(); - ByteStreams.copy(forwardDimWriters.get(i).combineStreams(), dimOuts.get(i)); - } - for (MetricColumnSerializer metWriter : metWriters) { metWriter.close(); } - ioPeon.cleanup(); log.info( "outDir[%s] completed walk through of %,d rows in %,d millis.", v8OutDir, @@ -869,7 +797,7 @@ public void close() throws IOException System.currentTimeMillis() - startTime ); - /************ Create Inverted Indexes *************/ + /************ Create Inverted Indexes and Finalize Columns *************/ startTime = System.currentTimeMillis(); final File invertedFile = new File(v8OutDir, "inverted.drd"); @@ -880,107 +808,33 @@ public void close() throws IOException Files.touch(geoFile); OutputSupplier spatialOut = Files.newOutputStreamSupplier(geoFile, true); - for (int i = 0; i < mergedDimensions.size(); ++i) { - long dimStartTime = System.currentTimeMillis(); - String dimension = mergedDimensions.get(i); - - File dimOutFile = dimOuts.get(i).getFile(); - final MappedByteBuffer dimValsMapped = Files.map(dimOutFile); - - if (!dimension.equals(serializerUtils.readString(dimValsMapped))) { - throw new ISE("dimensions[%s] didn't equate!? This is a major WTF moment.", dimension); - } - Indexed dimVals = GenericIndexed.read(dimValsMapped, GenericIndexed.STRING_STRATEGY); - log.info("Starting dimension[%s] with cardinality[%,d]", dimension, dimVals.size()); - - final BitmapSerdeFactory bitmapSerdeFactory = indexSpec.getBitmapSerdeFactory(); - GenericIndexedWriter writer = new GenericIndexedWriter<>( - ioPeon, dimension, bitmapSerdeFactory.getObjectStrategy() - ); - writer.open(); - - boolean isSpatialDim = columnCapabilities.get(dimension).hasSpatialIndexes(); - ByteBufferWriter spatialWriter = null; - RTree tree = null; - IOPeon spatialIoPeon = new TmpFileIOPeon(); - if (isSpatialDim) { - BitmapFactory bitmapFactory = bitmapSerdeFactory.getBitmapFactory(); - spatialWriter = new ByteBufferWriter( - spatialIoPeon, dimension, new IndexedRTree.ImmutableRTreeObjectStrategy(bitmapFactory) - ); - spatialWriter.open(); - tree = new RTree(2, new LinearGutmanSplitStrategy(0, 50, bitmapFactory), bitmapFactory); - } - - IndexSeeker[] dictIdSeeker = toIndexSeekers(indexes, dimConversions, dimension); - - //Iterate all dim values's dictionary id in ascending order which in line with dim values's compare result. - for (int dictId = 0; dictId < dimVals.size(); dictId++) { - progress.progress(); - List> convertedInverteds = Lists.newArrayListWithCapacity(indexes.size()); - for (int j = 0; j < indexes.size(); ++j) { - int seekedDictId = dictIdSeeker[j].seek(dictId); - if (seekedDictId != IndexSeeker.NOT_EXIST) { - convertedInverteds.add( - new ConvertingIndexedInts( - indexes.get(j).getBitmapIndex(dimension, seekedDictId), rowNumConversions.get(j) - ) - ); - } - } - - MutableBitmap bitset = bitmapSerdeFactory.getBitmapFactory().makeEmptyMutableBitmap(); - for (Integer row : CombiningIterable.createSplatted( - convertedInverteds, - Ordering.natural().nullsFirst() - )) { - if (row != INVALID_ROW) { - bitset.add(row); - } - } + for (DimensionMerger merger : mergers) { + merger.closeWritersV8(rowNumConversions, out, spatialOut); + } - writer.write( - bitmapSerdeFactory.getBitmapFactory().makeImmutableBitmap(bitset) - ); + log.info("outDir[%s] completed inverted.drd and wrote dimensions in %,d millis.", v8OutDir, System.currentTimeMillis() - startTime); - if (isSpatialDim) { - String dimVal = dimVals.get(dictId); - if (dimVal != null) { - List stringCoords = Lists.newArrayList(SPLITTER.split(dimVal)); - float[] coords = new float[stringCoords.size()]; - for (int j = 0; j < coords.length; j++) { - coords[j] = Float.valueOf(stringCoords.get(j)); - } - tree.insert(coords, bitset); - } + Function dimFilenameFunction = new Function() + { + @Override + public String apply(@Nullable String input) + { + String formatString; + if (columnCapabilities.get(input).isDictionaryEncoded()) { + formatString = "dim_%s.drd"; + } else { + formatString = String.format("numeric_dim_%%s_%s.drd", IndexIO.BYTE_ORDER); } + return GuavaUtils.formatFunction(formatString).apply(input); } - writer.close(); - - serializerUtils.writeString(out, dimension); - ByteStreams.copy(writer.combineStreams(), out); - ioPeon.cleanup(); - - log.info("Completed dimension[%s] in %,d millis.", dimension, System.currentTimeMillis() - dimStartTime); - - if (isSpatialDim) { - spatialWriter.write(ImmutableRTree.newImmutableFromMutable(tree)); - spatialWriter.close(); - - serializerUtils.writeString(spatialOut, dimension); - ByteStreams.copy(spatialWriter.combineStreams(), spatialOut); - spatialIoPeon.cleanup(); - } - } - - log.info("outDir[%s] completed inverted.drd in %,d millis.", v8OutDir, System.currentTimeMillis() - startTime); + }; final ArrayList expectedFiles = Lists.newArrayList( Iterables.concat( Arrays.asList( "index.drd", "inverted.drd", "spatial.drd", String.format("time_%s.drd", IndexIO.BYTE_ORDER) ), - Iterables.transform(mergedDimensions, GuavaUtils.formatFunction("dim_%s.drd")), + Iterables.transform(mergedDimensions, dimFilenameFunction), Iterables.transform( mergedMetrics, GuavaUtils.formatFunction(String.format("met_%%s_%s.drd", IndexIO.BYTE_ORDER)) ) @@ -1032,13 +886,25 @@ public void close() throws IOException } } + protected DimensionHandler[] makeDimensionHandlers(final List mergedDimensions, final List dimCapabilities) + { + final DimensionHandler[] handlers = new DimensionHandler[mergedDimensions.size()]; + for (int i = 0; i < mergedDimensions.size(); i++) { + ColumnCapabilities capabilities = dimCapabilities.get(i); + String dimName = mergedDimensions.get(i); + handlers[i] = DimensionHandlerUtil.getHandlerFromCapabilities(dimName, capabilities); + } + return handlers; + } + protected Iterable makeRowIterable( List indexes, final List mergedDimensions, final List mergedMetrics, - ArrayList> dimConversions, - ArrayList convertMissingDimsFlags, - Function>, Iterable> rowMergerFn + Function>, Iterable> rowMergerFn, + final List dimCapabilities, + final DimensionHandler[] handlers, + final List mergers ) { ArrayList> boats = Lists.newArrayListWithCapacity(indexes.size()); @@ -1046,8 +912,8 @@ protected Iterable makeRowIterable( for (int i = 0; i < indexes.size(); ++i) { final IndexableAdapter adapter = indexes.get(i); - final int[] dimLookup = toLookupMap(adapter.getDimensionNames(), mergedDimensions); - final int[] metricLookup = toLookupMap(adapter.getMetricNames(), mergedMetrics); + final int[] dimLookup = getColumnIndexReorderingMap(adapter.getDimensionNames(), mergedDimensions); + final int[] metricLookup = getColumnIndexReorderingMap(adapter.getMetricNames(), mergedMetrics); Iterable target = indexes.get(i).getRows(); if (dimLookup != null || metricLookup != null) { @@ -1059,14 +925,18 @@ protected Iterable makeRowIterable( @Override public Rowboat apply(Rowboat input) { - int[][] newDims = input.getDims(); + Comparable[][] newDims = new Comparable[mergedDimensions.size()][]; if (dimLookup != null) { - newDims = new int[mergedDimensions.size()][]; + newDims = new Comparable[mergedDimensions.size()][]; int j = 0; - for (int[] dim : input.getDims()) { + for (Comparable[] dim : input.getDims()) { newDims[dimLookup[j]] = dim; j++; } + } else { + // It's possible for getColumnIndexReorderingMap to return null when + // both column lists are identical. Copy the old array, no dimension reordering is needed. + newDims = input.getDims(); } Object[] newMetrics = input.getMetrics(); @@ -1077,13 +947,16 @@ public Rowboat apply(Rowboat input) newMetrics[metricLookup[j]] = met; j++; } + } else { + newMetrics = input.getMetrics(); } return new Rowboat( input.getTimestamp(), newDims, newMetrics, - input.getRowNum() + input.getRowNum(), + handlers ); } } @@ -1091,7 +964,7 @@ public Rowboat apply(Rowboat input) } boats.add( new MMappedIndexRowIterable( - target, mergedDimensions, dimConversions.get(i), i, convertMissingDimsFlags + target, mergedDimensions, i, dimCapabilities, mergers ) ); } @@ -1099,14 +972,16 @@ public Rowboat apply(Rowboat input) return rowMergerFn.apply(boats); } - private int[] toLookupMap(Indexed indexed, List values) + // If an adapter's column list differs from the merged column list across multiple indexes, + // return an array that maps the adapter's column orderings to the larger, merged column ordering + private int[] getColumnIndexReorderingMap(Indexed adapterColumnNames, List mergedColumnNames) { - if (isSame(indexed, values)) { - return null; // no need to convert + if (isSame(adapterColumnNames, mergedColumnNames)) { + return null; // no need to convert if column lists are identical } - int[] dimLookup = new int[values.size()]; - for (int i = 0; i < indexed.size(); i++) { - dimLookup[i] = values.indexOf(indexed.get(i)); + int[] dimLookup = new int[mergedColumnNames.size()]; + for (int i = 0; i < adapterColumnNames.size(); i++) { + dimLookup[i] = mergedColumnNames.indexOf(adapterColumnNames.get(i)); } return dimLookup; } @@ -1306,24 +1181,27 @@ public static class MMappedIndexRowIterable implements Iterable { private final Iterable index; private final List convertedDims; - private final Map converters; private final int indexNumber; - private final ArrayList convertMissingDimsFlags; - private static final int[] EMPTY_STR_DIM = new int[]{0}; + private final List dimCapabilities; + private final List mergers; + private static final Comparable[] EMPTY_STR_DIM = new Comparable[]{0}; + private static final Comparable[] EMPTY_LONG_DIM = new Comparable[]{0L}; + private static final Comparable[] EMPTY_FLOAT_DIM = new Comparable[]{0.0f}; + MMappedIndexRowIterable( Iterable index, List convertedDims, - Map converters, int indexNumber, - ArrayList convertMissingDimsFlags + final List dimCapabilities, + final List mergers ) { this.index = index; this.convertedDims = convertedDims; - this.converters = converters; this.indexNumber = indexNumber; - this.convertMissingDimsFlags = convertMissingDimsFlags; + this.dimCapabilities = dimCapabilities; + this.mergers = mergers; } public Iterable getIndex() @@ -1334,18 +1212,6 @@ public Iterable getIndex() @Override public Iterator iterator() { - final IntBuffer[] converterArray = FluentIterable - .from(convertedDims) - .transform( - new Function() - { - @Override - public IntBuffer apply(String input) - { - return converters.get(input); - } - } - ).toArray(IntBuffer.class); return Iterators.transform( index.iterator(), new Function() @@ -1353,28 +1219,46 @@ public IntBuffer apply(String input) @Override public Rowboat apply(@Nullable Rowboat input) { - int[][] dims = input.getDims(); - int[][] newDims = new int[convertedDims.size()][]; + Comparable[][] dims = input.getDims(); + Comparable[][] newDims = new Comparable[convertedDims.size()][]; for (int i = 0; i < convertedDims.size(); ++i) { - IntBuffer converter = converterArray[i]; if (i >= dims.length) { continue; } - if (dims[i] == null && convertMissingDimsFlags.get(i)) { - newDims[i] = EMPTY_STR_DIM; - continue; - } - - if (converter == null) { - newDims[i] = dims[i]; - continue; + // For strings, convert missing values to null/empty if conversion flag is set + // But if bitmap/dictionary is not used, always convert missing to 0 + if (dims[i] == null) { + if (mergers.get(i).needsConvertMissingValues()) { + switch (dimCapabilities.get(i).getType()) { + case STRING: + newDims[i] = EMPTY_STR_DIM; + break; + case LONG: + newDims[i] = EMPTY_LONG_DIM; + break; + case FLOAT: + newDims[i] = EMPTY_FLOAT_DIM; + break; + default: + break; + } + continue; + } else { + newDims[i] = null; + continue; + } } - newDims[i] = new int[dims[i].length]; + newDims[i] = new Comparable[dims[i].length]; for (int j = 0; j < dims[i].length; ++j) { - newDims[i][j] = converter.get(dims[i][j]); + try { + newDims[i][j] = mergers.get(i).getConvertedEncodedValue(dims[i][j], indexNumber); + } catch (Exception e) { + System.out.println("A"); + throw e; + } } } @@ -1382,7 +1266,8 @@ public Rowboat apply(@Nullable Rowboat input) input.getTimestamp(), newDims, input.getMetrics(), - input.getRowNum() + input.getRowNum(), + input.getHandlers() ); retVal.addRow(indexNumber, input.getRowNum()); @@ -1462,7 +1347,8 @@ public Rowboat apply(Rowboat lhs, Rowboat rhs) lhs.getTimestamp(), lhs.getDims(), metrics, - lhs.getRowNum() + lhs.getRowNum(), + lhs.getHandlers() ); for (Rowboat rowboat : Arrays.asList(lhs, rhs)) { diff --git a/processing/src/main/java/io/druid/segment/IndexMergerV9.java b/processing/src/main/java/io/druid/segment/IndexMergerV9.java index ae555ad967ff..9c9a0625f79d 100644 --- a/processing/src/main/java/io/druid/segment/IndexMergerV9.java +++ b/processing/src/main/java/io/druid/segment/IndexMergerV9.java @@ -37,11 +37,11 @@ import com.metamx.collections.spatial.ImmutableRTree; import com.metamx.collections.spatial.RTree; import com.metamx.collections.spatial.split.LinearGutmanSplitStrategy; +import com.metamx.common.IAE; import com.metamx.common.ISE; import com.metamx.common.io.smoosh.FileSmoosher; import com.metamx.common.io.smoosh.SmooshedWriter; import com.metamx.common.logger.Logger; -import io.druid.collections.CombiningIterable; import io.druid.common.utils.JodaUtils; import io.druid.query.aggregation.AggregatorFactory; import io.druid.segment.column.Column; @@ -49,25 +49,14 @@ import io.druid.segment.column.ColumnCapabilitiesImpl; import io.druid.segment.column.ColumnDescriptor; import io.druid.segment.column.ValueType; -import io.druid.segment.data.BitmapSerdeFactory; -import io.druid.segment.data.ByteBufferWriter; import io.druid.segment.data.CompressedObjectStrategy; -import io.druid.segment.data.CompressedVSizeIndexedV3Writer; -import io.druid.segment.data.CompressedVSizeIntsIndexedWriter; import io.druid.segment.data.GenericIndexed; -import io.druid.segment.data.GenericIndexedWriter; import io.druid.segment.data.IOPeon; -import io.druid.segment.data.Indexed; -import io.druid.segment.data.IndexedIntsWriter; -import io.druid.segment.data.IndexedRTree; import io.druid.segment.data.TmpFileIOPeon; -import io.druid.segment.data.VSizeIndexedIntsWriter; -import io.druid.segment.data.VSizeIndexedWriter; import io.druid.segment.serde.ComplexColumnPartSerde; import io.druid.segment.serde.ComplexColumnSerializer; import io.druid.segment.serde.ComplexMetricSerde; import io.druid.segment.serde.ComplexMetrics; -import io.druid.segment.serde.DictionaryEncodedColumnPartSerde; import io.druid.segment.serde.FloatGenericColumnPartSerde; import io.druid.segment.serde.LongGenericColumnPartSerde; import org.apache.commons.io.FileUtils; @@ -182,18 +171,21 @@ public void close() throws IOException final List dimCapabilities = Lists.newArrayListWithCapacity(mergedDimensions.size()); mergeCapabilities(adapters, mergedDimensions, metricsValueTypes, metricTypeNames, dimCapabilities); + final DimensionHandler[] handlers = makeDimensionHandlers(mergedDimensions, dimCapabilities); + final List mergers = new ArrayList<>(); + for (int i = 0; i < mergedDimensions.size(); i++) { + mergers.add(handlers[i].makeMerger(indexSpec, v9TmpDir, ioPeon, v9Smoosher, mapper, dimCapabilities.get(i), progress)); + mergers.get(i).initWriters(); + } + /************* Setup Dim Conversions **************/ progress.progress(); startTime = System.currentTimeMillis(); - final Map dimCardinalities = Maps.newHashMap(); - final ArrayList> dimValueWriters = setupDimValueWriters(ioPeon, mergedDimensions); final ArrayList> dimConversions = Lists.newArrayListWithCapacity(adapters.size()); final ArrayList dimensionSkipFlag = Lists.newArrayListWithCapacity(mergedDimensions.size()); - final ArrayList dimHasNullFlags = Lists.newArrayListWithCapacity(mergedDimensions.size()); final ArrayList convertMissingDimsFlags = Lists.newArrayListWithCapacity(mergedDimensions.size()); writeDimValueAndSetupDimConversion( - adapters, progress, mergedDimensions, dimCardinalities, dimValueWriters, dimensionSkipFlag, dimConversions, - convertMissingDimsFlags, dimHasNullFlags + adapters, progress, mergedDimensions, mergers ); log.info("Completed dim conversions in %,d millis.", System.currentTimeMillis() - startTime); @@ -203,53 +195,37 @@ public void close() throws IOException adapters, mergedDimensions, mergedMetrics, - dimConversions, - convertMissingDimsFlags, - rowMergerFn + rowMergerFn, + dimCapabilities, + handlers, + mergers ); final LongColumnSerializer timeWriter = setupTimeWriter(ioPeon); - final ArrayList dimWriters = setupDimensionWriters( - ioPeon, mergedDimensions, dimCapabilities, dimCardinalities, indexSpec - ); final ArrayList metWriters = setupMetricsWriters( ioPeon, mergedMetrics, metricsValueTypes, metricTypeNames, indexSpec ); final List rowNumConversions = Lists.newArrayListWithCapacity(adapters.size()); - final ArrayList nullRowsList = Lists.newArrayListWithCapacity(mergedDimensions.size()); - for (int i = 0; i < mergedDimensions.size(); ++i) { - nullRowsList.add(indexSpec.getBitmapSerdeFactory().getBitmapFactory().makeEmptyMutableBitmap()); - } - mergeIndexesAndWriteColumns( - adapters, progress, theRows, timeWriter, dimWriters, metWriters, - dimensionSkipFlag, rowNumConversions, nullRowsList, dimHasNullFlags - ); - /************ Create Inverted Indexes *************/ - progress.progress(); - final ArrayList> bitmapIndexWriters = setupBitmapIndexWriters( - ioPeon, mergedDimensions, indexSpec - ); - final ArrayList> spatialIndexWriters = setupSpatialIndexWriters( - ioPeon, mergedDimensions, indexSpec, dimCapabilities - ); - makeInvertedIndexes( - adapters, progress, mergedDimensions, indexSpec, v9TmpDir, rowNumConversions, - nullRowsList, dimValueWriters, bitmapIndexWriters, spatialIndexWriters, dimConversions + mergeIndexesAndWriteColumns( + adapters, progress, theRows, timeWriter, metWriters, rowNumConversions, mergers ); - /************ Finalize Build Columns *************/ - progress.progress(); + /************ Create Inverted Indexes and Finalize Build Columns *************/ + final String section = "build inverted index and columns"; + progress.startSection(section); makeTimeColumn(v9Smoosher, progress, timeWriter); makeMetricsColumns(v9Smoosher, progress, mergedMetrics, metricsValueTypes, metricTypeNames, metWriters); - makeDimensionColumns( - v9Smoosher, progress, indexSpec, mergedDimensions, dimensionSkipFlag, dimCapabilities, - dimValueWriters, dimWriters, bitmapIndexWriters, spatialIndexWriters - ); + + for (DimensionMerger merger : mergers) { + merger.closeWriters(rowNumConversions); + } + + progress.stopSection(section); /************* Make index.drd & metadata.drd files **************/ progress.progress(); makeIndexBinary( - v9Smoosher, adapters, outDir, mergedDimensions, dimensionSkipFlag, mergedMetrics, progress, indexSpec + v9Smoosher, adapters, outDir, mergedDimensions, dimensionSkipFlag, mergedMetrics, progress, indexSpec, mergers ); makeMetadataBinary(v9Smoosher, progress, segmentMetadata); @@ -284,7 +260,8 @@ private void makeIndexBinary( final ArrayList dimensionSkipFlag, final List mergedMetrics, final ProgressIndicator progress, - final IndexSpec indexSpec + final IndexSpec indexSpec, + final List mergers ) throws IOException { final String section = "make index.drd"; @@ -295,7 +272,7 @@ private void makeIndexBinary( final Set finalColumns = Sets.newLinkedHashSet(); finalColumns.addAll(mergedMetrics); for (int i = 0; i < mergedDimensions.size(); ++i) { - if (dimensionSkipFlag.get(i)) { + if (mergers.get(i).canSkip()) { continue; } finalColumns.add(mergedDimensions.get(i)); @@ -338,64 +315,6 @@ private void makeIndexBinary( progress.stopSection(section); } - private void makeDimensionColumns( - final FileSmoosher v9Smoosher, - final ProgressIndicator progress, - final IndexSpec indexSpec, - final List mergedDimensions, - final ArrayList dimensionSkipFlag, - final List dimCapabilities, - final ArrayList> dimValueWriters, - final ArrayList dimWriters, - final ArrayList> bitmapIndexWriters, - final ArrayList> spatialIndexWriters - ) throws IOException - { - final String section = "make dimension columns"; - progress.startSection(section); - - long startTime = System.currentTimeMillis(); - final BitmapSerdeFactory bitmapSerdeFactory = indexSpec.getBitmapSerdeFactory(); - final CompressedObjectStrategy.CompressionStrategy compressionStrategy = indexSpec.getDimensionCompressionStrategy(); - for (int i = 0; i < mergedDimensions.size(); ++i) { - long dimStartTime = System.currentTimeMillis(); - final String dim = mergedDimensions.get(i); - final IndexedIntsWriter dimWriter = dimWriters.get(i); - final GenericIndexedWriter bitmapIndexWriter = bitmapIndexWriters.get(i); - final ByteBufferWriter spatialIndexWriter = spatialIndexWriters.get(i); - - dimWriter.close(); - bitmapIndexWriter.close(); - if (spatialIndexWriter != null) { - spatialIndexWriter.close(); - } - if (dimensionSkipFlag.get(i)) { - continue; - } - - boolean hasMultiValue = dimCapabilities.get(i).hasMultipleValues(); - - final ColumnDescriptor.Builder builder = ColumnDescriptor.builder(); - builder.setValueType(ValueType.STRING); - builder.setHasMultipleValues(hasMultiValue); - final DictionaryEncodedColumnPartSerde.SerializerBuilder partBuilder = DictionaryEncodedColumnPartSerde - .serializerBuilder() - .withDictionary(dimValueWriters.get(i)) - .withValue(dimWriters.get(i), hasMultiValue, compressionStrategy != null) - .withBitmapSerdeFactory(bitmapSerdeFactory) - .withBitmapIndex(bitmapIndexWriters.get(i)) - .withSpatialIndex(spatialIndexWriters.get(i)) - .withByteOrder(IndexIO.BYTE_ORDER); - final ColumnDescriptor serdeficator = builder - .addSerde(partBuilder.build()) - .build(); - makeColumn(v9Smoosher, dim, serdeficator); - log.info("Completed dimension column[%s] in %,d millis.", dim, System.currentTimeMillis() - dimStartTime); - } - log.info("Completed dimension columns in %,d millis.", System.currentTimeMillis() - startTime); - progress.stopSection(section); - } - private void makeMetricsColumns( final FileSmoosher v9Smoosher, final ProgressIndicator progress, @@ -455,7 +374,6 @@ private void makeMetricsColumns( progress.stopSection(section); } - private void makeTimeColumn( final FileSmoosher v9Smoosher, final ProgressIndicator progress, @@ -505,164 +423,14 @@ private void makeColumn( } } - private void makeInvertedIndexes( - final List adapters, - final ProgressIndicator progress, - final List mergedDimensions, - final IndexSpec indexSpec, - final File v9OutDir, - final List rowNumConversions, - final ArrayList nullRowsList, - final ArrayList> dimValueWriters, - final ArrayList> bitmapIndexWriters, - final ArrayList> spatialIndexWriters, - final ArrayList> dimConversions - ) throws IOException - { - final String section = "build inverted index"; - progress.startSection(section); - - long startTime = System.currentTimeMillis(); - final BitmapSerdeFactory bitmapSerdeFactory = indexSpec.getBitmapSerdeFactory(); - for (int dimIndex = 0; dimIndex < mergedDimensions.size(); ++dimIndex) { - String dimension = mergedDimensions.get(dimIndex); - long dimStartTime = System.currentTimeMillis(); - - // write dim values to one single file because we need to read it - File dimValueFile = IndexIO.makeDimFile(v9OutDir, dimension); - FileOutputStream fos = new FileOutputStream(dimValueFile); - ByteStreams.copy(dimValueWriters.get(dimIndex).combineStreams(), fos); - fos.close(); - - final MappedByteBuffer dimValsMapped = Files.map(dimValueFile); - Indexed dimVals = GenericIndexed.read(dimValsMapped, GenericIndexed.STRING_STRATEGY); - - ByteBufferWriter spatialIndexWriter = spatialIndexWriters.get(dimIndex); - RTree tree = null; - if (spatialIndexWriter != null) { - BitmapFactory bitmapFactory = bitmapSerdeFactory.getBitmapFactory(); - tree = new RTree(2, new LinearGutmanSplitStrategy(0, 50, bitmapFactory), bitmapFactory); - } - - IndexSeeker[] dictIdSeeker = toIndexSeekers(adapters, dimConversions, dimension); - - ImmutableBitmap nullRowBitmap = bitmapSerdeFactory.getBitmapFactory().makeImmutableBitmap( - nullRowsList.get(dimIndex) - ); - - //Iterate all dim values's dictionary id in ascending order which in line with dim values's compare result. - for (int dictId = 0; dictId < dimVals.size(); dictId++) { - progress.progress(); - List> convertedInverteds = Lists.newArrayListWithCapacity(adapters.size()); - for (int j = 0; j < adapters.size(); ++j) { - int seekedDictId = dictIdSeeker[j].seek(dictId); - if (seekedDictId != IndexSeeker.NOT_EXIST) { - convertedInverteds.add( - new ConvertingIndexedInts( - adapters.get(j).getBitmapIndex(dimension, seekedDictId), rowNumConversions.get(j) - ) - ); - } - } - - MutableBitmap bitset = bitmapSerdeFactory.getBitmapFactory().makeEmptyMutableBitmap(); - for (Integer row : CombiningIterable.createSplatted( - convertedInverteds, - Ordering.natural().nullsFirst() - )) { - if (row != INVALID_ROW) { - bitset.add(row); - } - } - - ImmutableBitmap bitmapToWrite = bitmapSerdeFactory.getBitmapFactory().makeImmutableBitmap(bitset); - if ((dictId == 0) && (Iterables.getFirst(dimVals, "") == null)) { - bitmapToWrite = nullRowBitmap.union(bitmapToWrite); - } - bitmapIndexWriters.get(dimIndex).write(bitmapToWrite); - - if (spatialIndexWriter != null) { - String dimVal = dimVals.get(dictId); - if (dimVal != null) { - List stringCoords = Lists.newArrayList(SPLITTER.split(dimVal)); - float[] coords = new float[stringCoords.size()]; - for (int j = 0; j < coords.length; j++) { - coords[j] = Float.valueOf(stringCoords.get(j)); - } - tree.insert(coords, bitset); - } - } - } - if (spatialIndexWriter != null) { - spatialIndexWriter.write(ImmutableRTree.newImmutableFromMutable(tree)); - } - log.info( - "Completed dim[%s] inverted with cardinality[%,d] in %,d millis.", - dimension, - dimVals.size(), - System.currentTimeMillis() - dimStartTime - ); - } - log.info("Completed inverted index in %,d millis.", System.currentTimeMillis() - startTime); - progress.stopSection(section); - } - - - private ArrayList> setupBitmapIndexWriters( - final IOPeon ioPeon, - final List mergedDimensions, - final IndexSpec indexSpec - ) throws IOException - { - ArrayList> writers = Lists.newArrayListWithCapacity(mergedDimensions.size()); - final BitmapSerdeFactory bitmapSerdeFactory = indexSpec.getBitmapSerdeFactory(); - for (String dimension : mergedDimensions) { - GenericIndexedWriter writer = new GenericIndexedWriter<>( - ioPeon, String.format("%s.inverted", dimension), bitmapSerdeFactory.getObjectStrategy() - ); - writer.open(); - writers.add(writer); - } - return writers; - } - - private ArrayList> setupSpatialIndexWriters( - final IOPeon ioPeon, - final List mergedDimensions, - final IndexSpec indexSpec, - final List dimCapabilities - ) throws IOException - { - ArrayList> writers = Lists.newArrayListWithCapacity(mergedDimensions.size()); - final BitmapSerdeFactory bitmapSerdeFactory = indexSpec.getBitmapSerdeFactory(); - for (int dimIndex = 0; dimIndex < mergedDimensions.size(); ++dimIndex) { - if (dimCapabilities.get(dimIndex).hasSpatialIndexes()) { - BitmapFactory bitmapFactory = bitmapSerdeFactory.getBitmapFactory(); - ByteBufferWriter writer = new ByteBufferWriter<>( - ioPeon, - String.format("%s.spatial", mergedDimensions.get(dimIndex)), - new IndexedRTree.ImmutableRTreeObjectStrategy(bitmapFactory) - ); - writer.open(); - writers.add(writer); - } else { - writers.add(null); - } - } - return writers; - } - private void mergeIndexesAndWriteColumns( final List adapters, final ProgressIndicator progress, final Iterable theRows, final LongColumnSerializer timeWriter, - final ArrayList dimWriters, final ArrayList metWriters, - final ArrayList dimensionSkipFlag, final List rowNumConversions, - final ArrayList nullRowsList, - final ArrayList dimHasNullFlags + final List mergers ) throws IOException { final String section = "walk through and merge rows"; @@ -686,19 +454,9 @@ private void mergeIndexesAndWriteColumns( metWriters.get(i).serialize(metrics[i]); } - int[][] dims = theRow.getDims(); + Comparable[][] dims = theRow.getDims(); for (int i = 0; i < dims.length; ++i) { - if (dimensionSkipFlag.get(i)) { - continue; - } - if (dims[i] == null || dims[i].length == 0) { - nullRowsList.get(i).add(rowCount); - } else if (dimHasNullFlags.get(i) && dims[i].length == 1 && dims[i][0] == 0) { - // If this dimension has the null/empty str in its dictionary, a row with a single-valued dimension - // that matches the null/empty str's dictionary ID should also be added to nullRowsList. - nullRowsList.get(i).add(rowCount); - } - dimWriters.get(i).add(dims[i]); + mergers.get(i).addColumnValue(dims[i]); } for (Map.Entry> comprisedRow : theRow.getComprisedRows().entrySet()) { @@ -771,153 +529,18 @@ private ArrayList setupMetricsWriters( return metWriters; } - private ArrayList setupDimensionWriters( - final IOPeon ioPeon, - final List mergedDimensions, - final List dimCapabilities, - final Map dimCardinalities, - final IndexSpec indexSpec - ) throws IOException - { - ArrayList dimWriters = Lists.newArrayListWithCapacity(mergedDimensions.size()); - final CompressedObjectStrategy.CompressionStrategy dimCompression = indexSpec.getDimensionCompressionStrategy(); - for (int dimIndex = 0; dimIndex < mergedDimensions.size(); ++dimIndex) { - String dim = mergedDimensions.get(dimIndex); - int cardinality = dimCardinalities.get(dim); - ColumnCapabilitiesImpl capabilities = dimCapabilities.get(dimIndex); - String filenameBase = String.format("%s.forward_dim", dim); - IndexedIntsWriter writer; - if (capabilities.hasMultipleValues()) { - writer = (dimCompression != null) - ? CompressedVSizeIndexedV3Writer.create(ioPeon, filenameBase, cardinality, dimCompression) - : new VSizeIndexedWriter(ioPeon, filenameBase, cardinality); - } else { - writer = (dimCompression != null) - ? CompressedVSizeIntsIndexedWriter.create(ioPeon, filenameBase, cardinality, dimCompression) - : new VSizeIndexedIntsWriter(ioPeon, filenameBase, cardinality); - } - writer.open(); - // we will close these writers in another method after we added all the values - dimWriters.add(writer); - } - return dimWriters; - } - - private ArrayList> setupDimValueWriters( - final IOPeon ioPeon, - final List mergedDimensions - ) - throws IOException - { - ArrayList> dimValueWriters = Lists.newArrayListWithCapacity(mergedDimensions.size()); - for (String dimension : mergedDimensions) { - final GenericIndexedWriter writer = new GenericIndexedWriter<>( - ioPeon, String.format("%s.dim_values", dimension), GenericIndexed.STRING_STRATEGY - ); - writer.open(); - dimValueWriters.add(writer); - } - return dimValueWriters; - } - private void writeDimValueAndSetupDimConversion( final List indexes, final ProgressIndicator progress, final List mergedDimensions, - final Map dimensionCardinalities, - final ArrayList> dimValueWriters, - final ArrayList dimensionSkipFlag, - final List> dimConversions, - final ArrayList convertMissingDimsFlags, - final ArrayList dimHasNullFlags + final List mergers ) throws IOException { final String section = "setup dimension conversions"; progress.startSection(section); - for (int i = 0; i < indexes.size(); ++i) { - dimConversions.add(Maps.newHashMap()); - } - for (int dimIndex = 0; dimIndex < mergedDimensions.size(); ++dimIndex) { - long dimStartTime = System.currentTimeMillis(); - String dimension = mergedDimensions.get(dimIndex); - boolean dimHasNull = false; - boolean dimHasValues = false; - boolean dimAbsentFromSomeIndex = false; - - int numMergeIndex = 0; - Indexed dimValueLookup = null; - Indexed[] dimValueLookups = new Indexed[indexes.size() + 1]; - for (int i = 0; i < indexes.size(); i++) { - Indexed dimValues = indexes.get(i).getDimValueLookup(dimension); - if (!isNullColumn(dimValues)) { - dimHasValues = true; - dimHasNull |= dimValues.indexOf(null) >= 0; - dimValueLookups[i] = dimValueLookup = dimValues; - numMergeIndex++; - } else { - dimAbsentFromSomeIndex = true; - } - } - - boolean convertMissingDims = dimHasValues && dimAbsentFromSomeIndex; - convertMissingDimsFlags.add(convertMissingDims); - - /* - * Ensure the empty str is always in the dictionary if the dimension was missing from one index but - * has non-null values in another index. - * This is done so that MMappedIndexRowIterable can convert null columns to empty strings - * later on, to allow rows from indexes without a particular dimension to merge correctly with - * rows from indexes with null/empty str values for that dimension. - */ - if (convertMissingDims && !dimHasNull) { - dimHasNull = true; - dimValueLookups[indexes.size()] = dimValueLookup = EMPTY_STR_DIM_VAL; - numMergeIndex++; - } - - GenericIndexedWriter writer = dimValueWriters.get(dimIndex); - - int cardinality = 0; - if (numMergeIndex > 1) { - DictionaryMergeIterator iterator = new DictionaryMergeIterator(dimValueLookups, true); - - while (iterator.hasNext()) { - writer.write(iterator.next()); - } - - for (int i = 0; i < indexes.size(); i++) { - if (dimValueLookups[i] != null && iterator.needConversion(i)) { - dimConversions.get(i).put(dimension, iterator.conversions[i]); - } - } - cardinality = iterator.counter; - } else if (numMergeIndex == 1) { - for (String value : dimValueLookup) { - writer.write(value); - } - cardinality = dimValueLookup.size(); - } - - // Mark if this dim has the null/empty str value in its dictionary, used for determining nullRowsList later. - dimHasNullFlags.add(dimHasNull); - - log.info( - "Completed dim[%s] conversions with cardinality[%,d] in %,d millis.", - dimension, - cardinality, - System.currentTimeMillis() - dimStartTime - ); - dimensionCardinalities.put(dimension, cardinality); - writer.close(); - - if (cardinality == 0) { - log.info(String.format("Skipping [%s], it is empty!", dimension)); - dimensionSkipFlag.add(true); - continue; - } - dimensionSkipFlag.add(false); + mergers.get(dimIndex).mergeAcrossSegments(indexes); } progress.stopSection(section); } @@ -936,7 +559,7 @@ private void mergeCapabilities( ColumnCapabilitiesImpl mergedCapabilities = capabilitiesMap.get(dimension); if (mergedCapabilities == null) { mergedCapabilities = new ColumnCapabilitiesImpl(); - mergedCapabilities.setType(ValueType.STRING); + mergedCapabilities.setType(null); } capabilitiesMap.put(dimension, mergedCapabilities.merge(adapter.getCapabilities(dimension))); } diff --git a/processing/src/main/java/io/druid/segment/IndexSpec.java b/processing/src/main/java/io/druid/segment/IndexSpec.java index 2553c0f6f1cd..69a0f114efab 100644 --- a/processing/src/main/java/io/druid/segment/IndexSpec.java +++ b/processing/src/main/java/io/druid/segment/IndexSpec.java @@ -138,8 +138,7 @@ public CompressedObjectStrategy.CompressionStrategy getDimensionCompressionStrat private static CompressedObjectStrategy.CompressionStrategy dimensionCompressionStrategyForName(String compression) { - return compression.equals(UNCOMPRESSED) ? null : - CompressedObjectStrategy.CompressionStrategy.valueOf(compression.toUpperCase()); + return CompressedObjectStrategy.CompressionStrategy.valueOf(compression.toUpperCase()); } @Override diff --git a/processing/src/main/java/io/druid/segment/LongDimensionHandler.java b/processing/src/main/java/io/druid/segment/LongDimensionHandler.java new file mode 100644 index 000000000000..60ca5d1a9d4e --- /dev/null +++ b/processing/src/main/java/io/druid/segment/LongDimensionHandler.java @@ -0,0 +1,520 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.segment; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.base.Function; +import com.google.common.base.Predicate; +import com.google.common.io.OutputSupplier; +import com.google.common.primitives.Longs; +import com.metamx.collections.bitmap.BitmapFactory; +import com.metamx.collections.bitmap.ImmutableBitmap; +import com.metamx.collections.bitmap.MutableBitmap; +import com.metamx.collections.spatial.search.Bound; +import com.metamx.common.IAE; +import com.metamx.common.io.smoosh.FileSmoosher; +import com.metamx.common.io.smoosh.SmooshedWriter; +import com.metamx.common.logger.Logger; +import com.metamx.common.parsers.ParseException; +import io.druid.common.utils.SerializerUtils; +import io.druid.query.dimension.DimensionSpec; +import io.druid.segment.column.Column; +import io.druid.segment.column.ColumnCapabilities; +import io.druid.segment.column.ColumnDescriptor; +import io.druid.segment.column.GenericColumn; +import io.druid.segment.column.ValueType; +import io.druid.segment.data.IOPeon; +import io.druid.segment.data.Indexed; +import io.druid.segment.serde.LongGenericColumnPartSerde; + +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.IntBuffer; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; + +public class LongDimensionHandler implements DimensionHandler +{ + private static final Logger log = new Logger(LongDimensionHandler.class); + + public static final Function LONG_TRANSFORMER = new Function() + { + @Override + public Long apply(final Object o) + { + if (o == null) { + return null; + } + + if (o instanceof Long) { + return (Long) o; + } + + if (o instanceof String) { + if (((String) o).length() == 0) { + return 0L; + } + try { + return Long.valueOf((String) o); + } + catch (NumberFormatException nfe) { + throw new ParseException(nfe, "Unable to parse value[%s] as long in column: ", o); + } + } + if (o instanceof Number) { + return ((Number) o).longValue(); + } + return null; + } + }; + + private static final Comparator LONG_COMPARATOR = new Comparator() + { + @Override + public int compare(Long o1, Long o2) + { + if (o1 == null) { + return o2 == null ? 0 : -1; + } + if (o2 == null) { + return 1; + } + return Long.compare(o1, o2); + } + }; + + private final String dimensionName; + + public LongDimensionHandler(String dimensionName) + { + this.dimensionName = dimensionName; + } + + @Override + public DimensionSelector getDimensionSelector( + ColumnSelectorFactory cursor, DimensionSpec dimSpec, ColumnCapabilities capabilities + ) + { + return new LongDimensionSelector( + cursor.makeLongColumnSelector(dimensionName), + dimSpec.getExtractionFn(), + capabilities + ); + } + + @Override + public Function getValueTypeTransformer() + { + return LONG_TRANSFORMER; + } + + @Override + public Comparator getEncodedComparator() + { + return LONG_COMPARATOR; + } + + @Override + public Comparator getActualComparator() + { + return LONG_COMPARATOR; + } + + @Override + public byte[] getBytesFromRowValue(Long rowVal) + { + return Longs.toByteArray(rowVal); + } + + @Override + public Long getRowValueFromBytes(byte[] bytes) + { + return Longs.fromByteArray(bytes); + } + + @Override + public int getEncodedValueSize() + { + return Longs.BYTES; + } + + @Override + public DimensionIndexer makeIndexer(Object lock) + { + return new LongDimensionIndexer(); + } + + @Override + public DimensionMerger makeMerger( + IndexSpec indexSpec, + File outDir, + IOPeon ioPeon, + FileSmoosher v9Smoosher, + ObjectMapper mapper, + ColumnCapabilities capabilities, + ProgressIndicator progress + ) + { + return new LongDimensionMerger(indexSpec, outDir, ioPeon, v9Smoosher, mapper, capabilities); + } + + @Override + public DimensionColumnReader makeColumnReader(Column column) + { + return new LongDimensionColumnReader(column); + } + + + public class LongDimensionIndexer implements DimensionIndexer + { + @Override + public Comparable[] processRowValsToIndexKey(Object dimValues) + { + final Comparable[] dimensionValues; + Long transformedVal = 0L; + try { + if (dimValues == null) { + return null; + } else if (dimValues instanceof List) { + List dimValuesList = (List) dimValues; + + if (dimValuesList.size() > 1) { + throw new UnsupportedOperationException("Multi-value rows are not supported on Long dimensions."); + } + + dimensionValues = new Comparable[dimValuesList.size()]; + for (int i = 0; i < dimValuesList.size(); i++) { + transformedVal = LONG_TRANSFORMER.apply(dimValuesList.get(i)); + dimensionValues[i] = transformedVal; + } + } else { + transformedVal = LONG_TRANSFORMER.apply(dimValues); + dimensionValues = new Comparable[]{transformedVal}; + } + } + catch (ParseException pe) { + throw new ParseException(pe.getMessage() + dimensionName); + } + return dimensionValues; + } + + @Override + public Long getActualValue(Long intermediateValue, boolean idSorted) + { + return intermediateValue; + } + + @Override + public Long getEncodedValue(Long fullValue, boolean idSorted) + { + return fullValue; + } + + @Override + public Long getSortedEncodedValueFromUnsorted(Long unsortedIntermediateValue) + { + return unsortedIntermediateValue; + } + + @Override + public Long getUnsortedEncodedValueFromSorted(Long sortedIntermediateValue) + { + return sortedIntermediateValue; + } + + @Override + public Indexed getSortedIndexedValues() + { + throw new UnsupportedOperationException("getSortedIndexedValues() is not supported for Long column: " + + dimensionName); + } + + @Override + public Long getMinValue() + { + return null; + } + + @Override + public Long getMaxValue() + { + return null; + } + + @Override + public void addNullLookup() + { + } + + @Override + public int getCardinality() + { + // DimensionSelector.getValueCardinality() returns Integer.MAX_VALUE for Long and Floats. + // Return the same here for consistency. + return Integer.MAX_VALUE; + } + } + + public class LongDimensionMerger implements DimensionMerger + { + private final IndexSpec indexSpec; + private final File outDir; + private final IOPeon ioPeon; + private final FileSmoosher v9Smoosher; + private final ObjectMapper mapper; + private final ColumnCapabilities capabilities; + + private boolean useV8 = false; + + private GenericColumnSerializer valueWriter; + + private MetricColumnSerializer valueWriterV8; + + public LongDimensionMerger( + IndexSpec indexSpec, + File outDir, + IOPeon ioPeon, + FileSmoosher v9Smoosher, + ObjectMapper mapper, + ColumnCapabilities capabilities + ) + { + this.indexSpec = indexSpec; + this.ioPeon = ioPeon; + this.outDir = outDir; + this.v9Smoosher = v9Smoosher; + this.mapper = mapper; + this.capabilities = capabilities; + } + + + @Override + public void initWriters() throws IOException + { + if (useV8) { + valueWriterV8 = new LongMetricColumnSerializer(dimensionName, outDir, ioPeon); + valueWriterV8.open(); + } else { + String filenameBase = String.format("%s.forward_dim", dimensionName); + valueWriter = LongColumnSerializer.create(ioPeon, filenameBase, indexSpec.getDimensionCompressionStrategy()); + valueWriter.open(); + } + } + + @Override + public void mergeAcrossSegments(List adapters) throws IOException + { + return; + } + + @Override + public void addColumnValue(Comparable[] values) throws IOException + { + // Only singleValue numerics are supported right now. + if (useV8) { + valueWriterV8.serialize(values[0]); + } else { + valueWriter.serialize(values[0]); + } + } + + @Override + public void closeWriters(List rowNumConversions) throws IOException + { + long startTime = System.currentTimeMillis(); + + valueWriter.close(); + + final ColumnDescriptor.Builder builder = ColumnDescriptor.builder() + .setValueType(ValueType.LONG); + + builder.addSerde( + LongGenericColumnPartSerde.serializerBuilder() + .withByteOrder(IndexIO.BYTE_ORDER) + .withDelegate((LongColumnSerializer) valueWriter) + .build()); + final ColumnDescriptor serdeficator = builder.build(); + + makeColumn(v9Smoosher, dimensionName, serdeficator); + + log.info("Completed dimension column[%s] in %,d millis.", dimensionName, System.currentTimeMillis() - startTime); + } + + @Override + public boolean needsConvertMissingValues() + { + return true; + } + + @Override + public boolean canSkip() + { + return false; + } + + @Override + public Long getConvertedEncodedValue(Long value, int indexNumber) + { + return value; + } + + @Override + public void setWriteV8() + { + useV8 = true; + } + + @Override + public void closeWritersV8( + List rowNumConversions, + OutputSupplier invertedOut, + OutputSupplier spatialOut + ) throws IOException + { + valueWriterV8.closeFile( + IndexIO.makeNumericDimFile(outDir, dimensionName, IndexIO.BYTE_ORDER) + ); + } + + private void makeColumn( + final FileSmoosher v9Smoosher, + final String columnName, + final ColumnDescriptor serdeficator + ) throws IOException + { + final SerializerUtils serializerUtils = new SerializerUtils(); + + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + serializerUtils.writeString(baos, mapper.writeValueAsString(serdeficator)); + byte[] specBytes = baos.toByteArray(); + + final SmooshedWriter channel = v9Smoosher.addWithSmooshedWriter( + columnName, serdeficator.numBytes() + specBytes.length + ); + try { + channel.write(ByteBuffer.wrap(specBytes)); + serdeficator.write(channel); + } + finally { + channel.close(); + } + } + } + + public class LongDimensionColumnReader implements DimensionColumnReader + { + private final Column column; + private final GenericColumn genericColumn; + + public LongDimensionColumnReader(Column column) + { + this.column = column; + this.genericColumn = column == null ? null : column.getGenericColumn(); + } + + @Override + public ImmutableBitmap getBitmapIndex( + String value, BitmapFactory bitmapFactory, int numRows + ) + { + if (column == null) { + return bitmapFactory.makeEmptyImmutableBitmap(); + } + final long parsedLong; + try { + parsedLong = Long.parseLong(value); + } + catch (NumberFormatException nfe) { + throw new ParseException("Could not parse value[%s] as long!", value); + } + + final MutableBitmap bitmap = bitmapFactory.makeEmptyMutableBitmap(); + for (int rowIdx = 0; rowIdx < column.getLength(); rowIdx++) { + long rowVal = genericColumn.getLongSingleValueRow(rowIdx); + if (rowVal == parsedLong) { + bitmap.add(rowIdx); + } + } + + return bitmapFactory.makeImmutableBitmap(bitmap); + } + + @Override + public ImmutableBitmap getBitmapIndex( + Predicate predicate, BitmapFactory bitmapFactory + ) + { + if (column == null) { + return bitmapFactory.makeEmptyImmutableBitmap(); + } + + final MutableBitmap bitmap = bitmapFactory.makeEmptyMutableBitmap(); + + Long longVal; + for (int rowIdx = 0; rowIdx < column.getLength(); rowIdx++) { + longVal = genericColumn.getLongSingleValueRow(rowIdx); + if (predicate.apply(longVal)) { + bitmap.add(rowIdx); + } + } + return bitmapFactory.makeImmutableBitmap(bitmap); + } + + @Override + public ImmutableBitmap getBitmapIndex( + Bound bound, BitmapFactory bitmapFactory + ) + { + throw new UnsupportedOperationException("Long column does not support spatial filtering: " + dimensionName); + } + + @Override + public Long getMinValue() + { + return null; + } + + @Override + public Long getMaxValue() + { + return null; + } + + @Override + public Indexed getSortedIndexedValues() + { + throw new UnsupportedOperationException("getSortedIndexedValues() is not supported for Long column: " + + dimensionName); + } + + @Override + public ColumnCapabilities getCapabilities() + { + return column.getCapabilities(); + } + + } + + +} diff --git a/processing/src/main/java/io/druid/segment/LongDimensionSelector.java b/processing/src/main/java/io/druid/segment/LongDimensionSelector.java new file mode 100644 index 000000000000..f9098c48dfdb --- /dev/null +++ b/processing/src/main/java/io/druid/segment/LongDimensionSelector.java @@ -0,0 +1,145 @@ +package io.druid.segment; + +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import com.metamx.common.IAE; +import io.druid.query.extraction.ExtractionFn; +import io.druid.segment.column.ColumnCapabilities; +import io.druid.segment.column.ValueType; +import io.druid.segment.data.IndexedFloats; +import io.druid.segment.data.IndexedInts; +import io.druid.segment.data.IndexedLongs; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +public class LongDimensionSelector implements DimensionSelector +{ + private final LongColumnSelector selector; + private final ExtractionFn extractionFn; + private final ColumnCapabilities capabilities; + private final Map remappedValues = Maps.newHashMap(); + + public LongDimensionSelector(Object selector, ExtractionFn extractionFn, ColumnCapabilities capabilities) + { + this.selector = (LongColumnSelector) selector; + this.extractionFn = extractionFn; + this.capabilities = capabilities; + } + + @Override + public IndexedInts getRow() + { + throw new UnsupportedOperationException("long column does not support getRow"); + } + + @Override + public int getValueCardinality() + { + return Integer.MAX_VALUE; + } + + @Override + public int lookupId(String name) + { + throw new UnsupportedOperationException("long column does not support lookupId"); + } + + @Override + public String lookupName(int id) + { + throw new UnsupportedOperationException("long column does not support lookupName"); + } + + @Override + public IndexedLongs getLongRow() + { + final long val = selector.get(); + return new IndexedLongs() + { + @Override + public int size() + { + // only single-value numerics are supported + return 1; + } + + @Override + public long get(int index) + { + return val; + } + + @Override + public void fill(int index, long[] toFill) + { + throw new UnsupportedOperationException("fill not supported"); + } + + @Override + public int binarySearch(long key) + { + throw new UnsupportedOperationException("binarySearch not supported"); + } + + @Override + public int binarySearch(long key, int from, int to) + { + throw new UnsupportedOperationException("binarySearch not supported"); + } + + @Override + public void close() throws IOException + { + + } + }; + } + + @Override + public Comparable getExtractedValueLong(long val) + { + if (extractionFn == null) { + return val; + } + + String extractedVal = remappedValues.get(val); + if (extractedVal == null) { + extractedVal = extractionFn.apply(val); + remappedValues.put(val, extractedVal); + } + return extractedVal; + } + + @Override + public IndexedFloats getFloatRow() + { + throw new UnsupportedOperationException("long column does not support getFloatRow"); + } + + @Override + public Comparable getExtractedValueFloat(float val) + { + throw new UnsupportedOperationException("long column does not support getExtractedValueFloat"); + } + + @Override + public Comparable getComparableRow() + { + throw new UnsupportedOperationException("long column does not support getComparableRow"); + } + + @Override + public Comparable getExtractedValueComparable(Comparable val) + { + throw new UnsupportedOperationException("long column does not support getExtractedValueComparable"); + } + + @Override + public ColumnCapabilities getDimCapabilities() + { + return capabilities; + } + +} diff --git a/processing/src/main/java/io/druid/segment/LongMetricColumnSerializer.java b/processing/src/main/java/io/druid/segment/LongMetricColumnSerializer.java index acaf20b02249..1af6663632de 100644 --- a/processing/src/main/java/io/druid/segment/LongMetricColumnSerializer.java +++ b/processing/src/main/java/io/druid/segment/LongMetricColumnSerializer.java @@ -70,6 +70,12 @@ public void serialize(Object obj) throws IOException public void close() throws IOException { final File outFile = IndexIO.makeMetricFile(outDir, metricName, IndexIO.BYTE_ORDER); + closeFile(outFile); + } + + @Override + public void closeFile(File outFile) throws IOException + { outFile.delete(); MetricHolder.writeLongMetric( Files.newOutputStreamSupplier(outFile, true), metricName, writer diff --git a/processing/src/main/java/io/druid/segment/MetricColumnSerializer.java b/processing/src/main/java/io/druid/segment/MetricColumnSerializer.java index c3841960be6f..6433b342ca15 100644 --- a/processing/src/main/java/io/druid/segment/MetricColumnSerializer.java +++ b/processing/src/main/java/io/druid/segment/MetricColumnSerializer.java @@ -19,6 +19,7 @@ package io.druid.segment; +import java.io.File; import java.io.IOException; /** @@ -28,4 +29,5 @@ public interface MetricColumnSerializer public void open() throws IOException; public void serialize(Object aggs) throws IOException; public void close() throws IOException; + public void closeFile(File outFile) throws IOException; } diff --git a/processing/src/main/java/io/druid/segment/NullDimensionSelector.java b/processing/src/main/java/io/druid/segment/NullDimensionSelector.java index b5bee79be193..eab6bb0583ef 100644 --- a/processing/src/main/java/io/druid/segment/NullDimensionSelector.java +++ b/processing/src/main/java/io/druid/segment/NullDimensionSelector.java @@ -21,13 +21,33 @@ import com.google.common.base.Strings; import com.google.common.collect.Iterators; +import com.google.common.collect.Lists; +import io.druid.segment.column.ColumnCapabilities; +import io.druid.segment.column.ColumnCapabilitiesImpl; +import io.druid.segment.column.ValueType; +import io.druid.segment.data.IndexedFloats; import io.druid.segment.data.IndexedInts; +import io.druid.segment.data.IndexedLongs; import java.io.IOException; import java.util.Iterator; +import java.util.List; public class NullDimensionSelector implements DimensionSelector { + private final ValueType type; + private ColumnCapabilitiesImpl capabilities; + + public NullDimensionSelector(ValueType type) + { + this.type = type; + this.capabilities = new ColumnCapabilitiesImpl(); + capabilities.setType(type); + if (type == ValueType.STRING) { + capabilities.setHasBitmapIndexes(true); + capabilities.setDictionaryEncoded(true); + } + } private static final IndexedInts SINGLETON = new IndexedInts() { @Override @@ -81,4 +101,108 @@ public int lookupId(String name) { return Strings.isNullOrEmpty(name) ? 0 : -1; } + + @Override + public IndexedLongs getLongRow() + { + return new IndexedLongs() + { + @Override + public int size() + { + return 1; + } + + @Override + public long get(int index) + { + return 0L; + } + + @Override + public void fill(int index, long[] toFill) + { + throw new UnsupportedOperationException("NullDimensionSelector does not support fill"); + } + + @Override + public int binarySearch(long key) + { + return 0; + } + + @Override + public int binarySearch(long key, int from, int to) + { + return 0; + } + + @Override + public void close() throws IOException + { + + } + }; + } + + @Override + public Comparable getExtractedValueLong(long val) + { + return 0L; + } + + @Override + public IndexedFloats getFloatRow() + { + return new IndexedFloats() + { + @Override + public int size() + { + return 1; + } + + @Override + public float get(int index) + { + return 0.0f; + } + + @Override + public void fill(int index, float[] toFill) + { + throw new UnsupportedOperationException("NullDimensionSelector does not support fill"); + } + + @Override + public void close() throws IOException + { + + } + }; + } + + @Override + public Comparable getExtractedValueFloat(float val) + { + return 0.0f; + } + + @Override + public Comparable getComparableRow() + { + return null; + } + + @Override + public Comparable getExtractedValueComparable(Comparable val) + { + return val; + } + + @Override + public ColumnCapabilities getDimCapabilities() + { + return capabilities; + } } diff --git a/processing/src/main/java/io/druid/segment/QueryableIndex.java b/processing/src/main/java/io/druid/segment/QueryableIndex.java index 3daa70896d8f..a5c7fd7709f6 100644 --- a/processing/src/main/java/io/druid/segment/QueryableIndex.java +++ b/processing/src/main/java/io/druid/segment/QueryableIndex.java @@ -25,6 +25,7 @@ import java.io.Closeable; import java.io.IOException; +import java.util.Map; /** */ @@ -36,6 +37,8 @@ public interface QueryableIndex extends ColumnSelector, Closeable public Indexed getAvailableDimensions(); public BitmapFactory getBitmapFactoryForDimensions(); public Metadata getMetadata(); + public Map getDimensionHandlers(); + public Map getDimensionReaders(); /** * The close method shouldn't actually be here as this is nasty. We will adjust it in the future. diff --git a/processing/src/main/java/io/druid/segment/QueryableIndexIndexableAdapter.java b/processing/src/main/java/io/druid/segment/QueryableIndexIndexableAdapter.java index 7cb680a242c7..a225aef378d9 100644 --- a/processing/src/main/java/io/druid/segment/QueryableIndexIndexableAdapter.java +++ b/processing/src/main/java/io/druid/segment/QueryableIndexIndexableAdapter.java @@ -46,6 +46,7 @@ import org.joda.time.Interval; import java.io.Closeable; +import java.util.Collection; import java.util.HashSet; import java.util.Iterator; import java.util.List; @@ -76,10 +77,11 @@ public QueryableIndexIndexableAdapter(QueryableIndex input) if (col == null) { log.warn("Wtf!? column[%s] didn't exist!?!?!?", dim); - } else if (col.getDictionaryEncoding() != null) { - availableDimensions.add(dim); } else { - log.info("No dictionary on dimension[%s]", dim); + if (col.getDictionaryEncoding() == null) { + log.info("No dictionary on dimension[%s]", dim); + } + availableDimensions.add(dim); } } @@ -125,44 +127,8 @@ public Indexed getDimValueLookup(String dimension) return null; } - final DictionaryEncodedColumn dict = column.getDictionaryEncoding(); - - if (dict == null) { - return null; - } - - return new Indexed() - { - @Override - public Class getClazz() - { - return String.class; - } - - @Override - public int size() - { - return dict.getCardinality(); - } - - @Override - public String get(int index) - { - return dict.lookupName(index); - } - - @Override - public int indexOf(String value) - { - return dict.lookupId(value); - } - - @Override - public Iterator iterator() - { - return IndexedIterable.create(this).iterator(); - } - }; + DimensionColumnReader reader = input.getDimensionReaders().get(dimension); + return reader.getSortedIndexedValues(); } @Override @@ -177,28 +143,32 @@ public Iterator iterator() { final GenericColumn timestamps = input.getColumn(Column.TIME_COLUMN_NAME).getGenericColumn(); final Object[] metrics; - - final DictionaryEncodedColumn[] dictionaryEncodedColumns; + final Closeable[] columns; final int numMetrics = getMetricNames().size(); + final DimensionHandler[] handlers = new DimensionHandler[availableDimensions.size()]; + Collection handlerSet = input.getDimensionHandlers().values(); + int currRow = 0; boolean done = false; { - this.dictionaryEncodedColumns = FluentIterable + handlerSet.toArray(handlers); + this.columns = FluentIterable .from(getDimensionNames()) .transform( - new Function() + new Function() { @Override - public DictionaryEncodedColumn apply(String dimName) + public Closeable apply(String dimName) { - return input.getColumn(dimName) - .getDictionaryEncoding(); + Column column = input.getColumn(dimName); + DictionaryEncodedColumn encodedColumn = column.getDictionaryEncoding(); + return encodedColumn != null ? encodedColumn : column.getGenericColumn(); } } - ).toArray(DictionaryEncodedColumn.class); + ).toArray(Closeable.class); final Indexed availableMetrics = getMetricNames(); metrics = new Object[availableMetrics.size()]; @@ -230,7 +200,7 @@ public boolean hasNext() CloseQuietly.close((Closeable) metric); } } - for (Object dimension : dictionaryEncodedColumns) { + for (Object dimension : columns) { if (dimension instanceof Closeable) { CloseQuietly.close((Closeable) dimension); } @@ -247,22 +217,35 @@ public Rowboat next() throw new NoSuchElementException(); } - final int[][] dims = new int[dictionaryEncodedColumns.length][]; + final Comparable[][] dims = new Comparable[columns.length][]; int dimIndex = 0; - for (final DictionaryEncodedColumn dict : dictionaryEncodedColumns) { - final IndexedInts dimVals; - if (dict.hasMultipleValues()) { - dimVals = dict.getMultiValueRow(currRow); - } else { - dimVals = new ArrayBasedIndexedInts(new int[]{dict.getSingleValueRow(currRow)}); - } + for (final Closeable column : columns) { + if (column instanceof DictionaryEncodedColumn) { + DictionaryEncodedColumn dict = (DictionaryEncodedColumn) column; + final IndexedInts dimVals; + if (dict.hasMultipleValues()) { + dimVals = dict.getMultiValueRow(currRow); + } else { + dimVals = new ArrayBasedIndexedInts(new int[]{dict.getSingleValueRow(currRow)}); + } - int[] theVals = new int[dimVals.size()]; - for (int j = 0; j < theVals.length; ++j) { - theVals[j] = dimVals.get(j); - } + Integer[] theVals = new Integer[dimVals.size()]; + for (int j = 0; j < theVals.length; ++j) { + theVals[j] = dimVals.get(j); + } - dims[dimIndex++] = theVals; + dims[dimIndex++] = theVals; + } else if (column instanceof GenericColumn) { + GenericColumn genericCol = (GenericColumn) column; + switch(genericCol.getType()) { + case LONG: + dims[dimIndex++] = new Long[]{genericCol.getLongSingleValueRow(currRow)}; + break; + case FLOAT: + dims[dimIndex++] = new Float[]{genericCol.getFloatSingleValueRow(currRow)}; + break; + } + } } Object[] metricArray = new Object[numMetrics]; @@ -277,7 +260,7 @@ public Rowboat next() } final Rowboat retVal = new Rowboat( - timestamps.getLongSingleValueRow(currRow), dims, metricArray, currRow + timestamps.getLongSingleValueRow(currRow), dims, metricArray, currRow, handlers ); ++currRow; diff --git a/processing/src/main/java/io/druid/segment/QueryableIndexStorageAdapter.java b/processing/src/main/java/io/druid/segment/QueryableIndexStorageAdapter.java index f990f451a6aa..9a2b8a3c76be 100644 --- a/processing/src/main/java/io/druid/segment/QueryableIndexStorageAdapter.java +++ b/processing/src/main/java/io/druid/segment/QueryableIndexStorageAdapter.java @@ -43,7 +43,9 @@ import io.druid.segment.column.GenericColumn; import io.druid.segment.column.ValueType; import io.druid.segment.data.Indexed; +import io.druid.segment.data.IndexedFloats; import io.druid.segment.data.IndexedInts; +import io.druid.segment.data.IndexedLongs; import io.druid.segment.data.Offset; import org.joda.time.DateTime; import org.joda.time.Interval; @@ -51,13 +53,14 @@ import java.io.Closeable; import java.io.IOException; import java.util.Iterator; +import java.util.List; import java.util.Map; /** */ public class QueryableIndexStorageAdapter implements StorageAdapter { - private static final NullDimensionSelector NULL_DIMENSION_SELECTOR = new NullDimensionSelector(); + private static final NullDimensionSelector NULL_DIMENSION_SELECTOR = new NullDimensionSelector(ValueType.STRING); private final QueryableIndex index; @@ -144,23 +147,23 @@ public DateTime getMaxTime() @Override public Comparable getMinValue(String dimension) { - Column column = index.getColumn(dimension); - if (column != null && column.getCapabilities().hasBitmapIndexes()) { - BitmapIndex bitmap = column.getBitmapIndex(); - return bitmap.getCardinality() > 0 ? bitmap.getValue(0) : null; + DimensionColumnReader reader = index.getDimensionReaders().get(dimension); + if (reader == null) { + return null; } - return null; + + return reader.getMinValue(); } @Override public Comparable getMaxValue(String dimension) { - Column column = index.getColumn(dimension); - if (column != null && column.getCapabilities().hasBitmapIndexes()) { - BitmapIndex bitmap = column.getBitmapIndex(); - return bitmap.getCardinality() > 0 ? bitmap.getValue(bitmap.getCardinality() - 1) : null; + DimensionColumnReader reader = index.getDimensionReaders().get(dimension); + if (reader == null) { + return null; } - return null; + + return reader.getMaxValue(); } @Override @@ -172,7 +175,14 @@ public Capabilities getCapabilities() @Override public ColumnCapabilities getColumnCapabilities(String column) { - return index.getColumn(column).getCapabilities(); + Column retColumn = index.getColumn(column); + return retColumn == null ? null : retColumn.getCapabilities(); + } + + @Override + public Map getDimensionHandlers() + { + return index.getDimensionHandlers(); } @Override @@ -219,7 +229,8 @@ public Sequence makeCursors(Filter filter, Interval interval, QueryGranu } else { final ColumnSelectorBitmapIndexSelector selector = new ColumnSelectorBitmapIndexSelector( index.getBitmapFactoryForDimensions(), - index + index, + index.getDimensionReaders() ); offset = new BitmapOffset(selector.getBitmapFactory(), filter.getBitmapIndex(selector), descending); @@ -241,7 +252,7 @@ public Sequence makeCursors(Filter filter, Interval interval, QueryGranu private static class CursorSequenceBuilder { - private final ColumnSelector index; + private final QueryableIndex index; private final Interval interval; private final QueryGranularity gran; private final Offset offset; @@ -250,7 +261,7 @@ private static class CursorSequenceBuilder private final boolean descending; public CursorSequenceBuilder( - ColumnSelector index, + QueryableIndex index, Interval interval, QueryGranularity gran, Offset offset, @@ -371,10 +382,35 @@ public DimensionSelector makeDimensionSelector( DimensionSpec dimensionSpec ) { - return dimensionSpec.decorate(makeDimensionSelectorUndecorated(dimensionSpec)); + final String dimension = dimensionSpec.getDimension(); + final ExtractionFn extractionFn = dimensionSpec.getExtractionFn(); + + if (dimension.equals(Column.TIME_COLUMN_NAME)) { + return new SingleScanTimeDimSelector( + makeLongColumnSelector(dimension), + extractionFn, + descending + ); + } + + final Column columnDesc = index.getColumn(dimension); + if (columnDesc == null) { + return NULL_DIMENSION_SELECTOR; + } + + String dimName = dimensionSpec.getDimension(); + DimensionHandler handler = index.getDimensionHandlers().get(dimName); + if (handler == null) { + return NULL_DIMENSION_SELECTOR; + } + + return dimensionSpec.decorate(handler.getDimensionSelector(this, + dimensionSpec, + index.getColumn(dimName).getCapabilities())); } - private DimensionSelector makeDimensionSelectorUndecorated( + @Override + public DimensionSelector makeDictEncodedStringDimensionSelector( DimensionSpec dimensionSpec ) { @@ -386,14 +422,7 @@ private DimensionSelector makeDimensionSelectorUndecorated( return NULL_DIMENSION_SELECTOR; } - if (dimension.equals(Column.TIME_COLUMN_NAME)) { - return new SingleScanTimeDimSelector( - makeLongColumnSelector(dimension), - extractionFn, - descending - ); - } - + final ColumnCapabilities capabilities = columnDesc.getCapabilities(); DictionaryEncodedColumn cachedColumn = dictionaryColumnCache.get(dimension); if (cachedColumn == null) { cachedColumn = columnDesc.getDictionaryEncoding(); @@ -410,7 +439,8 @@ private DimensionSelector makeDimensionSelectorUndecorated( @Override public IndexedInts getRow() { - return column.getMultiValueRow(cursorOffset.getOffset()); + IndexedInts vals = column.getMultiValueRow(cursorOffset.getOffset()); + return vals; } @Override @@ -423,7 +453,7 @@ public int getValueCardinality() public String lookupName(int id) { final String value = column.lookupName(id); - return extractionFn == null ? + return extractionFn == null ? Strings.nullToEmpty(value) : extractionFn.apply(Strings.nullToEmpty(value)); } @@ -438,15 +468,60 @@ public int lookupId(String name) } return column.lookupId(name); } + + @Override + public IndexedLongs getLongRow() + { + throw new UnsupportedOperationException("getLongRow() is not supported."); + } + + @Override + public Comparable getExtractedValueLong(long val) + { + throw new UnsupportedOperationException("getExtractedValueLong() is not supported."); + } + + @Override + public IndexedFloats getFloatRow() + { + throw new UnsupportedOperationException("getFloatRow() is not supported."); + } + + @Override + public Comparable getExtractedValueFloat(float val) + { + throw new UnsupportedOperationException("getExtractedValueFloat() is not supported."); + } + + @Override + public Comparable getComparableRow() + { + throw new UnsupportedOperationException("getComparableRow() is not supported."); + } + + @Override + public Comparable getExtractedValueComparable(Comparable val) + { + throw new UnsupportedOperationException("getExtractedValueComparable() is not supported."); + + } + + @Override + public ColumnCapabilities getDimCapabilities() + { + return capabilities; + } }; } else { + //return new DimensionSelector() return new DimensionSelector() { @Override + //public DimensionSelectorRow getRow() public IndexedInts getRow() { // using an anonymous class is faster than creating a class that stores a copy of the value - return new IndexedInts() + IndexedInts vals = new IndexedInts() { @Override public int size() @@ -478,6 +553,8 @@ public void close() throws IOException } }; + //return new IndexedIntsDimensionSelectorRow(vals); + return vals; } @Override @@ -503,6 +580,49 @@ public int lookupId(String name) } return column.lookupId(name); } + + @Override + public IndexedLongs getLongRow() + { + throw new UnsupportedOperationException("getLongRow() is not supported."); + } + + @Override + public Comparable getExtractedValueLong(long val) + { + throw new UnsupportedOperationException("getExtractedValueLong() is not supported."); + } + + @Override + public IndexedFloats getFloatRow() + { + throw new UnsupportedOperationException("getFloatRow() is not supported."); + } + + @Override + public Comparable getExtractedValueFloat(float val) + { + throw new UnsupportedOperationException("getExtractedValueFloat() is not supported."); + } + + @Override + public Comparable getComparableRow() + { + throw new UnsupportedOperationException("getComparableRow() is not supported."); + } + + @Override + public Comparable getExtractedValueComparable(Comparable val) + { + throw new UnsupportedOperationException("getExtractedValueComparable() is not supported."); + + } + + @Override + public ColumnCapabilities getDimCapabilities() + { + return capabilities; + } }; } } @@ -808,7 +928,8 @@ public void increment() } @Override - public Offset clone() { + public Offset clone() + { throw new IllegalStateException("clone"); } } diff --git a/processing/src/main/java/io/druid/segment/Rowboat.java b/processing/src/main/java/io/druid/segment/Rowboat.java index bfe6049250ab..31c859263d03 100644 --- a/processing/src/main/java/io/druid/segment/Rowboat.java +++ b/processing/src/main/java/io/druid/segment/Rowboat.java @@ -19,35 +19,41 @@ package io.druid.segment; +import com.google.common.base.Objects; import com.google.common.collect.Maps; import com.google.common.collect.Sets; import com.google.common.primitives.Ints; import com.google.common.primitives.Longs; +import org.apache.commons.lang.ObjectUtils; import org.joda.time.DateTime; import java.util.Arrays; +import java.util.Comparator; import java.util.Map; import java.util.TreeSet; public class Rowboat implements Comparable { private final long timestamp; - private final int[][] dims; + private final Comparable[][] dims; private final Object[] metrics; private final int rowNum; private final Map> comprisedRows; + private final DimensionHandler[] handlers; public Rowboat( long timestamp, - int[][] dims, + Comparable[][] dims, Object[] metrics, - int rowNum + int rowNum, + DimensionHandler[] handlers ) { this.timestamp = timestamp; this.dims = dims; this.metrics = metrics; this.rowNum = rowNum; + this.handlers = handlers; this.comprisedRows = Maps.newHashMap(); } @@ -57,7 +63,7 @@ public long getTimestamp() return timestamp; } - public int[][] getDims() + public Comparable[][] getDims() { return dims; } @@ -82,6 +88,11 @@ public Map> getComprisedRows() return comprisedRows; } + public DimensionHandler[] getHandlers() + { + return handlers; + } + public int getRowNum() { return rowNum; @@ -98,8 +109,8 @@ public int compareTo(Rowboat rhs) int index = 0; while (retVal == 0 && index < dims.length) { - int[] lhsVals = dims[index]; - int[] rhsVals = rhs.dims[index]; + Comparable[] lhsVals = dims[index]; + Comparable[] rhsVals = rhs.dims[index]; if (lhsVals == null) { if (rhsVals == null) { @@ -116,8 +127,9 @@ public int compareTo(Rowboat rhs) retVal = Ints.compare(lhsVals.length, rhsVals.length); int valsIndex = 0; + Comparator comparator = handlers[index].getEncodedComparator(); while (retVal == 0 && valsIndex < lhsVals.length) { - retVal = Ints.compare(lhsVals[valsIndex], rhsVals[valsIndex]); + retVal = comparator.compare(lhsVals[valsIndex], rhsVals[valsIndex]); ++valsIndex; } ++index; diff --git a/processing/src/main/java/io/druid/segment/SimpleQueryableIndex.java b/processing/src/main/java/io/druid/segment/SimpleQueryableIndex.java index 5d6aa0452c34..2949c8e444f4 100644 --- a/processing/src/main/java/io/druid/segment/SimpleQueryableIndex.java +++ b/processing/src/main/java/io/druid/segment/SimpleQueryableIndex.java @@ -20,9 +20,12 @@ package io.druid.segment; import com.google.common.base.Preconditions; +import com.google.common.collect.Maps; import com.metamx.collections.bitmap.BitmapFactory; import com.metamx.common.io.smoosh.SmooshedFileMapper; import io.druid.segment.column.Column; +import io.druid.segment.column.ColumnCapabilities; +import io.druid.segment.column.ValueType; import io.druid.segment.data.Indexed; import org.joda.time.Interval; @@ -40,6 +43,8 @@ public class SimpleQueryableIndex implements QueryableIndex private final Map columns; private final SmooshedFileMapper fileMapper; private final Metadata metadata; + private final Map dimensionHandlers; + private final Map dimensionReaders; public SimpleQueryableIndex( Interval dataInterval, @@ -59,6 +64,10 @@ public SimpleQueryableIndex( this.columns = columns; this.fileMapper = fileMapper; this.metadata = metadata; + + dimensionHandlers = Maps.newLinkedHashMap(); + dimensionReaders = Maps.newLinkedHashMap(); + initDimensionHandlers(); } @Override @@ -108,4 +117,28 @@ public Metadata getMetadata() { return metadata; } + + @Override + public Map getDimensionHandlers() + { + return dimensionHandlers; + } + + + public Map getDimensionReaders() + { + return dimensionReaders; + } + + + private void initDimensionHandlers() + { + for (String dim : availableDimensions) { + ColumnCapabilities capabilities = getColumn(dim).getCapabilities(); + DimensionHandler handler = DimensionHandlerUtil.getHandlerFromCapabilities(dim, capabilities); + DimensionColumnReader reader = handler.makeColumnReader(getColumn(dim)); + dimensionHandlers.put(dim, handler); + dimensionReaders.put(dim, reader); + } + } } diff --git a/processing/src/main/java/io/druid/segment/SingleScanTimeDimSelector.java b/processing/src/main/java/io/druid/segment/SingleScanTimeDimSelector.java index 2b5f085e2cbe..61e850d124a2 100644 --- a/processing/src/main/java/io/druid/segment/SingleScanTimeDimSelector.java +++ b/processing/src/main/java/io/druid/segment/SingleScanTimeDimSelector.java @@ -22,10 +22,14 @@ import com.google.common.collect.Iterators; import com.google.common.collect.Maps; import io.druid.query.extraction.ExtractionFn; +import io.druid.segment.column.ColumnCapabilities; +import io.druid.segment.data.IndexedFloats; import io.druid.segment.data.IndexedInts; +import io.druid.segment.data.IndexedLongs; import java.io.IOException; import java.util.Iterator; +import java.util.List; import java.util.Map; public class SingleScanTimeDimSelector implements DimensionSelector @@ -148,4 +152,47 @@ public int lookupId(String name) { throw new UnsupportedOperationException("time column does not support lookups"); } + + @Override + public IndexedLongs getLongRow() + { + throw new UnsupportedOperationException("getLongRow() is not supported."); + } + + @Override + public Comparable getExtractedValueLong(long val) + { + throw new UnsupportedOperationException("getExtractedValueLong() is not supported."); + } + + @Override + public IndexedFloats getFloatRow() + { + throw new UnsupportedOperationException("getFloatRow() is not supported."); + } + + @Override + public Comparable getExtractedValueFloat(float val) + { + throw new UnsupportedOperationException("getExtractedValueFloat() is not supported."); + } + + @Override + public Comparable getComparableRow() + { + throw new UnsupportedOperationException("getComparableRow() is not supported."); + } + + @Override + public Comparable getExtractedValueComparable(Comparable val) + { + throw new UnsupportedOperationException("getExtractedValueComparable() is not supported."); + + } + + @Override + public ColumnCapabilities getDimCapabilities() + { + throw new UnsupportedOperationException("getDimCapabilities not supprted yet"); + } } diff --git a/processing/src/main/java/io/druid/segment/StorageAdapter.java b/processing/src/main/java/io/druid/segment/StorageAdapter.java index 418a45e4ac10..603c8dde7413 100644 --- a/processing/src/main/java/io/druid/segment/StorageAdapter.java +++ b/processing/src/main/java/io/druid/segment/StorageAdapter.java @@ -24,6 +24,8 @@ import org.joda.time.DateTime; import org.joda.time.Interval; +import java.util.Map; + /** */ public interface StorageAdapter extends CursorFactory @@ -48,6 +50,7 @@ public interface StorageAdapter extends CursorFactory public Comparable getMaxValue(String column); public Capabilities getCapabilities(); public ColumnCapabilities getColumnCapabilities(String column); + public Map getDimensionHandlers(); /** * Like {@link ColumnCapabilities#getType()}, but may return a more descriptive string for complex columns. diff --git a/processing/src/main/java/io/druid/segment/StringDimensionHandler.java b/processing/src/main/java/io/druid/segment/StringDimensionHandler.java new file mode 100644 index 000000000000..a88e222a08c2 --- /dev/null +++ b/processing/src/main/java/io/druid/segment/StringDimensionHandler.java @@ -0,0 +1,1236 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.segment; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.base.Function; +import com.google.common.base.Predicate; +import com.google.common.base.Splitter; +import com.google.common.base.Strings; +import com.google.common.collect.Iterables; +import com.google.common.collect.Iterators; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import com.google.common.collect.Ordering; +import com.google.common.io.ByteStreams; +import com.google.common.io.Files; +import com.google.common.io.OutputSupplier; +import com.google.common.primitives.Ints; +import com.metamx.collections.bitmap.BitmapFactory; +import com.metamx.collections.bitmap.ImmutableBitmap; +import com.metamx.collections.bitmap.MutableBitmap; +import com.metamx.collections.spatial.ImmutableRTree; +import com.metamx.collections.spatial.RTree; +import com.metamx.collections.spatial.search.Bound; +import com.metamx.collections.spatial.split.LinearGutmanSplitStrategy; +import com.metamx.common.ISE; +import com.metamx.common.guava.FunctionalIterable; +import com.metamx.common.io.smoosh.FileSmoosher; +import com.metamx.common.io.smoosh.SmooshedWriter; +import com.metamx.common.logger.Logger; +import com.metamx.common.parsers.ParseException; +import io.druid.collections.CombiningIterable; +import io.druid.common.guava.FileOutputSupplier; +import io.druid.common.utils.SerializerUtils; +import io.druid.query.dimension.DimensionSpec; +import io.druid.segment.column.BitmapIndex; +import io.druid.segment.column.Column; +import io.druid.segment.column.ColumnCapabilities; +import io.druid.segment.column.ColumnDescriptor; +import io.druid.segment.column.DictionaryEncodedColumn; +import io.druid.segment.column.ValueType; +import io.druid.segment.data.BitmapSerdeFactory; +import io.druid.segment.data.ByteBufferWriter; +import io.druid.segment.data.CompressedObjectStrategy; +import io.druid.segment.data.CompressedVSizeIndexedV3Writer; +import io.druid.segment.data.CompressedVSizeIntsIndexedWriter; +import io.druid.segment.data.GenericIndexed; +import io.druid.segment.data.GenericIndexedWriter; +import io.druid.segment.data.IOPeon; +import io.druid.segment.data.Indexed; +import io.druid.segment.data.IndexedInts; +import io.druid.segment.data.IndexedIntsWriter; +import io.druid.segment.data.IndexedIterable; +import io.druid.segment.data.IndexedRTree; +import io.druid.segment.data.ListIndexed; +import io.druid.segment.data.TmpFileIOPeon; +import io.druid.segment.data.VSizeIndexedIntsWriter; +import io.druid.segment.data.VSizeIndexedWriter; +import io.druid.segment.serde.DictionaryEncodedColumnPartSerde; +import org.apache.commons.lang.ObjectUtils; + +import javax.annotation.Nullable; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.IntBuffer; +import java.nio.MappedByteBuffer; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Comparator; +import java.util.Iterator; +import java.util.List; +import java.util.Map; + +public class StringDimensionHandler implements DimensionHandler +{ + private static final Logger log = new Logger(StringDimensionHandler.class); + + private static final ListIndexed EMPTY_STR_DIM_VAL = new ListIndexed<>(Arrays.asList(""), String.class); + protected static final Splitter SPLITTER = Splitter.on(","); + + private final String dimensionName; + + public StringDimensionHandler(String dimensionName) + { + this.dimensionName = dimensionName; + } + + @Override + public DimensionSelector getDimensionSelector(ColumnSelectorFactory cursor, DimensionSpec dimSpec, ColumnCapabilities capabilities) + { + return cursor.makeDictEncodedStringDimensionSelector(dimSpec); + } + + @Override + public Function getValueTypeTransformer() + { + return STRING_TRANSFORMER; + } + + @Override + public Comparator getEncodedComparator() + { + return ENCODED_COMPARATOR; + } + + @Override + public Comparator getActualComparator() + { + return UNENCODED_COMPARATOR; + } + + @Override + public byte[] getBytesFromRowValue(Integer rowVal) + { + return Ints.toByteArray(rowVal); + } + + @Override + public Integer getRowValueFromBytes(byte[] bytes) + { + return Ints.fromByteArray(bytes); + } + + @Override + public int getEncodedValueSize() + { + return Ints.BYTES; + } + + @Override + public DimensionIndexer makeIndexer(Object lock) + { + return new StringDimensionIndexer(lock); + } + + @Override + public DimensionMerger makeMerger( + IndexSpec indexSpec, + File outDir, + IOPeon ioPeon, + FileSmoosher v9Smoosher, + ObjectMapper mapper, + ColumnCapabilities capabilities, + ProgressIndicator progress + ) + { + return new StringDimensionMerger(indexSpec, outDir, ioPeon, v9Smoosher, mapper, capabilities, progress); + } + + @Override + public DimensionColumnReader makeColumnReader(Column column) + { + return new StringDimensionColumnReader(column); + } + + public static final Function STRING_TRANSFORMER = new Function() + { + @Override + public String apply(final Object o) + { + if (o == null) { + return null; + } + if (o instanceof String) { + return (String) o; + } + return String.valueOf(o); + } + }; + + private static final Comparator ENCODED_COMPARATOR = new Comparator() + { + @Override + public int compare(Integer o1, Integer o2) + { + if (o1 == null) { + return o2 == null ? 0 : -1; + } + if (o2 == null) { + return 1; + } + return Integer.compare(o1, o2); + } + }; + + private static final Comparator UNENCODED_COMPARATOR = new Comparator() + { + @Override + public int compare(String o1, String o2) + { + if (o1 == null) { + return o2 == null ? 0 : -1; + } + if (o2 == null) { + return 1; + } + return ObjectUtils.compare(o1, o2); + } + }; + + private class DimensionDictionary + { + private String minValue = null; + private String maxValue = null; + + private final Map valueToId = Maps.newHashMap(); + + private final List idToValue = Lists.newArrayList(); + private final Object lock; + + public DimensionDictionary(Object lock) + { + this.lock = lock; + } + + public int getId(String value) + { + synchronized (lock) { + final Integer id = valueToId.get(Strings.nullToEmpty(value)); + return id == null ? -1 : id; + } + } + + public String getValue(int id) + { + synchronized (lock) { + return Strings.emptyToNull(idToValue.get(id)); + } + } + + public boolean contains(String value) + { + synchronized (lock) { + return valueToId.containsKey(value); + } + } + + public int size() + { + synchronized (lock) { + return valueToId.size(); + } + } + + public int add(String originalValue) + { + String value = Strings.nullToEmpty(originalValue); + synchronized (lock) { + Integer prev = valueToId.get(value); + if (prev != null) { + return prev; + } + final int index = size(); + valueToId.put(value, index); + idToValue.add(value); + minValue = minValue == null || minValue.compareTo(value) > 0 ? value : minValue; + maxValue = maxValue == null || maxValue.compareTo(value) < 0 ? value : maxValue; + return index; + } + } + + public String getMinValue() + { + return minValue; + } + + public String getMaxValue() + { + return maxValue; + } + + public SortedDimensionDictionary sort() + { + synchronized (lock) { + return new SortedDimensionDictionary(idToValue, size()); + } + } + } + + private class SortedDimensionDictionary + { + private final List sortedVals; + private final int[] idToIndex; + private final int[] indexToId; + + public SortedDimensionDictionary(List idToValue, int length) + { + Map sortedMap = Maps.newTreeMap(); + for (int id = 0; id < length; id++) { + sortedMap.put(idToValue.get(id), id); + } + this.sortedVals = Lists.newArrayList(sortedMap.keySet()); + this.idToIndex = new int[length]; + this.indexToId = new int[length]; + int index = 0; + for (Integer id : sortedMap.values()) { + idToIndex[id] = index; + indexToId[index] = id; + index++; + } + } + + public int size() + { + return sortedVals.size(); + } + + public int getUnsortedIdFromSortedId(int index) + { + return indexToId[index]; + } + + public int getSortedIdFromUnsortedId(int id) + { + return idToIndex[id]; + } + + public String getValueFromSortedId(int index) + { + return Strings.emptyToNull(sortedVals.get(index)); + } + } + + public class StringDimensionIndexer implements DimensionIndexer + { + @Override + public void addNullLookup() + { + dimLookup.add(null); + sortedLookup = dimLookup.sort(); + } + + private DimensionDictionary dimLookup; + private SortedDimensionDictionary sortedLookup; + private Object lock; + + public StringDimensionIndexer(Object lock) + { + this.lock = lock; + this.dimLookup = new DimensionDictionary(this.lock); + } + + @Override + public Comparable[] processRowValsToIndexKey(Object dimValues) + { + final Comparable[] dimensionValues; + String transformedVal; + int oldDictSize = dimLookup.size(); + try { + if (dimValues == null) { + dimLookup.add(null); + dimensionValues = null; + } else if (dimValues instanceof List) { + List dimValuesList = (List) dimValues; + dimensionValues = new Comparable[dimValuesList.size()]; + for (int i = 0; i < dimValuesList.size(); i++) { + transformedVal = STRING_TRANSFORMER.apply(dimValuesList.get(i)); + dimensionValues[i] = dimLookup.add(transformedVal); + } + } else { + transformedVal = STRING_TRANSFORMER.apply(dimValues); + dimensionValues = new Comparable[]{dimLookup.add(transformedVal)}; + } + } + catch (ParseException pe) { + throw new ParseException(pe.getMessage() + dimensionName); + } + + // If dictionary size has changed, the sorted lookup is no longer valid. + if (oldDictSize != dimLookup.size()) { + sortedLookup = null; + } + + return dimensionValues; + } + + @Override + public String getActualValue(Integer intermediateValue, boolean idSorted) + { + if (idSorted) { + if (sortedLookup == null) { + sortedLookup = dimLookup.sort(); + } + return sortedLookup.getValueFromSortedId(intermediateValue); + } else { + return dimLookup.getValue(intermediateValue); + + } + } + + @Override + public Integer getEncodedValue(String fullValue, boolean idSorted) + { + int unsortedId = dimLookup.getId(fullValue); + + if (idSorted) { + if (sortedLookup == null) { + sortedLookup = dimLookup.sort(); + } + return sortedLookup.getSortedIdFromUnsortedId(unsortedId); + } else { + return unsortedId; + } + } + + @Override + public Integer getSortedEncodedValueFromUnsorted(Integer unsortedIntermediateValue) + { + if (sortedLookup == null) { + sortedLookup = dimLookup.sort(); + } + return sortedLookup.getSortedIdFromUnsortedId(unsortedIntermediateValue); + } + + @Override + public Integer getUnsortedEncodedValueFromSorted(Integer sortedIntermediateValue) + { + if (sortedLookup == null) { + sortedLookup = dimLookup.sort(); + } + return sortedLookup.getUnsortedIdFromSortedId(sortedIntermediateValue); + } + + @Override + public Indexed getSortedIndexedValues() + { + if(sortedLookup == null) { + sortedLookup = dimLookup.sort(); + } + + return new Indexed() + { + @Override + public Class getClazz() + { + return String.class; + } + + @Override + public int size() + { + return getCardinality(); + } + + @Override + public String get(int index) + { + Comparable val = getActualValue(index, true); + String strVal = val != null ? val.toString() : null; + return strVal; + } + + @Override + public int indexOf(String value) + { + int id = (Integer) getEncodedValue(value, false); + return id < 0 ? -1 : (Integer) getSortedEncodedValueFromUnsorted(id); + } + + @Override + public Iterator iterator() + { + return IndexedIterable.create(this).iterator(); + } + }; + } + + @Override + public String getMinValue() + { + return dimLookup.getMinValue(); + } + + @Override + public String getMaxValue() + { + return dimLookup.getMaxValue(); + } + + @Override + public int getCardinality() + { + return dimLookup.size(); + } + } + + + public class StringDimensionMerger implements DimensionMerger + { + private GenericIndexedWriter dictionaryWriter; + private GenericIndexedWriter bitmapWriter; + private IndexedIntsWriter encodedValueWriter; + private ByteBufferWriter spatialWriter; + private ArrayList dimConversions; + private int cardinality = 0; + private boolean convertMissingValues = false; + private boolean hasNull = false; + private MutableBitmap nullRowsBitmap; + private IOPeon ioPeon; + private int rowCount = 0; + private ColumnCapabilities capabilities; + private final File outDir; + private final FileSmoosher v9Smoosher; + private List adapters; + private ObjectMapper mapper; + private ProgressIndicator progress; + + // Remove these once V8 IndexMerger is removed + private boolean useV8 = false; + private VSizeIndexedWriter encodedValueWriterV8; + + private final IndexSpec indexSpec; + + public StringDimensionMerger( + IndexSpec indexSpec, + File outDir, + IOPeon ioPeon, + FileSmoosher v9Smoosher, + ObjectMapper mapper, + ColumnCapabilities capabilities, + ProgressIndicator progress + ) + { + this.indexSpec = indexSpec; + this.capabilities = capabilities; + this.outDir = outDir; + this.ioPeon = ioPeon; + this.v9Smoosher = v9Smoosher; + this.mapper = mapper; + this.progress = progress; + nullRowsBitmap = indexSpec.getBitmapSerdeFactory().getBitmapFactory().makeEmptyMutableBitmap(); + } + + @Override + public void initWriters() throws IOException + { + String dictFilename = String.format("%s.dim_values", dimensionName); + dictionaryWriter = new GenericIndexedWriter<>( + ioPeon, + dictFilename, + GenericIndexed.STRING_STRATEGY + ); + dictionaryWriter.open(); + + String bmpFilename = String.format("%s.inverted", dimensionName); + bitmapWriter = new GenericIndexedWriter<>( + ioPeon, + bmpFilename, + indexSpec.getBitmapSerdeFactory().getObjectStrategy() + ); + bitmapWriter.open(); + } + + @Override + public void mergeAcrossSegments(List adapters) throws IOException + { + boolean dimHasValues = false; + boolean dimAbsentFromSomeIndex = false; + + long dimStartTime = System.currentTimeMillis(); + + this.adapters = adapters; + + dimConversions = Lists.newArrayListWithCapacity(adapters.size()); + for (int i = 0; i < adapters.size(); ++i) { + dimConversions.add(null); + } + + int numMergeIndex = 0; + Indexed dimValueLookup = null; + Indexed[] dimValueLookups = new Indexed[adapters.size() + 1]; + for (int i = 0; i < adapters.size(); i++) { + Indexed dimValues = adapters.get(i).getDimValueLookup(dimensionName); + if (!isNullColumn(dimValues)) { + dimHasValues = true; + hasNull |= dimValues.indexOf(null) >= 0; + dimValueLookups[i] = dimValueLookup = dimValues; + numMergeIndex++; + } else { + dimAbsentFromSomeIndex = true; + } + } + + convertMissingValues = dimHasValues && dimAbsentFromSomeIndex; + + /* + * Ensure the empty str is always in the dictionary if the dimension was missing from one index but + * has non-null values in another index. + * This is done so that MMappedIndexRowIterable can convert null columns to empty strings + * later on, to allow rows from indexes without a particular dimension to merge correctly with + * rows from indexes with null/empty str values for that dimension. + */ + if (convertMissingValues && !hasNull) { + hasNull = true; + dimValueLookups[adapters.size()] = dimValueLookup = EMPTY_STR_DIM_VAL; + numMergeIndex++; + } + + cardinality = 0; + if (numMergeIndex > 1) { + IndexMerger.DictionaryMergeIterator iterator = new IndexMerger.DictionaryMergeIterator(dimValueLookups, true); + + while (iterator.hasNext()) { + dictionaryWriter.write(iterator.next()); + } + + for (int i = 0; i < adapters.size(); i++) { + if (dimValueLookups[i] != null && iterator.needConversion(i)) { + dimConversions.set(i, iterator.conversions[i]); + } + } + cardinality = iterator.counter; + } else if (numMergeIndex == 1) { + for (String value : dimValueLookup) { + dictionaryWriter.write(value); + } + cardinality = dimValueLookup.size(); + } + + log.info( + "Completed dim[%s] conversions with cardinality[%,d] in %,d millis.", + dimensionName, + cardinality, + System.currentTimeMillis() - dimStartTime + ); + dictionaryWriter.close(); + + if (useV8) { + encodedValueWriterV8 = new VSizeIndexedWriter(ioPeon, dimensionName, cardinality); + encodedValueWriterV8.open(); + return; + } + + if (cardinality == 0) { + log.info(String.format("Skipping [%s], it is empty!", dimensionName)); + } + + final CompressedObjectStrategy.CompressionStrategy compressionStrategy = indexSpec.getDimensionCompressionStrategy(); + + String filenameBase = String.format("%s.forward_dim", dimensionName); + if (capabilities.hasMultipleValues()) { + encodedValueWriter = (compressionStrategy != null) + ? CompressedVSizeIndexedV3Writer.create(ioPeon, filenameBase, cardinality, compressionStrategy) + : new VSizeIndexedWriter(ioPeon, filenameBase, cardinality); + } else { + encodedValueWriter = (compressionStrategy != null) + ? CompressedVSizeIntsIndexedWriter.create(ioPeon, filenameBase, cardinality, compressionStrategy) + : new VSizeIndexedIntsWriter(ioPeon, filenameBase, cardinality); + } + encodedValueWriter.open(); + } + + @Override + public void addColumnValue(Comparable[] vals) throws IOException + { + if (vals == null || vals.length == 0) { + nullRowsBitmap.add(rowCount); + } else if (hasNull && vals.length == 1 && ((Integer) vals[0]) == 0) { + // Dictionary encoded, so it's safe to cast dim value to integer + // If this dimension has the null/empty str in its dictionary, a row with a single-valued dimension + // that matches the null/empty str's dictionary ID should also be added to nullRowBitmap. + nullRowsBitmap.add(rowCount); + } + if (useV8) { + List listToWrite = (vals == null) + ? null + : Lists.newArrayList(vals); + encodedValueWriterV8.add(listToWrite); + } else { + encodedValueWriter.add(vals); + } + + rowCount++; + } + + @Override + public void closeWritersV8(List rowNumConversions, OutputSupplier invertedOut, OutputSupplier spatialOut) throws IOException + { + final SerializerUtils serializerUtils = new SerializerUtils(); + long dimStartTime = System.currentTimeMillis(); + + dictionaryWriter.close(); + + FileOutputSupplier dimOut = new FileOutputSupplier(IndexIO.makeDimFile(outDir, dimensionName), true); + serializerUtils.writeString(dimOut, dimensionName); + ByteStreams.copy(dictionaryWriter.combineStreams(), dimOut); + File dimOutFile = dimOut.getFile(); + + final MappedByteBuffer dimValsMapped = Files.map(dimOutFile); + + if (!dimensionName.equals(serializerUtils.readString(dimValsMapped))) { + throw new ISE("dimensions[%s] didn't equate!? This is a major WTF moment.", dimensionName); + } + Indexed dimVals = GenericIndexed.read(dimValsMapped, GenericIndexed.STRING_STRATEGY); + log.info("Starting dimension[%s] with cardinality[%,d]", dimensionName, dimVals.size()); + + final BitmapSerdeFactory bitmapSerdeFactory = indexSpec.getBitmapSerdeFactory(); + final BitmapFactory bitmapFactory = bitmapSerdeFactory.getBitmapFactory(); + + RTree tree = null; + ByteBufferWriter spatialWriter = null; + boolean hasSpatial = capabilities.hasSpatialIndexes(); + IOPeon spatialIoPeon = new TmpFileIOPeon(); + if (hasSpatial) { + BitmapFactory bmpFactory = bitmapSerdeFactory.getBitmapFactory(); + String spatialFilename = String.format("%s.spatial", dimensionName); + spatialWriter = new ByteBufferWriter( + spatialIoPeon, spatialFilename, new IndexedRTree.ImmutableRTreeObjectStrategy(bmpFactory) + ); + spatialWriter.open(); + tree = new RTree(2, new LinearGutmanSplitStrategy(0, 50, bitmapFactory), bitmapFactory); + } + + IndexSeeker[] dictIdSeeker = toIndexSeekers(adapters, dimConversions, dimensionName); + + //Iterate all dim values's dictionary id in ascending order which in line with dim values's compare result. + for (int dictId = 0; dictId < dimVals.size(); dictId++) { + progress.progress(); + List> convertedInverteds = Lists.newArrayListWithCapacity(adapters.size()); + for (int j = 0; j < adapters.size(); ++j) { + int seekedDictId = dictIdSeeker[j].seek(dictId); + if (seekedDictId != IndexSeeker.NOT_EXIST) { + convertedInverteds.add( + new ConvertingIndexedInts( + adapters.get(j).getBitmapIndex(dimensionName, seekedDictId), rowNumConversions.get(j) + ) + ); + } + } + + MutableBitmap bitset = bitmapSerdeFactory.getBitmapFactory().makeEmptyMutableBitmap(); + for (Integer row : CombiningIterable.createSplatted( + convertedInverteds, + Ordering.natural().nullsFirst() + )) { + if (row != IndexMerger.INVALID_ROW) { + bitset.add(row); + } + } + + bitmapWriter.write( + bitmapSerdeFactory.getBitmapFactory().makeImmutableBitmap(bitset) + ); + + if (hasSpatial) { + String dimVal = dimVals.get(dictId); + if (dimVal != null) { + List stringCoords = Lists.newArrayList(SPLITTER.split(dimVal)); + float[] coords = new float[stringCoords.size()]; + for (int j = 0; j < coords.length; j++) { + coords[j] = Float.valueOf(stringCoords.get(j)); + } + tree.insert(coords, bitset); + } + } + + } + bitmapWriter.close(); + serializerUtils.writeString(invertedOut, dimensionName); + ByteStreams.copy(bitmapWriter.combineStreams(), invertedOut); + + + log.info("Completed dimension[%s] in %,d millis.", dimensionName, System.currentTimeMillis() - dimStartTime); + + if (hasSpatial) { + spatialWriter.write(ImmutableRTree.newImmutableFromMutable(tree)); + spatialWriter.close(); + + serializerUtils.writeString(spatialOut, dimensionName); + ByteStreams.copy(spatialWriter.combineStreams(), spatialOut); + spatialIoPeon.cleanup(); + } + + + encodedValueWriterV8.close(); + ByteStreams.copy(encodedValueWriterV8.combineStreams(), dimOut); + + + return; + } + + @Override + public void closeWriters(List rowNumConversions) throws IOException + { + long dimStartTime = System.currentTimeMillis(); + final BitmapSerdeFactory bitmapSerdeFactory = indexSpec.getBitmapSerdeFactory(); + + // write dim values to one single file because we need to read it + File dimValueFile = IndexIO.makeDimFile(outDir, dimensionName); + FileOutputStream fos = new FileOutputStream(dimValueFile); + ByteStreams.copy(dictionaryWriter.combineStreams(), fos); + fos.close(); + + final MappedByteBuffer dimValsMapped = Files.map(dimValueFile); + Indexed dimVals = GenericIndexed.read(dimValsMapped, GenericIndexed.STRING_STRATEGY); + BitmapFactory bmpFactory = bitmapSerdeFactory.getBitmapFactory(); + + RTree tree = null; + boolean hasSpatial = capabilities.hasSpatialIndexes(); + if(hasSpatial) { + BitmapFactory bitmapFactory = indexSpec.getBitmapSerdeFactory().getBitmapFactory(); + spatialWriter = new ByteBufferWriter<>( + ioPeon, + String.format("%s.spatial", dimensionName), + new IndexedRTree.ImmutableRTreeObjectStrategy(bitmapFactory) + ); + spatialWriter.open(); + tree = new RTree(2, new LinearGutmanSplitStrategy(0, 50, bmpFactory), bmpFactory); + } + + IndexSeeker[] dictIdSeeker = toIndexSeekers(adapters, dimConversions, dimensionName); + + //Iterate all dim values's dictionary id in ascending order which in line with dim values's compare result. + for (int dictId = 0; dictId < dimVals.size(); dictId++) { + progress.progress(); + List> convertedInverteds = Lists.newArrayListWithCapacity(adapters.size()); + for (int j = 0; j < adapters.size(); ++j) { + int seekedDictId = dictIdSeeker[j].seek(dictId); + if (seekedDictId != IndexSeeker.NOT_EXIST) { + convertedInverteds.add( + new ConvertingIndexedInts( + adapters.get(j).getBitmapIndex(dimensionName, seekedDictId), rowNumConversions.get(j) + ) + ); + } + } + + MutableBitmap bitset = bmpFactory.makeEmptyMutableBitmap(); + for (Integer row : CombiningIterable.createSplatted( + convertedInverteds, + Ordering.natural().nullsFirst() + )) { + if (row != IndexMerger.INVALID_ROW) { + bitset.add(row); + } + } + + ImmutableBitmap bitmapToWrite = bitmapSerdeFactory.getBitmapFactory().makeImmutableBitmap(bitset); + if ((dictId == 0) && (Iterables.getFirst(dimVals, "") == null)) { + bitmapToWrite = bmpFactory.makeImmutableBitmap(nullRowsBitmap).union(bitmapToWrite); + } + bitmapWriter.write(bitmapToWrite); + + if (hasSpatial) { + String dimVal = dimVals.get(dictId); + if (dimVal != null) { + List stringCoords = Lists.newArrayList(SPLITTER.split(dimVal)); + float[] coords = new float[stringCoords.size()]; + for (int j = 0; j < coords.length; j++) { + coords[j] = Float.valueOf(stringCoords.get(j)); + } + tree.insert(coords, bitset); + } + } + } + + if (hasSpatial) { + spatialWriter.write(ImmutableRTree.newImmutableFromMutable(tree)); + spatialWriter.close(); + } + + log.info( + "Completed dim[%s] inverted with cardinality[%,d] in %,d millis.", + dimensionName, + dimVals.size(), + System.currentTimeMillis() - dimStartTime + ); + + bitmapWriter.close(); + encodedValueWriter.close(); + + // Now write everything + boolean hasMultiValue = capabilities.hasMultipleValues(); + final CompressedObjectStrategy.CompressionStrategy compressionStrategy = indexSpec.getDimensionCompressionStrategy(); + + final ColumnDescriptor.Builder builder = ColumnDescriptor.builder(); + builder.setValueType(ValueType.STRING); + builder.setHasMultipleValues(hasMultiValue); + final DictionaryEncodedColumnPartSerde.SerializerBuilder partBuilder = DictionaryEncodedColumnPartSerde + .serializerBuilder() + .withDictionary(dictionaryWriter) + .withValue(encodedValueWriter, hasMultiValue, compressionStrategy != null) + .withBitmapSerdeFactory(bitmapSerdeFactory) + .withBitmapIndex(bitmapWriter) + .withSpatialIndex(spatialWriter) + .withByteOrder(IndexIO.BYTE_ORDER); + final ColumnDescriptor serdeficator = builder + .addSerde(partBuilder.build()) + .build(); + makeColumn(v9Smoosher, dimensionName, serdeficator); + log.info("Completed dimension column[%s] in %,d millis.", dimensionName, System.currentTimeMillis() - dimStartTime); + } + + @Override + public boolean needsConvertMissingValues() + { + return convertMissingValues; + } + + @Override + public boolean canSkip() + { + return cardinality == 0; + } + + @Override + public Integer getConvertedEncodedValue(Integer value, int indexNumber) + { + IntBuffer converter = dimConversions.get(indexNumber); + if (converter != null) { + Integer convertedValue = converter.get(value); + return convertedValue; + } + return converter == null ? value : converter.get(value); + } + + @Override + public void setWriteV8() + { + useV8 = true; + } + + private boolean isNullColumn(Iterable dimValues) + { + if (dimValues == null) { + return true; + } + for (String val : dimValues) { + if (val != null) { + return false; + } + } + return true; + } + + private void makeColumn( + final FileSmoosher v9Smoosher, + final String columnName, + final ColumnDescriptor serdeficator + ) throws IOException + { + final SerializerUtils serializerUtils = new SerializerUtils(); + + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + serializerUtils.writeString(baos, mapper.writeValueAsString(serdeficator)); + byte[] specBytes = baos.toByteArray(); + + final SmooshedWriter channel = v9Smoosher.addWithSmooshedWriter( + columnName, serdeficator.numBytes() + specBytes.length + ); + try { + channel.write(ByteBuffer.wrap(specBytes)); + serdeficator.write(channel); + } + finally { + channel.close(); + } + } + + private IndexSeeker[] toIndexSeekers( + List adapters, + ArrayList dimConversions, + String dimension + ) + { + IndexSeeker[] seekers = new IndexSeeker[adapters.size()]; + for (int i = 0; i < adapters.size(); i++) { + IntBuffer dimConversion = dimConversions.get(i); + if (dimConversion != null) { + seekers[i] = new IndexSeekerWithConversion((IntBuffer) dimConversion.asReadOnlyBuffer().rewind()); + } else { + Indexed dimValueLookup = adapters.get(i).getDimValueLookup(dimension); + seekers[i] = new IndexSeekerWithoutConversion(dimValueLookup == null ? 0 : dimValueLookup.size()); + } + } + return seekers; + } + } + + private interface IndexSeeker + { + int NOT_EXIST = -1; + int NOT_INIT = -1; + + int seek(int dictId); + } + + private class IndexSeekerWithoutConversion implements IndexSeeker + { + private final int limit; + + public IndexSeekerWithoutConversion(int limit) + { + this.limit = limit; + } + + @Override + public int seek(int dictId) + { + return dictId < limit ? dictId : NOT_EXIST; + } + } + + /** + * Get old dictId from new dictId, and only support access in order + */ + private class IndexSeekerWithConversion implements IndexSeeker + { + private final IntBuffer dimConversions; + private int currIndex; + private int currVal; + private int lastVal; + + IndexSeekerWithConversion(IntBuffer dimConversions) + { + this.dimConversions = dimConversions; + this.currIndex = 0; + this.currVal = NOT_INIT; + this.lastVal = NOT_INIT; + } + + public int seek(int dictId) + { + if (dimConversions == null) { + return NOT_EXIST; + } + if (lastVal != NOT_INIT) { + if (dictId <= lastVal) { + throw new ISE( + "Value dictId[%d] is less than the last value dictId[%d] I have, cannot be.", + dictId, lastVal + ); + } + return NOT_EXIST; + } + if (currVal == NOT_INIT) { + currVal = dimConversions.get(); + } + if (currVal == dictId) { + int ret = currIndex; + ++currIndex; + if (dimConversions.hasRemaining()) { + currVal = dimConversions.get(); + } else { + lastVal = dictId; + } + return ret; + } else if (currVal < dictId) { + throw new ISE( + "Skipped currValue dictId[%d], currIndex[%d]; incoming value dictId[%d]", + currVal, currIndex, dictId + ); + } else { + return NOT_EXIST; + } + } + } + + public static class ConvertingIndexedInts implements Iterable + { + private final IndexedInts baseIndex; + private final IntBuffer conversionBuffer; + + public ConvertingIndexedInts( + IndexedInts baseIndex, + IntBuffer conversionBuffer + ) + { + this.baseIndex = baseIndex; + this.conversionBuffer = conversionBuffer; + } + + public int size() + { + return baseIndex.size(); + } + + public int get(int index) + { + return conversionBuffer.get(baseIndex.get(index)); + } + + @Override + public Iterator iterator() + { + return Iterators.transform( + baseIndex.iterator(), + new Function() + { + @Override + public Integer apply(@Nullable Integer input) + { + return conversionBuffer.get(input); + } + } + ); + } + } + + public class StringDimensionColumnReader implements DimensionColumnReader + { + private final Column column; + private final DictionaryEncodedColumn dictionary; + public StringDimensionColumnReader(Column column) + { + this.column = column; + this.dictionary = column.getDictionaryEncoding(); + } + + @Override + public ImmutableBitmap getBitmapIndex(String value, BitmapFactory bitmapFactory, int numRows) + { + if (column == null) { + if (Strings.isNullOrEmpty(value)) { + return bitmapFactory.complement(bitmapFactory.makeEmptyImmutableBitmap(), numRows); + } else { + return bitmapFactory.makeEmptyImmutableBitmap(); + } + } + + return column.getBitmapIndex().getBitmap(value); + } + + @Override + public ImmutableBitmap getBitmapIndex(Predicate predicate, BitmapFactory bitmapFactory) + { + Indexed dimValues = getSortedIndexedValues(); + if (dimValues == null || dimValues.size() == 0) { + return bitmapFactory.makeEmptyImmutableBitmap(); + } + + return bitmapFactory.union( + FunctionalIterable.create(dimValues) + .filter(predicate) + .transform( + new Function() + { + @Override + public ImmutableBitmap apply(@Nullable String input) + { + return column.getBitmapIndex().getBitmap(input); + } + } + ) + ); + } + + @Override + public ImmutableBitmap getBitmapIndex(Bound bound, BitmapFactory bitmapFactory) + { + if (!getCapabilities().hasSpatialIndexes()) { + throw new UnsupportedOperationException("Column does not support spatial filtering: " + dimensionName); + } + ImmutableRTree spatialIndex = column.getSpatialIndex().getRTree(); + Iterable search = spatialIndex.search(bound); + return bitmapFactory.union(search); + } + + @Override + public String getMinValue() + { + BitmapIndex bitmap = column.getBitmapIndex(); + return bitmap.getCardinality() > 0 ? bitmap.getValue(0) : null; + } + + @Override + public String getMaxValue() + { + BitmapIndex bitmap = column.getBitmapIndex(); + return bitmap.getCardinality() > 0 ? bitmap.getValue(bitmap.getCardinality() - 1) : null; + } + + @Override + public ColumnCapabilities getCapabilities() + { + return column.getCapabilities(); + } + + @Override + public Indexed getSortedIndexedValues() + { + return new Indexed() + { + @Override + public Class getClazz() + { + return String.class; + } + + @Override + public int size() + { + return dictionary.getCardinality(); + } + + @Override + public String get(int index) + { + return dictionary.lookupName(index); + } + + @Override + public int indexOf(String value) + { + return dictionary.lookupId(value); + } + + @Override + public Iterator iterator() + { + return IndexedIterable.create(this).iterator(); + } + }; + } + } +} diff --git a/processing/src/main/java/io/druid/segment/column/ColumnCapabilitiesImpl.java b/processing/src/main/java/io/druid/segment/column/ColumnCapabilitiesImpl.java index f8b14e37ceed..2d10f29791e7 100644 --- a/processing/src/main/java/io/druid/segment/column/ColumnCapabilitiesImpl.java +++ b/processing/src/main/java/io/druid/segment/column/ColumnCapabilitiesImpl.java @@ -118,11 +118,13 @@ public ColumnCapabilitiesImpl merge(ColumnCapabilities other) return this; } + ValueType otherType = other.getType(); + if (type == null) { - type = other.getType(); + type = otherType; } - if (!type.equals(other.getType())) { + if (otherType != null && !type.equals(other.getType())) { throw new ISE("Cannot merge columns of type[%s] and [%s]", type, other.getType()); } diff --git a/processing/src/main/java/io/druid/segment/data/IndexedRTree.java b/processing/src/main/java/io/druid/segment/data/IndexedRTree.java index ce6f778dc1c9..de5fb9c0ca03 100644 --- a/processing/src/main/java/io/druid/segment/data/IndexedRTree.java +++ b/processing/src/main/java/io/druid/segment/data/IndexedRTree.java @@ -86,6 +86,10 @@ public Class getClazz() @Override public ImmutableRTree fromByteBuffer(ByteBuffer buffer, int numBytes) { + if (numBytes == 0) { + //return new ImmutableRTree(); + } + final ByteBuffer readOnlyBuffer = buffer.asReadOnlyBuffer(); readOnlyBuffer.limit(readOnlyBuffer.position() + numBytes); return new ImmutableRTree(readOnlyBuffer, bitmapFactory); diff --git a/processing/src/main/java/io/druid/segment/data/MultiValueIndexedIntsWriter.java b/processing/src/main/java/io/druid/segment/data/MultiValueIndexedIntsWriter.java index 3aebdc5a0582..acc2f948bf95 100644 --- a/processing/src/main/java/io/druid/segment/data/MultiValueIndexedIntsWriter.java +++ b/processing/src/main/java/io/druid/segment/data/MultiValueIndexedIntsWriter.java @@ -19,10 +19,12 @@ package io.druid.segment.data; +import com.google.common.collect.Lists; import com.google.common.primitives.Ints; import com.metamx.common.IAE; import java.io.IOException; +import java.util.ArrayList; import java.util.List; public abstract class MultiValueIndexedIntsWriter implements IndexedIntsWriter @@ -36,6 +38,13 @@ public void add(Object obj) throws IOException addValues(Ints.asList((int[]) obj)); } else if (obj instanceof List) { addValues((List) obj); + } else if (obj instanceof Comparable[]) { + Comparable[] comparables = (Comparable[]) obj; + List intList = new ArrayList<>(); + for (Comparable comp : comparables) { + intList.add((Integer) comp); + } + addValues(intList); } else { throw new IAE("unsupported multi-value type: " + obj.getClass()); } diff --git a/processing/src/main/java/io/druid/segment/data/SingleValueIndexedIntsWriter.java b/processing/src/main/java/io/druid/segment/data/SingleValueIndexedIntsWriter.java index c6caf733039a..0467bc4ae043 100644 --- a/processing/src/main/java/io/druid/segment/data/SingleValueIndexedIntsWriter.java +++ b/processing/src/main/java/io/druid/segment/data/SingleValueIndexedIntsWriter.java @@ -39,6 +39,13 @@ public void add(Object obj) throws IOException } else { addValue(vals[0]); } + } else if (obj instanceof Comparable[]) { + Comparable[] vals = (Comparable[]) obj; + if (vals.length == 0) { + addValue(0); + } else { + addValue((Integer) vals[0]); + } } else { throw new IAE("Unsupported single value type: " + obj.getClass()); } diff --git a/processing/src/main/java/io/druid/segment/filter/BoundFilter.java b/processing/src/main/java/io/druid/segment/filter/BoundFilter.java index 2d2e01e8909d..ce5c203cee60 100644 --- a/processing/src/main/java/io/druid/segment/filter/BoundFilter.java +++ b/processing/src/main/java/io/druid/segment/filter/BoundFilter.java @@ -32,17 +32,17 @@ public class BoundFilter extends DimensionPredicateFilter public BoundFilter(final BoundDimFilter boundDimFilter) { super( - boundDimFilter.getDimension(), new Predicate() + boundDimFilter.getDimension(), new Predicate() { - private volatile Predicate predicate; + private volatile Predicate predicate; @Override - public boolean apply(String input) + public boolean apply(Object input) { return function().apply(input); } - private Predicate function() + private Predicate function() { if (predicate == null) { final Comparator comparator; @@ -51,11 +51,12 @@ private Predicate function() } else { comparator = new LexicographicTopNMetricSpec(null).getComparator(null, null); } - predicate = new Predicate() + predicate = new Predicate() { @Override - public boolean apply(String input) + public boolean apply(Object inputObj) { + String input = inputObj == null ? null : inputObj.toString(); if (input == null) { return false; } diff --git a/processing/src/main/java/io/druid/segment/filter/DimensionPredicateFilter.java b/processing/src/main/java/io/druid/segment/filter/DimensionPredicateFilter.java index 72d48af16a38..24baf3c31a9a 100644 --- a/processing/src/main/java/io/druid/segment/filter/DimensionPredicateFilter.java +++ b/processing/src/main/java/io/druid/segment/filter/DimensionPredicateFilter.java @@ -36,11 +36,11 @@ class DimensionPredicateFilter implements Filter { private final String dimension; - private final Predicate predicate; + private final Predicate predicate; public DimensionPredicateFilter( String dimension, - Predicate predicate + Predicate predicate ) { this.dimension = dimension; @@ -50,8 +50,16 @@ public DimensionPredicateFilter( @Override public ImmutableBitmap getBitmapIndex(final BitmapIndexSelector selector) { + if (predicate == null) { + return selector.getBitmapFactory().makeEmptyImmutableBitmap(); + } + + if (!selector.hasBitmapIndexes(dimension)) { + return selector.getBitmapIndexFromColumnScan(dimension, predicate); + } + Indexed dimValues = selector.getDimensionValues(dimension); - if (dimValues == null || dimValues.size() == 0 || predicate == null) { + if (dimValues == null || dimValues.size() == 0) { return selector.getBitmapFactory().makeEmptyImmutableBitmap(); } diff --git a/processing/src/main/java/io/druid/segment/filter/ExtractionFilter.java b/processing/src/main/java/io/druid/segment/filter/ExtractionFilter.java index 144b56326b63..b7771434bf88 100644 --- a/processing/src/main/java/io/druid/segment/filter/ExtractionFilter.java +++ b/processing/src/main/java/io/druid/segment/filter/ExtractionFilter.java @@ -94,6 +94,22 @@ public int indexOf(String value) @Override public ImmutableBitmap getBitmapIndex(BitmapIndexSelector selector) { + if (!selector.hasBitmapIndexes(dimension)) { + Predicate predicate = new Predicate() + { + @Override + public boolean apply(Object input) + { + if (input instanceof String) { + input = Strings.emptyToNull((String) input); + } + // Assuming that a null/absent/empty dimension are equivalent from the druid perspective + return value.equals(Strings.nullToEmpty(fn.apply(input))); + } + }; + return selector.getBitmapIndexFromColumnScan(dimension, predicate); + } + final List filters = makeFilters(selector); if (filters.isEmpty()) { return selector.getBitmapFactory().makeEmptyImmutableBitmap(); @@ -105,13 +121,16 @@ public ImmutableBitmap getBitmapIndex(BitmapIndexSelector selector) public ValueMatcher makeMatcher(ValueMatcherFactory factory) { return factory.makeValueMatcher( - dimension, new Predicate() + dimension, new Predicate() { @Override - public boolean apply(String input) + public boolean apply(Object input) { + if (input instanceof String) { + input = Strings.emptyToNull((String) input); + } // Assuming that a null/absent/empty dimension are equivalent from the druid perspective - return value.equals(Strings.nullToEmpty(fn.apply(Strings.emptyToNull(input)))); + return value.equals(Strings.nullToEmpty(fn.apply(input))); } } ); diff --git a/processing/src/main/java/io/druid/segment/filter/JavaScriptFilter.java b/processing/src/main/java/io/druid/segment/filter/JavaScriptFilter.java index 0703bc71d13c..7c7ecc125c8d 100644 --- a/processing/src/main/java/io/druid/segment/filter/JavaScriptFilter.java +++ b/processing/src/main/java/io/druid/segment/filter/JavaScriptFilter.java @@ -50,6 +50,19 @@ public ImmutableBitmap getBitmapIndex(final BitmapIndexSelector selector) { final Context cx = Context.enter(); try { + if (!selector.hasBitmapIndexes(dimension)) { + Predicate predicateInContext = new Predicate() + { + @Override + public boolean apply(@Nullable Object input) + { + return predicate.applyInContext(cx, input); + } + }; + + return selector.getBitmapIndexFromColumnScan(dimension, predicateInContext); + } + final Indexed dimValues = selector.getDimensionValues(dimension); ImmutableBitmap bitmap; if (dimValues == null) { @@ -93,7 +106,7 @@ public ValueMatcher makeMatcher(ValueMatcherFactory factory) return factory.makeValueMatcher(dimension, predicate); } - static class JavaScriptPredicate implements Predicate + static class JavaScriptPredicate implements Predicate { final ScriptableObject scope; final Function fnApply; @@ -117,12 +130,13 @@ public JavaScriptPredicate(final String script) } @Override - public boolean apply(final String input) + public boolean apply(final Object input) { + String inputStr = input == null ? null : input.toString(); // one and only one context per thread final Context cx = Context.enter(); try { - return applyInContext(cx, input); + return applyInContext(cx, inputStr); } finally { Context.exit(); @@ -130,9 +144,9 @@ public boolean apply(final String input) } - public boolean applyInContext(Context cx, String input) + public boolean applyInContext(Context cx, Object input) { - return Context.toBoolean(fnApply.call(cx, scope, scope, new String[]{input})); + return Context.toBoolean(fnApply.call(cx, scope, scope, new Object[]{input})); } @Override diff --git a/processing/src/main/java/io/druid/segment/filter/RegexFilter.java b/processing/src/main/java/io/druid/segment/filter/RegexFilter.java index 4c1960dad573..924471ff68d3 100644 --- a/processing/src/main/java/io/druid/segment/filter/RegexFilter.java +++ b/processing/src/main/java/io/druid/segment/filter/RegexFilter.java @@ -34,14 +34,14 @@ public RegexFilter( { super( dimension, - new Predicate() + new Predicate() { Pattern compiled = Pattern.compile(pattern); @Override - public boolean apply(String input) + public boolean apply(Object input) { - return (input != null) && compiled.matcher(input).find(); + return (input != null) && compiled.matcher(input.toString()).find(); } } ); diff --git a/processing/src/main/java/io/druid/segment/filter/SearchQueryFilter.java b/processing/src/main/java/io/druid/segment/filter/SearchQueryFilter.java index fbd8b1a2ed56..a3fa50cfb347 100644 --- a/processing/src/main/java/io/druid/segment/filter/SearchQueryFilter.java +++ b/processing/src/main/java/io/druid/segment/filter/SearchQueryFilter.java @@ -38,12 +38,13 @@ public SearchQueryFilter( { super( dimension, - new Predicate() + new Predicate() { @Override - public boolean apply(@Nullable String input) + public boolean apply(@Nullable Object input) { - return query.accept(input); + String inputStr = input == null ? null : input.toString(); + return query.accept(inputStr); } } ); diff --git a/processing/src/main/java/io/druid/segment/filter/SpatialFilter.java b/processing/src/main/java/io/druid/segment/filter/SpatialFilter.java index d9bc9fe2cf5f..ab0885d745ac 100644 --- a/processing/src/main/java/io/druid/segment/filter/SpatialFilter.java +++ b/processing/src/main/java/io/druid/segment/filter/SpatialFilter.java @@ -49,8 +49,7 @@ public SpatialFilter( @Override public ImmutableBitmap getBitmapIndex(final BitmapIndexSelector selector) { - Iterable search = selector.getSpatialIndex(dimension).search(bound); - return selector.getBitmapFactory().union(search); + return selector.getBitmapIndex(dimension, bound); } @Override diff --git a/processing/src/main/java/io/druid/segment/incremental/IncrementalIndex.java b/processing/src/main/java/io/druid/segment/incremental/IncrementalIndex.java index 5b45dff302cc..00cf439d8c2e 100644 --- a/processing/src/main/java/io/druid/segment/incremental/IncrementalIndex.java +++ b/processing/src/main/java/io/druid/segment/incremental/IncrementalIndex.java @@ -32,13 +32,11 @@ import com.google.common.primitives.Longs; import com.metamx.common.IAE; import com.metamx.common.ISE; -import com.metamx.common.parsers.ParseException; import io.druid.data.input.InputRow; import io.druid.data.input.MapBasedRow; import io.druid.data.input.Row; import io.druid.data.input.impl.DimensionSchema; import io.druid.data.input.impl.DimensionsSpec; -import io.druid.data.input.impl.NewSpatialDimensionSchema; import io.druid.data.input.impl.SpatialDimensionSchema; import io.druid.granularity.QueryGranularity; import io.druid.query.aggregation.AggregatorFactory; @@ -46,16 +44,24 @@ import io.druid.query.dimension.DimensionSpec; import io.druid.query.extraction.ExtractionFn; import io.druid.segment.ColumnSelectorFactory; +import io.druid.segment.DimensionHandler; +import io.druid.segment.DimensionHandlerUtil; +import io.druid.segment.DimensionIndexer; import io.druid.segment.DimensionSelector; import io.druid.segment.FloatColumnSelector; +import io.druid.segment.FloatDimensionHandler; import io.druid.segment.LongColumnSelector; +import io.druid.segment.LongDimensionHandler; import io.druid.segment.Metadata; import io.druid.segment.ObjectColumnSelector; +import io.druid.segment.StringDimensionHandler; import io.druid.segment.column.Column; import io.druid.segment.column.ColumnCapabilities; import io.druid.segment.column.ColumnCapabilitiesImpl; import io.druid.segment.column.ValueType; +import io.druid.segment.data.IndexedFloats; import io.druid.segment.data.IndexedInts; +import io.druid.segment.data.IndexedLongs; import io.druid.segment.serde.ComplexMetricExtractor; import io.druid.segment.serde.ComplexMetricSerde; import io.druid.segment.serde.ComplexMetrics; @@ -96,71 +102,23 @@ public abstract class IncrementalIndex implements Iterable, .put(DimensionSchema.ValueType.STRING, ValueType.STRING) .build(); - private static final Function STRING_TRANSFORMER = new Function() + public static int[] castComparablesAsInts(Comparable[] vals) { - @Override - public String apply(final Object o) - { - return String.valueOf(o); - } - }; - - private static final Function LONG_TRANSFORMER = new Function() - { - @Override - public Long apply(final Object o) - { - if (o == null) { - return null; - } - if (o instanceof String) { - try { - return Long.valueOf((String) o); - } - catch (NumberFormatException nfe) { - throw new ParseException(nfe, "Unable to parse value[%s] as long in column: ", o); - } - } - if (o instanceof Number) { - return ((Number) o).longValue(); - } + if (vals == null) { return null; } - }; - - private static final Function FLOAT_TRANSFORMER = new Function() - { - @Override - public Float apply(final Object o) - { - if (o == null) { - return null; - } - if (o instanceof String) { - try { - return Float.valueOf((String) o); - } - catch (NumberFormatException nfe) { - throw new ParseException(nfe, "Unable to parse value[%s] as float in column: ", o); - } - } - if (o instanceof Number) { - return ((Number) o).floatValue(); - } - return null; + int[] newVals = new int[vals.length]; + for (int i = 0; i < vals.length; i++) { + newVals[i] = (Integer) vals[i]; } - }; - - private static final Map VALUE_TRANSFORMS = ImmutableMap.builder() - .put(ValueType.LONG, LONG_TRANSFORMER) - .put(ValueType.FLOAT, FLOAT_TRANSFORMER) - .put(ValueType.STRING, STRING_TRANSFORMER) - .build(); + return newVals; + } public static ColumnSelectorFactory makeColumnSelectorFactory( final AggregatorFactory agg, final Supplier in, - final boolean deserializeComplexMetrics + final boolean deserializeComplexMetrics, + final Map columnCapabilities ) { return new ColumnSelectorFactory() @@ -256,10 +214,11 @@ public DimensionSelector makeDimensionSelector( DimensionSpec dimensionSpec ) { - return dimensionSpec.decorate(makeDimensionSelectorUndecorated(dimensionSpec)); + return dimensionSpec.decorate(makeDictEncodedStringDimensionSelector(dimensionSpec)); } - private DimensionSelector makeDimensionSelectorUndecorated( + @Override + public DimensionSelector makeDictEncodedStringDimensionSelector( DimensionSpec dimensionSpec ) { @@ -334,6 +293,49 @@ public int lookupId(String name) } return in.get().getDimension(dimension).indexOf(name); } + + @Override + public IndexedLongs getLongRow() + { + throw new UnsupportedOperationException("getLongRow() is not supported."); + } + + @Override + public Comparable getExtractedValueLong(long val) + { + throw new UnsupportedOperationException("getExtractedValueLong() is not supported."); + } + + @Override + public IndexedFloats getFloatRow() + { + throw new UnsupportedOperationException("getFloatRow() is not supported."); + } + + @Override + public Comparable getExtractedValueFloat(float val) + { + throw new UnsupportedOperationException("getExtractedValueFloat() is not supported."); + } + + @Override + public Comparable getComparableRow() + { + throw new UnsupportedOperationException("getComparableRow() is not supported."); + } + + @Override + public Comparable getExtractedValueComparable(Comparable val) + { + throw new UnsupportedOperationException("getExtractedValueComparable() is not supported."); + + } + + @Override + public ColumnCapabilities getDimCapabilities() + { + return columnCapabilities.get(dimension); + } }; } }; @@ -347,11 +349,14 @@ public int lookupId(String name) private final boolean deserializeComplexMetrics; private final boolean reportParseExceptions; private final boolean sortFacts; + private final boolean allowTypeDiscovery; private final Metadata metadata; + private final Map metricDescs; private final Map dimensionDescs; + private final List dimensionDescsList; private final Map columnCapabilities; private final List dimValues; @@ -382,7 +387,8 @@ public IncrementalIndex( final IncrementalIndexSchema incrementalIndexSchema, final boolean deserializeComplexMetrics, final boolean reportParseExceptions, - final boolean sortFacts + final boolean sortFacts, + final boolean allowTypeDiscovery ) { this.minTimestamp = incrementalIndexSchema.getMinTimestamp(); @@ -392,11 +398,13 @@ public IncrementalIndex( this.deserializeComplexMetrics = deserializeComplexMetrics; this.reportParseExceptions = reportParseExceptions; this.sortFacts = sortFacts; + this.allowTypeDiscovery = allowTypeDiscovery; + this.columnCapabilities = Maps.newHashMap(); + this.metadata = new Metadata().setAggregators(getCombiningAggregators(metrics)); this.aggs = initAggs(metrics, rowSupplier, deserializeComplexMetrics); - this.columnCapabilities = Maps.newHashMap(); this.metricDescs = Maps.newLinkedHashMap(); for (AggregatorFactory metric : metrics) { @@ -410,16 +418,18 @@ public IncrementalIndex( this.dimensionDescs = Maps.newLinkedHashMap(); this.dimValues = Collections.synchronizedList(Lists.newArrayList()); + this.dimensionDescsList = new ArrayList<>(); for (DimensionSchema dimSchema : dimensionsSpec.getDimensions()) { - ColumnCapabilitiesImpl capabilities = new ColumnCapabilitiesImpl(); ValueType type = TYPE_MAP.get(dimSchema.getValueType()); - capabilities.setType(type); + String dimName = dimSchema.getName(); + ColumnCapabilitiesImpl capabilities = makeCapabilitesFromValueType(type); if (dimSchema.getTypeName().equals(DimensionSchema.SPATIAL_TYPE_NAME)) { capabilities.setHasSpatialIndexes(true); } else { - addNewDimension(dimSchema.getName(), capabilities); + DimensionHandler handler = DimensionHandlerUtil.getHandlerFromCapabilities(dimName, capabilities); + addNewDimension(dimName, capabilities, handler); } - columnCapabilities.put(dimSchema.getName(), capabilities); + columnCapabilities.put(dimName, capabilities); } // This should really be more generic @@ -429,27 +439,6 @@ public IncrementalIndex( } } - private DimDim newDimDim(String dimension, ValueType type) { - DimDim newDimDim; - switch (type) { - case LONG: - newDimDim = makeDimDim(dimension, getDimensionDescs()); - break; - case FLOAT: - newDimDim = makeDimDim(dimension, getDimensionDescs()); - break; - case STRING: - newDimDim = new NullValueConverterDimDim(makeDimDim(dimension, getDimensionDescs())); - break; - default: - throw new IAE("Invalid column type: " + type); - } - return newDimDim; - } - - // use newDimDim() to create a DimDim, makeDimDim() provides the subclass-specific implementation - protected abstract DimDim makeDimDim(String dimension, Object lock); - public abstract ConcurrentMap getFacts(); public abstract boolean canAppendRow(); @@ -487,7 +476,6 @@ protected abstract Integer addToFacts( @Override public void close() { - dimValues.clear(); } public InputRow formatRow(InputRow row) @@ -519,29 +507,14 @@ private ValueType getTypeFromDimVal(Object dimVal) return TYPE_MAP.get(singleVal.getClass()); } - private List getRowDimensionAsComparables(InputRow row, String dimension, ValueType type) + public Map getDimensionDescs() { - final Object dimVal = row.getRaw(dimension); - final Function transformer = VALUE_TRANSFORMS.get(type); - final List dimensionValues; - try { - if (dimVal == null) { - dimensionValues = Collections.emptyList(); - } else if (dimVal instanceof List) { - dimensionValues = Lists.transform((List) dimVal, transformer); - } else { - dimensionValues = Collections.singletonList((Comparable) transformer.apply(dimVal)); - } - } - catch (ParseException pe) { - throw new ParseException(pe.getMessage() + dimension); - } - return dimensionValues; + return dimensionDescs; } - public Map getDimensionDescs() + public Map getColumnCapabilities() { - return dimensionDescs; + return columnCapabilities; } /** @@ -582,50 +555,53 @@ TimeAndDims toTimeAndDims(InputRow row) throws IndexSizeExceededException final List rowDimensions = row.getDimensions(); - int[][] dims; + Comparable[][] dims; ValueType[] types; - List overflow = null; + List overflow = null; List overflowTypes = null; synchronized (dimensionDescs) { - dims = new int[dimensionDescs.size()][]; + dims = new Comparable[dimensionDescs.size()][]; types = new ValueType[dimensionDescs.size()]; for (String dimension : rowDimensions) { + boolean wasNewDim = false; List dimensionValues; - ColumnCapabilitiesImpl capabilities; ValueType valType = null; DimensionDesc desc = dimensionDescs.get(dimension); if (desc != null) { capabilities = desc.getCapabilities(); } else { + wasNewDim = true; capabilities = columnCapabilities.get(dimension); if (capabilities == null) { - capabilities = new ColumnCapabilitiesImpl(); - // For schemaless type discovery, assume everything is a String for now, can change later. - //valType = getTypeFromDimVal(row.getRaw(dimension)); + if (allowTypeDiscovery) { + valType = getTypeFromDimVal(row.getRaw(dimension)); + } if (valType == null) { - valType = ValueType.STRING; + valType = ValueType.STRING; // default to string if type discovery was not enabled/failed } - capabilities.setType(valType); + capabilities = makeCapabilitesFromValueType(valType); columnCapabilities.put(dimension, capabilities); } + valType = capabilities.getType(); + DimensionHandler handler = DimensionHandlerUtil.getHandlerFromCapabilities(dimension, capabilities); + desc = addNewDimension(dimension, capabilities, handler); } valType = capabilities.getType(); - dimensionValues = getRowDimensionAsComparables(row, dimension, valType); + DimensionIndexer indexer = desc.getIndexer(); + Comparable[] dimsKey = indexer.processRowValsToIndexKey(row.getRaw(dimension)); // Set column capabilities as data is coming in - if (!capabilities.hasMultipleValues() && dimensionValues.size() > 1) { + if (!capabilities.hasMultipleValues() && dimsKey != null && dimsKey.length > 1) { capabilities.setHasMultipleValues(true); } - if (desc == null) { - desc = addNewDimension(dimension, capabilities); - + if (wasNewDim) { if (overflow == null) { overflow = Lists.newArrayList(); overflowTypes = Lists.newArrayList(); } - overflow.add(getDimVals(desc.getValues(), dimensionValues)); + overflow.add(dimsKey); overflowTypes.add(valType); } else if (desc.getIndex() > dims.length || dims[desc.getIndex()] != null) { /* @@ -639,7 +615,7 @@ TimeAndDims toTimeAndDims(InputRow row) throws IndexSizeExceededException */ throw new ISE("Dimension[%s] occurred more than once in InputRow", dimension); } else { - dims[desc.getIndex()] = getDimVals(desc.getValues(), dimensionValues); + dims[desc.getIndex()] = dimsKey; types[desc.getIndex()] = valType; } } @@ -647,7 +623,7 @@ TimeAndDims toTimeAndDims(InputRow row) throws IndexSizeExceededException if (overflow != null) { // Merge overflow and non-overflow - int[][] newDims = new int[dims.length + overflow.size()][]; + Comparable[][] newDims = new Comparable[dims.length + overflow.size()][]; ValueType[] newTypes = new ValueType[dims.length + overflow.size()]; System.arraycopy(dims, 0, newDims, 0, dims.length); System.arraycopy(types, 0, newTypes, 0, dims.length); @@ -698,33 +674,6 @@ private long getMaxTimeMillis() } } - private int[] getDimVals(final DimDim dimLookup, final List dimValues) - { - if (dimValues.size() == 0) { - // NULL VALUE - dimLookup.add(null); - return null; - } - - if (dimValues.size() == 1) { - Comparable dimVal = dimValues.get(0); - // For Strings, return an array of dictionary-encoded IDs - // For numerics, return the numeric values directly - return new int[]{dimLookup.add(dimVal)}; - } - - Comparable[] dimArray = dimValues.toArray(new Comparable[dimValues.size()]); - Arrays.sort(dimArray); - - final int[] retVal = new int[dimArray.length]; - - for (int i = 0; i < dimArray.length; i++) { - retVal[i] = dimLookup.add(dimArray[i]); - } - - return retVal; - } - public AggregatorType[] getAggs() { return aggs; @@ -777,10 +726,10 @@ public DateTime getMaxTime() return isEmpty() ? null : new DateTime(getMaxTimeMillis()); } - public DimDim getDimensionValues(String dimension) + public Integer getDimensionIndex(String dimension) { DimensionDesc dimSpec = getDimension(dimension); - return dimSpec == null ? null : dimSpec.getValues(); + return dimSpec == null ? null : dimSpec.getIndex(); } public List getDimensionOrder() @@ -790,12 +739,27 @@ public List getDimensionOrder() } } + private ColumnCapabilitiesImpl makeCapabilitesFromValueType(ValueType type) + { + ColumnCapabilitiesImpl capabilities = new ColumnCapabilitiesImpl(); + capabilities.setDictionaryEncoded(type == ValueType.STRING); + capabilities.setHasBitmapIndexes(type == ValueType.STRING); + capabilities.setType(type); + return capabilities; + } + + public void loadDimensionCapability(String dimension, ValueType type) + { + ColumnCapabilitiesImpl capabilities = makeCapabilitesFromValueType(type); + columnCapabilities.put(dimension, capabilities); + } + /* * Currently called to initialize IncrementalIndex dimension order during index creation * Index dimension ordering could be changed to initialize from DimensionsSpec after resolution of * https://github.com/druid-io/druid/issues/2011 */ - public void loadDimensionIterable(Iterable oldDimensionOrder) + public void loadDimensionIterable(Iterable oldDimensionOrder, Map oldColumnCapabilities) { synchronized (dimensionDescs) { if (!dimensionDescs.isEmpty()) { @@ -803,25 +767,21 @@ public void loadDimensionIterable(Iterable oldDimensionOrder) } for (String dim : oldDimensionOrder) { if (dimensionDescs.get(dim) == null) { - ColumnCapabilitiesImpl capabilities = new ColumnCapabilitiesImpl(); - capabilities.setType(ValueType.STRING); + ColumnCapabilitiesImpl capabilities = oldColumnCapabilities.get(dim); columnCapabilities.put(dim, capabilities); - addNewDimension(dim, capabilities); + DimensionHandler handler = DimensionHandlerUtil.getHandlerFromCapabilities(dim, capabilities); + addNewDimension(dim, capabilities, handler); } } } } @GuardedBy("dimensionDescs") - private DimensionDesc addNewDimension(String dim, ColumnCapabilitiesImpl capabilities) + private DimensionDesc addNewDimension(String dim, ColumnCapabilitiesImpl capabilities, DimensionHandler handler) { - DimensionDesc desc = new DimensionDesc(dimensionDescs.size(), dim, newDimDim(dim, capabilities.getType()), capabilities); - if (dimValues.size() != desc.getIndex()) { - throw new ISE("dimensionDescs and dimValues for [%s] is out of sync!!", dim); - } - + DimensionDesc desc = new DimensionDesc(dimensionDescs.size(), dim, capabilities, handler, dimensionDescs); dimensionDescs.put(dim, desc); - dimValues.add(desc.getValues()); + dimensionDescsList.add(desc); return desc; } @@ -869,6 +829,15 @@ private static AggregatorFactory[] getCombiningAggregators(AggregatorFactory[] a return combiningAggregators; } + public Map getDimensionHandlers() + { + Map handlers = Maps.newLinkedHashMap(); + for (DimensionDesc desc : dimensionDescsList) { + handlers.put(desc.getName(), desc.getHandler()); + } + return handlers; + } + @Override public Iterator iterator() { @@ -901,12 +870,12 @@ public Row apply(final Map.Entry input) final TimeAndDims timeAndDims = input.getKey(); final int rowOffset = input.getValue(); - int[][] theDims = timeAndDims.getDims(); //TODO: remove dictionary encoding for numerics later + Comparable[][] theDims = timeAndDims.getDims(); ValueType[] types = timeAndDims.getTypes(); Map theVals = Maps.newLinkedHashMap(); for (int i = 0; i < theDims.length; ++i) { - int[] dim = theDims[i]; + Comparable[] dim = theDims[i]; ValueType type = types[i]; DimensionDesc dimensionDesc = dimensions.get(i); if (dimensionDesc == null) { @@ -917,8 +886,12 @@ public Row apply(final Map.Entry input) theVals.put(dimensionName, null); continue; } + + final ColumnCapabilities capabilities = dimensionDesc.getCapabilities(); + final DimensionIndexer indexer = dimensionDesc.getIndexer(); + if (dim.length == 1) { - Comparable val = dimensionDesc.getValues().getValue(dim[0]); + Comparable val = indexer.getActualValue(dim[0], false); if (type == ValueType.STRING) { val = Strings.nullToEmpty((String) val); } @@ -926,7 +899,7 @@ public Row apply(final Map.Entry input) } else { Comparable[] dimVals = new Comparable[dim.length]; for (int j = 0; j < dimVals.length; j++) { - Comparable val = dimensionDesc.getValues().getValue(dim[j]); + Comparable val = indexer.getActualValue(dim[j], false); if (type == ValueType.STRING) { val = Strings.nullToEmpty((String) val); } @@ -964,15 +937,17 @@ public static final class DimensionDesc { private final int index; private final String name; - private final DimDim values; private final ColumnCapabilitiesImpl capabilities; + private final DimensionHandler handler; + private final DimensionIndexer indexer; - public DimensionDesc(int index, String name, DimDim values, ColumnCapabilitiesImpl capabilities) + public DimensionDesc(int index, String name, ColumnCapabilitiesImpl capabilities, DimensionHandler handler, Object lock) { this.index = index; this.name = name; - this.values = values; this.capabilities = capabilities; + this.handler = handler; + this.indexer = handler.makeIndexer(lock); } public int getIndex() @@ -985,15 +960,21 @@ public String getName() return name; } - public DimDim getValues() + public ColumnCapabilitiesImpl getCapabilities() + { + return capabilities; + } + + public DimensionHandler getHandler() { - return values; + return handler; } - public ColumnCapabilitiesImpl getCapabilities() + public DimensionIndexer getIndexer() { - return capabilities; + return indexer; } + } public static final class MetricDesc @@ -1167,12 +1148,12 @@ public String getValueFromSortedId(int index) static final class TimeAndDims { private final long timestamp; - private final int[][] dims; + private final Comparable[][] dims; private final ValueType[] types; TimeAndDims( long timestamp, - int[][] dims, + Comparable[][] dims, ValueType[] types ) { @@ -1186,7 +1167,7 @@ long getTimestamp() return timestamp; } - int[][] getDims() + Comparable[][] getDims() { return dims; } @@ -1202,10 +1183,10 @@ public String toString() return "TimeAndDims{" + "timestamp=" + new DateTime(timestamp) + ", dims=" + Lists.transform( - Arrays.asList(dims), new Function() + Arrays.asList(dims), new Function() { @Override - public Object apply(@Nullable int[] input) + public Object apply(@Nullable Comparable[] input) { if (input == null || input.length == 0) { return Arrays.asList("null"); @@ -1219,17 +1200,17 @@ public Object apply(@Nullable int[] input) protected final Comparator dimsComparator() { - return new TimeAndDimsComp(dimValues); + return new TimeAndDimsComp(dimensionDescsList); } @VisibleForTesting static final class TimeAndDimsComp implements Comparator { - private final List dimValues; + private List dimensionDescs; - public TimeAndDimsComp(List dimValues) + public TimeAndDimsComp(List dimDescs) { - this.dimValues = dimValues; + this.dimensionDescs = dimDescs; } @Override @@ -1260,8 +1241,8 @@ public int compare(TimeAndDims lhs, TimeAndDims rhs) return retVal; } - final int[] lhsIdxs = lhs.dims[index]; - final int[] rhsIdxs = rhs.dims[index]; + final Comparable[] lhsIdxs = lhs.dims[index]; + final Comparable[] rhsIdxs = rhs.dims[index]; if (lhsIdxs == null) { if (rhsIdxs == null) { @@ -1276,15 +1257,18 @@ public int compare(TimeAndDims lhs, TimeAndDims rhs) } retVal = Ints.compare(lhsIdxs.length, rhsIdxs.length); + final DimensionHandler handler = dimensionDescs.get(index).getHandler(); + final DimensionIndexer indexer = dimensionDescs.get(index).getIndexer(); + final Comparator comparator = handler.getActualComparator(); int valsIndex = 0; while (retVal == 0 && valsIndex < lhsIdxs.length) { if (lhsIdxs[valsIndex] != rhsIdxs[valsIndex]) { - final DimDim dimLookup = dimValues.get(index); - final Comparable lhsVal = dimLookup.getValue(lhsIdxs[valsIndex]); - final Comparable rhsVal = dimLookup.getValue(rhsIdxs[valsIndex]); + final Comparable lhsVal = indexer.getActualValue(lhsIdxs[valsIndex], false); + final Comparable rhsVal = indexer.getActualValue(rhsIdxs[valsIndex], false); + if (lhsVal != null && rhsVal != null) { - retVal = lhsVal.compareTo(rhsVal); + retVal = comparator.compare(lhsVal, rhsVal); } else if (lhsVal == null ^ rhsVal == null) { retVal = lhsVal == null ? -1 : 1; } diff --git a/processing/src/main/java/io/druid/segment/incremental/IncrementalIndexAdapter.java b/processing/src/main/java/io/druid/segment/incremental/IncrementalIndexAdapter.java index 057efbc67292..5b9b1b9a1c1e 100644 --- a/processing/src/main/java/io/druid/segment/incremental/IncrementalIndexAdapter.java +++ b/processing/src/main/java/io/druid/segment/incremental/IncrementalIndexAdapter.java @@ -26,6 +26,8 @@ import com.metamx.collections.bitmap.BitmapFactory; import com.metamx.collections.bitmap.MutableBitmap; import com.metamx.common.logger.Logger; +import io.druid.segment.DimensionHandler; +import io.druid.segment.DimensionIndexer; import io.druid.segment.IndexableAdapter; import io.druid.segment.Metadata; import io.druid.segment.Rowboat; @@ -53,39 +55,30 @@ public class IncrementalIndexAdapter implements IndexableAdapter private final IncrementalIndex index; private final Set hasNullValueDimensions; - private final Map indexers; + private final Map accessors; - private class DimensionIndexer + + private class DimensionAccessor { private final IncrementalIndex.DimensionDesc dimensionDesc; private final MutableBitmap[] invertedIndexes; + private final DimensionIndexer indexer; - private IncrementalIndex.SortedDimLookup dimLookup; - - public DimensionIndexer(IncrementalIndex.DimensionDesc dimensionDesc) + public DimensionAccessor(IncrementalIndex.DimensionDesc dimensionDesc) { this.dimensionDesc = dimensionDesc; - this.invertedIndexes = new MutableBitmap[dimensionDesc.getValues().size() + 1]; - } - - private IncrementalIndex.DimDim getDimValues() - { - return dimensionDesc.getValues(); - } - - private IncrementalIndex.SortedDimLookup getDimLookup() - { - if (dimLookup == null) { - final IncrementalIndex.DimDim dimDim = dimensionDesc.getValues(); - if (hasNullValueDimensions.contains(dimensionDesc.getName()) && !dimDim.contains(null)) { - dimDim.add(null); - } - dimLookup = dimDim.sort(); + this.indexer = dimensionDesc.getIndexer(); + if(dimensionDesc.getCapabilities().hasBitmapIndexes()) { + this.invertedIndexes = new MutableBitmap[indexer.getCardinality() + 1]; + } else { + this.invertedIndexes = null; } - return dimLookup; } + } + private Map invertedIndexes; + public IncrementalIndexAdapter( Interval dataInterval, IncrementalIndex index, BitmapFactory bitmapFactory ) @@ -106,39 +99,39 @@ public IncrementalIndexAdapter( final List dimensions = index.getDimensions(); - indexers = Maps.newHashMapWithExpectedSize(dimensions.size()); + accessors = Maps.newHashMapWithExpectedSize(dimensions.size()); for (IncrementalIndex.DimensionDesc dimension : dimensions) { - indexers.put(dimension.getName(), new DimensionIndexer(dimension)); + accessors.put(dimension.getName(), new DimensionAccessor(dimension)); } int rowNum = 0; for (IncrementalIndex.TimeAndDims timeAndDims : index.getFacts().keySet()) { - final int[][] dims = timeAndDims.getDims(); + final Comparable[][] dims = timeAndDims.getDims(); for (IncrementalIndex.DimensionDesc dimension : dimensions) { final int dimIndex = dimension.getIndex(); - DimensionIndexer indexer = indexers.get(dimension.getName()); + DimensionAccessor accessor = accessors.get(dimension.getName()); if (dimIndex >= dims.length || dims[dimIndex] == null) { hasNullValueDimensions.add(dimension.getName()); + accessor.indexer.addNullLookup(); continue; } - final IncrementalIndex.DimDim values = dimension.getValues(); - if (hasNullValue(values, dims[dimIndex])) { - hasNullValueDimensions.add(dimension.getName()); - } + final ColumnCapabilities capabilities = dimension.getCapabilities(); - final MutableBitmap[] bitmapIndexes = indexer.invertedIndexes; + if(capabilities.hasBitmapIndexes()) { + final MutableBitmap[] bitmapIndexes = accessor.invertedIndexes; - for (Comparable dimIdxComparable : dims[dimIndex]) { - Integer dimIdx = (Integer) dimIdxComparable; - if (bitmapIndexes[dimIdx] == null) { - bitmapIndexes[dimIdx] = bitmapFactory.makeEmptyMutableBitmap(); - } - try { - bitmapIndexes[dimIdx].add(rowNum); - } - catch (Exception e) { - log.info(e.toString()); + for (Comparable dimIdxComparable : dims[dimIndex]) { + Integer dimIdx = (Integer) dimIdxComparable; + if (bitmapIndexes[dimIdx] == null) { + bitmapIndexes[dimIdx] = bitmapFactory.makeEmptyMutableBitmap(); + } + try { + bitmapIndexes[dimIdx].add(rowNum); + } + catch (Exception e) { + log.info(e.toString()); + } } } } @@ -174,48 +167,14 @@ public Indexed getMetricNames() @Override public Indexed getDimValueLookup(String dimension) { - final DimensionIndexer indexer = indexers.get(dimension); - if (indexer == null) { + final DimensionAccessor accessor = accessors.get(dimension); + if (accessor == null) { return null; } - final IncrementalIndex.DimDim dimDim = indexer.getDimValues(); - final IncrementalIndex.SortedDimLookup dimLookup = indexer.getDimLookup(); - return new Indexed() - { - @Override - public Class getClazz() - { - return String.class; - } - - @Override - public int size() - { - return dimLookup.size(); - } + final DimensionIndexer indexer = accessor.dimensionDesc.getIndexer(); - @Override - public String get(int index) - { - Comparable val = dimLookup.getValueFromSortedId(index); - String strVal = val != null ? val.toString() : null; - return strVal; - } - - @Override - public int indexOf(String value) - { - int id = dimDim.getId(value); - return id < 0 ? -1 : dimLookup.getSortedIdFromUnsortedId(id); - } - - @Override - public Iterator iterator() - { - return IndexedIterable.create(this).iterator(); - } - }; + return indexer.getSortedIndexedValues(); } @Override @@ -227,9 +186,11 @@ public Iterable getRows() public Iterator iterator() { final List dimensions = index.getDimensions(); - final IncrementalIndex.SortedDimLookup[] sortedDimLookups = new IncrementalIndex.SortedDimLookup[dimensions.size()]; + final DimensionHandler[] handlers = new DimensionHandler[dimensions.size()]; + final DimensionIndexer[] indexers = new DimensionIndexer[dimensions.size()]; for (IncrementalIndex.DimensionDesc dimension : dimensions) { - sortedDimLookups[dimension.getIndex()] = indexers.get(dimension.getName()).getDimLookup(); + handlers[dimension.getIndex()] = dimension.getHandler(); + indexers[dimension.getIndex()] = dimension.getIndexer(); } /* @@ -247,27 +208,31 @@ public Iterator iterator() public Rowboat apply(Map.Entry input) { final IncrementalIndex.TimeAndDims timeAndDims = input.getKey(); - final int[][] dimValues = timeAndDims.getDims(); + final Comparable[][] dimValues = timeAndDims.getDims(); final int rowOffset = input.getValue(); - int[][] dims = new int[dimValues.length][]; + Comparable[][] dims = new Comparable[dimValues.length][]; for (IncrementalIndex.DimensionDesc dimension : dimensions) { final int dimIndex = dimension.getIndex(); + final ColumnCapabilities capabilities = dimension.getCapabilities(); if (dimIndex >= dimValues.length || dimValues[dimIndex] == null) { continue; } - dims[dimIndex] = new int[dimValues[dimIndex].length]; + dims[dimIndex] = new Comparable[dimValues[dimIndex].length]; if (dimIndex >= dims.length || dims[dimIndex] == null) { continue; } - + final DimensionIndexer indexer = indexers[dimIndex]; for (int i = 0; i < dimValues[dimIndex].length; ++i) { - dims[dimIndex][i] = sortedDimLookups[dimIndex].getSortedIdFromUnsortedId(dimValues[dimIndex][i]); - //TODO: in later PR, Rowboat will use Comparable[][] instead of int[][] - // Can remove dictionary encoding for numeric dims then. + // The encoded values in the TimeAndDims key are not sorted based on their final unencoded values, so need this lookup. + try { + dims[dimIndex][i] = indexer.getSortedEncodedValueFromUnsorted(dimValues[dimIndex][i]); + } catch (Exception e) { + System.out.println(e.getLocalizedMessage()); + } } } @@ -280,7 +245,8 @@ public Rowboat apply(Map.Entry input) timeAndDims.getTimestamp(), dims, metrics, - count++ + count++, + handlers ); } } @@ -292,14 +258,19 @@ public Rowboat apply(Map.Entry input) @Override public IndexedInts getBitmapIndex(String dimension, int index) { - DimensionIndexer accessor = indexers.get(dimension); + DimensionAccessor accessor = accessors.get(dimension); if (accessor == null) { return EmptyIndexedInts.EMPTY_INDEXED_INTS; } + ColumnCapabilities capabilities = accessor.dimensionDesc.getCapabilities(); + DimensionIndexer indexer = accessor.dimensionDesc.getIndexer(); - IncrementalIndex.SortedDimLookup dimLookup = accessor.getDimLookup(); - final int id = dimLookup.getUnsortedIdFromSortedId(index); - if (id < 0 || id >= dimLookup.size()) { + if (!capabilities.hasBitmapIndexes()) { + return EmptyIndexedInts.EMPTY_INDEXED_INTS; + } + + final int id = (Integer) indexer.getUnsortedEncodedValueFromSorted(index); + if (id < 0 || id >= indexer.getCardinality()) { return EmptyIndexedInts.EMPTY_INDEXED_INTS; } @@ -324,27 +295,6 @@ public ColumnCapabilities getCapabilities(String column) return index.getCapabilities(column); } - private boolean hasNullValue(IncrementalIndex.DimDim dimDim, int[] dimIndices) - { - if (dimIndices == null || dimIndices.length == 0) { - return true; - } - for (int dimIndex : dimIndices) { - Comparable val = dimDim.getValue(dimIndex); - - if (val == null) { - return true; - } - - if (val instanceof String) { - if (((String) val).length() == 0) { - return true; - } - } - } - return false; - } - static class BitmapIndexedInts implements IndexedInts { diff --git a/processing/src/main/java/io/druid/segment/incremental/IncrementalIndexStorageAdapter.java b/processing/src/main/java/io/druid/segment/incremental/IncrementalIndexStorageAdapter.java index 56a0e103dd00..3ffc0043d48f 100644 --- a/processing/src/main/java/io/druid/segment/incremental/IncrementalIndexStorageAdapter.java +++ b/processing/src/main/java/io/druid/segment/incremental/IncrementalIndexStorageAdapter.java @@ -39,6 +39,8 @@ import io.druid.query.filter.ValueMatcherFactory; import io.druid.segment.Capabilities; import io.druid.segment.Cursor; +import io.druid.segment.DimensionHandler; +import io.druid.segment.DimensionIndexer; import io.druid.segment.DimensionSelector; import io.druid.segment.FloatColumnSelector; import io.druid.segment.LongColumnSelector; @@ -47,10 +49,14 @@ import io.druid.segment.ObjectColumnSelector; import io.druid.segment.SingleScanTimeDimSelector; import io.druid.segment.StorageAdapter; +import io.druid.segment.StringDimensionHandler; import io.druid.segment.column.Column; import io.druid.segment.column.ColumnCapabilities; +import io.druid.segment.column.ValueType; import io.druid.segment.data.Indexed; +import io.druid.segment.data.IndexedFloats; import io.druid.segment.data.IndexedInts; +import io.druid.segment.data.IndexedLongs; import io.druid.segment.data.ListIndexed; import io.druid.segment.filter.BooleanValueMatcher; import io.druid.segment.serde.ComplexMetricSerde; @@ -62,6 +68,7 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Collections; +import java.util.Comparator; import java.util.Iterator; import java.util.List; import java.util.Map; @@ -72,7 +79,7 @@ public class IncrementalIndexStorageAdapter implements StorageAdapter { private static final Splitter SPLITTER = Splitter.on(","); - private static final NullDimensionSelector NULL_DIMENSION_SELECTOR = new NullDimensionSelector(); + private static final NullDimensionSelector NULL_DIMENSION_SELECTOR = new NullDimensionSelector(ValueType.STRING); private final IncrementalIndex index; @@ -113,11 +120,14 @@ public int getDimensionCardinality(String dimension) if (dimension.equals(Column.TIME_COLUMN_NAME)) { return Integer.MAX_VALUE; } - IncrementalIndex.DimDim dimDim = index.getDimensionValues(dimension); - if (dimDim == null) { + + IncrementalIndex.DimensionDesc desc = index.getDimension(dimension); + if (desc == null) { return 0; } - return dimDim.size(); + + DimensionIndexer indexer = index.getDimension(dimension).getIndexer(); + return indexer.getCardinality(); } @Override @@ -141,15 +151,25 @@ public DateTime getMaxTime() @Override public Comparable getMinValue(String column) { - IncrementalIndex.DimDim dimDim = index.getDimensionValues(column); - return dimDim == null ? null : dimDim.getMinValue(); + IncrementalIndex.DimensionDesc desc = index.getDimension(column); + if (desc == null) { + return null; + } + + DimensionIndexer indexer = desc.getIndexer(); + return indexer.getMinValue(); } @Override public Comparable getMaxValue(String column) { - IncrementalIndex.DimDim dimDim = index.getDimensionValues(column); - return dimDim == null ? null : dimDim.getMaxValue(); + IncrementalIndex.DimensionDesc desc = index.getDimension(column); + if (desc == null) { + return null; + } + + DimensionIndexer indexer = desc.getIndexer(); + return indexer.getMaxValue(); } @Override @@ -164,6 +184,12 @@ public ColumnCapabilities getColumnCapabilities(String column) return index.getCapabilities(column); } + @Override + public Map getDimensionHandlers() + { + return index.getDimensionHandlers(); + } + @Override public String getColumnTypeName(String column) { @@ -178,7 +204,12 @@ public DateTime getMaxIngestedEventTime() } @Override - public Sequence makeCursors(final Filter filter, final Interval interval, final QueryGranularity gran, final boolean descending) + public Sequence makeCursors( + final Filter filter, + final Interval interval, + final QueryGranularity gran, + final boolean descending + ) { if (index.isEmpty()) { return Sequences.empty(); @@ -236,10 +267,10 @@ public Cursor apply(@Nullable final Long input) { cursorMap = index.getSubMap( new IncrementalIndex.TimeAndDims( - timeStart, new int[][]{}, null + timeStart, new Comparable[][]{}, null ), new IncrementalIndex.TimeAndDims( - Math.min(actualInterval.getEndMillis(), gran.next(input)), new int[][]{}, null + Math.min(actualInterval.getEndMillis(), gran.next(input)), new Comparable[][]{}, null ) ); if (descending) { @@ -330,46 +361,65 @@ public void reset() public DimensionSelector makeDimensionSelector( DimensionSpec dimensionSpec ) - { - return dimensionSpec.decorate(makeDimensionSelectorUndecorated(dimensionSpec)); - } - - private DimensionSelector makeDimensionSelectorUndecorated( - DimensionSpec dimensionSpec - ) { final String dimension = dimensionSpec.getDimension(); final ExtractionFn extractionFn = dimensionSpec.getExtractionFn(); if (dimension.equals(Column.TIME_COLUMN_NAME)) { - return new SingleScanTimeDimSelector(makeLongColumnSelector(dimension), extractionFn, descending); + DimensionSelector selector = new SingleScanTimeDimSelector( + makeLongColumnSelector(dimension), + extractionFn, + descending + ); + return dimensionSpec.decorate(selector); } - final IncrementalIndex.DimensionDesc dimensionDesc = index.getDimension(dimension); + final IncrementalIndex.DimensionDesc dimensionDesc = index.getDimension(dimensionSpec.getDimension()); if (dimensionDesc == null) { - return NULL_DIMENSION_SELECTOR; + return dimensionSpec.decorate(NULL_DIMENSION_SELECTOR); } + DimensionHandler handler = dimensionDesc.getHandler(); + return dimensionSpec.decorate(handler.getDimensionSelector( + this, + dimensionSpec, + index.getCapabilities(dimensionSpec.getDimension()) + )); + } - final int dimIndex = dimensionDesc.getIndex(); - final IncrementalIndex.DimDim dimValLookup = dimensionDesc.getValues(); + @Override + public DimensionSelector makeDictEncodedStringDimensionSelector( + DimensionSpec dimensionSpec + ) + { + final String dimension = dimensionSpec.getDimension(); + final ExtractionFn extractionFn = dimensionSpec.getExtractionFn(); + final ColumnCapabilities capabilities = index.getCapabilities(dimension); + final IncrementalIndex.DimensionDesc dimensionDesc = index.getDimension(dimension); - final int maxId = dimValLookup.size(); + final int dimIndex = dimensionDesc.getIndex(); + final StringDimensionHandler.StringDimensionIndexer indexer = (StringDimensionHandler.StringDimensionIndexer) dimensionDesc.getIndexer(); + final int maxId = indexer.getCardinality(); return new DimensionSelector() { @Override public IndexedInts getRow() { - final int[][] dims = currEntry.getKey().getDims(); + final Comparable[][] dims = currEntry.getKey().getDims(); - int[] indices = dimIndex < dims.length ? dims[dimIndex] : null; + int[] indices; + if (dimIndex < dims.length) { + indices = IncrementalIndex.castComparablesAsInts(dims[dimIndex]); + } else { + indices = null; + } + int nullId = indexer.getEncodedValue(null, false); List valsTmp = null; - if ((indices == null || indices.length == 0) && dimValLookup.contains(null)) { - int id = dimValLookup.getId(null); - if (id < maxId) { + if ((indices == null || indices.length == 0) && nullId > -1) { + if (nullId < maxId) { valsTmp = new ArrayList<>(1); - valsTmp.add(id); + valsTmp.add(nullId); } } else if (indices != null && indices.length > 0) { valsTmp = new ArrayList<>(indices.length); @@ -425,8 +475,7 @@ public int getValueCardinality() @Override public String lookupName(int id) { - // TODO: needs update to DimensionSelector interface to allow multi-types, just use Strings for now - final Comparable value = dimValLookup.getValue(id); + final Comparable value = indexer.getActualValue(id, false); final String strValue = value == null ? null : value.toString(); return extractionFn == null ? strValue : extractionFn.apply(strValue); @@ -440,7 +489,50 @@ public int lookupId(String name) "cannot perform lookup when applying an extraction function" ); } - return dimValLookup.getId(name); + return indexer.getEncodedValue(name, false); + } + + @Override + public IndexedLongs getLongRow() + { + throw new UnsupportedOperationException("getLongRow() is not supported."); + } + + @Override + public Comparable getExtractedValueLong(long val) + { + throw new UnsupportedOperationException("getExtractedValueLong() is not supported."); + } + + @Override + public IndexedFloats getFloatRow() + { + throw new UnsupportedOperationException("getFloatRow() is not supported."); + } + + @Override + public Comparable getExtractedValueFloat(float val) + { + throw new UnsupportedOperationException("getExtractedValueFloat() is not supported."); + } + + @Override + public Comparable getComparableRow() + { + throw new UnsupportedOperationException("getComparableRow() is not supported."); + } + + @Override + public Comparable getExtractedValueComparable(Comparable val) + { + throw new UnsupportedOperationException("getExtractedValueComparable() is not supported."); + + } + + @Override + public ColumnCapabilities getDimCapabilities() + { + return capabilities; } }; } @@ -448,6 +540,20 @@ public int lookupId(String name) @Override public FloatColumnSelector makeFloatColumnSelector(String columnName) { + final Integer dimIndex = index.getDimensionIndex(columnName); + if (dimIndex != null) { + return new FloatColumnSelector() + { + @Override + public float get() + { + Comparable[][] rowVals = currEntry.getKey().getDims(); + Comparable[] dimVals = rowVals[dimIndex]; + return dimVals == null ? 0L : (Float) dimVals[0]; + } + }; + } + final Integer metricIndexInt = index.getMetricIndex(columnName); if (metricIndexInt == null) { return new FloatColumnSelector() @@ -484,6 +590,21 @@ public long get() } }; } + + final Integer dimIndex = index.getDimensionIndex(columnName); + if (dimIndex != null) { + return new LongColumnSelector() + { + @Override + public long get() + { + Comparable[][] rowVals = currEntry.getKey().getDims(); + Comparable[] dimVals = rowVals[dimIndex]; + return dimVals == null ? 0L : (Long) dimVals[0]; + } + }; + } + final Integer metricIndexInt = index.getMetricIndex(columnName); if (metricIndexInt == null) { return new LongColumnSelector() @@ -560,7 +681,8 @@ public Object get() if (dimensionDesc != null) { final int dimensionIndex = dimensionDesc.getIndex(); - final IncrementalIndex.DimDim dimDim = dimensionDesc.getValues(); + final StringDimensionHandler.StringDimensionIndexer indexer = (StringDimensionHandler.StringDimensionIndexer) dimensionDesc.getIndexer(); + final ColumnCapabilities capabilities = dimensionDesc.getCapabilities(); return new ObjectColumnSelector() { @@ -578,21 +700,21 @@ public Object get() return null; } - int[][] dims = key.getDims(); + Comparable[][] dims = key.getDims(); if (dimensionIndex >= dims.length) { return null; } - final int[] dimIdx = dims[dimensionIndex]; + final Comparable[] dimIdx = dims[dimensionIndex]; if (dimIdx == null || dimIdx.length == 0) { return null; } if (dimIdx.length == 1) { - return dimDim.getValue(dimIdx[0]); + return indexer.getActualValue((Integer) dimIdx[0], false); } Comparable[] dimVals = new String[dimIdx.length]; for (int i = 0; i < dimIdx.length; i++) { - dimVals[i] = dimDim.getValue(dimIdx[i]); + dimVals[i] = indexer.getActualValue((Integer) dimIdx[i], false); } return dimVals; } @@ -659,24 +781,53 @@ public EntryHolderValueMatcherFactory( } @Override - public ValueMatcher makeValueMatcher(String dimension, final Comparable value) + public ValueMatcher makeValueMatcher(String dimension, final Comparable originalValue) { IncrementalIndex.DimensionDesc dimensionDesc = index.getDimension(dimension); if (dimensionDesc == null) { - return new BooleanValueMatcher(isComparableNullOrEmpty(value)); + return new BooleanValueMatcher(isComparableNullOrEmpty(originalValue)); } + + final DimensionHandler handler = dimensionDesc.getHandler(); + final DimensionIndexer indexer = dimensionDesc.getIndexer(); + final int dimIndex = dimensionDesc.getIndex(); - final IncrementalIndex.DimDim dimDim = dimensionDesc.getValues(); + final ColumnCapabilities capabilities = index.getCapabilities(dimension); + final Comparable value = (Comparable) handler.getValueTypeTransformer().apply(originalValue); + final Comparator encodedComparator = handler.getEncodedComparator(); + + if (!capabilities.isDictionaryEncoded()) { + return new ValueMatcher() + { + @Override + public boolean matches() + { + Comparable[][] dims = holder.getKey().getDims(); + Comparable[] dimVals = dims[dimIndex]; + if (dimIndex >= dims.length || dimVals == null) { + return value == null; + } + + for (Comparable dimVal : dimVals) { + if (encodedComparator.compare(dimVal, value) == 0) { + return true; + } + } + + return false; + } + }; + } - final Integer id = dimDim.getId(value); - if (id == null) { + final Comparable encodedVal = indexer.getEncodedValue(value, false); + if (encodedVal == null) { if (isComparableNullOrEmpty(value)) { return new ValueMatcher() { @Override public boolean matches() { - int[][] dims = holder.getKey().getDims(); + Comparable[][] dims = holder.getKey().getDims(); if (dimIndex >= dims.length || dims[dimIndex] == null) { return true; } @@ -692,12 +843,16 @@ public boolean matches() @Override public boolean matches() { - int[][] dims = holder.getKey().getDims(); + Comparable[][] dims = holder.getKey().getDims(); if (dimIndex >= dims.length || dims[dimIndex] == null) { return isComparableNullOrEmpty(value); } - - return Ints.indexOf(dims[dimIndex], id) >= 0; + for (Comparable dimVal : dims[dimIndex]) { + if (encodedComparator.compare(dimVal, encodedVal) == 0) { + return true; + } + } + return false; } }; } @@ -709,21 +864,26 @@ public ValueMatcher makeValueMatcher(String dimension, final Predicate predicate if (dimensionDesc == null) { return new BooleanValueMatcher(false); } + + final DimensionHandler handler = dimensionDesc.getHandler(); + final DimensionIndexer indexer = dimensionDesc.getIndexer(); + final int dimIndex = dimensionDesc.getIndex(); - final IncrementalIndex.DimDim dimDim = dimensionDesc.getValues(); + final ColumnCapabilities capabilities = dimensionDesc.getCapabilities(); return new ValueMatcher() { @Override public boolean matches() { - int[][] dims = holder.getKey().getDims(); + Comparable[][] dims = holder.getKey().getDims(); if (dimIndex >= dims.length || dims[dimIndex] == null) { return predicate.apply(null); } - for (int dimVal : dims[dimIndex]) { - if (predicate.apply(dimDim.getValue(dimVal))) { + for (Comparable dimVal : dims[dimIndex]) { + Comparable finalDimVal = indexer.getActualValue(dimVal, false); + if (predicate.apply(finalDimVal)) { return true; } } diff --git a/processing/src/main/java/io/druid/segment/incremental/OffheapIncrementalIndex.java b/processing/src/main/java/io/druid/segment/incremental/OffheapIncrementalIndex.java index 35fe24260244..867d75bb1ba0 100644 --- a/processing/src/main/java/io/druid/segment/incremental/OffheapIncrementalIndex.java +++ b/processing/src/main/java/io/druid/segment/incremental/OffheapIncrementalIndex.java @@ -80,7 +80,7 @@ public OffheapIncrementalIndex( StupidPool bufferPool ) { - super(incrementalIndexSchema, deserializeComplexMetrics, reportParseExceptions, sortFacts); + super(incrementalIndexSchema, deserializeComplexMetrics, reportParseExceptions, sortFacts, false); this.maxRowCount = maxRowCount; this.bufferPool = bufferPool; @@ -155,12 +155,6 @@ public ConcurrentMap getFacts() return facts; } - @Override - protected DimDim makeDimDim(String dimension, Object lock) - { - return new OnheapIncrementalIndex.OnHeapDimDim(lock); - } - @Override protected BufferAggregator[] initAggs( AggregatorFactory[] metrics, Supplier rowSupplier, boolean deserializeComplexMetrics @@ -175,7 +169,8 @@ protected BufferAggregator[] initAggs( ColumnSelectorFactory columnSelectorFactory = makeColumnSelectorFactory( agg, rowSupplier, - deserializeComplexMetrics + deserializeComplexMetrics, + getColumnCapabilities() ); selectors.put( @@ -226,7 +221,7 @@ protected Integer addToFacts( for (int i = 0; i < metrics.length; i++) { final AggregatorFactory agg = metrics[i]; getAggs()[i] = agg.factorizeBuffered( - makeColumnSelectorFactory(agg, rowSupplier, deserializeComplexMetrics) + makeColumnSelectorFactory(agg, rowSupplier, deserializeComplexMetrics, getColumnCapabilities()) ); } rowContainer.set(null); diff --git a/processing/src/main/java/io/druid/segment/incremental/OnheapIncrementalIndex.java b/processing/src/main/java/io/druid/segment/incremental/OnheapIncrementalIndex.java index 14ffffed2e73..f35f48065738 100644 --- a/processing/src/main/java/io/druid/segment/incremental/OnheapIncrementalIndex.java +++ b/processing/src/main/java/io/druid/segment/incremental/OnheapIncrementalIndex.java @@ -61,10 +61,11 @@ public OnheapIncrementalIndex( boolean deserializeComplexMetrics, boolean reportParseExceptions, boolean sortFacts, + boolean allowTypeDiscovery, int maxRowCount ) { - super(incrementalIndexSchema, deserializeComplexMetrics, reportParseExceptions, sortFacts); + super(incrementalIndexSchema, deserializeComplexMetrics, reportParseExceptions, sortFacts, allowTypeDiscovery); this.maxRowCount = maxRowCount; if (sortFacts) { @@ -81,6 +82,7 @@ public OnheapIncrementalIndex( boolean deserializeComplexMetrics, boolean reportParseExceptions, boolean sortFacts, + boolean allowTypeDiscovery, int maxRowCount ) { @@ -92,6 +94,30 @@ public OnheapIncrementalIndex( deserializeComplexMetrics, reportParseExceptions, sortFacts, + allowTypeDiscovery, + maxRowCount + ); + } + + public OnheapIncrementalIndex( + long minTimestamp, + QueryGranularity gran, + final AggregatorFactory[] metrics, + boolean deserializeComplexMetrics, + boolean reportParseExceptions, + boolean sortFacts, + int maxRowCount + ) + { + this( + new IncrementalIndexSchema.Builder().withMinTimestamp(minTimestamp) + .withQueryGranularity(gran) + .withMetrics(metrics) + .build(), + deserializeComplexMetrics, + reportParseExceptions, + sortFacts, + false, maxRowCount ); } @@ -111,6 +137,7 @@ public OnheapIncrementalIndex( true, true, true, + false, maxRowCount ); } @@ -121,7 +148,7 @@ public OnheapIncrementalIndex( int maxRowCount ) { - this(incrementalIndexSchema, true, reportParseExceptions, true, maxRowCount); + this(incrementalIndexSchema, true, reportParseExceptions, true, false, maxRowCount); } @Override @@ -130,12 +157,6 @@ public ConcurrentMap getFacts() return facts; } - @Override - protected DimDim makeDimDim(String dimension, Object lock) - { - return new OnHeapDimDim(lock); - } - @Override protected Aggregator[] initAggs( AggregatorFactory[] metrics, Supplier rowSupplier, boolean deserializeComplexMetrics @@ -145,7 +166,7 @@ protected Aggregator[] initAggs( for (AggregatorFactory agg : metrics) { selectors.put( agg.getName(), - new ObjectCachingColumnSelectorFactory(makeColumnSelectorFactory(agg, rowSupplier, deserializeComplexMetrics)) + new ObjectCachingColumnSelectorFactory(makeColumnSelectorFactory(agg, rowSupplier, deserializeComplexMetrics, getColumnCapabilities())) ); } @@ -303,133 +324,6 @@ public void close() } } - static class OnHeapDimDim> implements DimDim - { - private final Map valueToId = Maps.newHashMap(); - private T minValue = null; - private T maxValue = null; - - private final List idToValue = Lists.newArrayList(); - private final Object lock; - - public OnHeapDimDim(Object lock) - { - this.lock = lock; - } - - public int getId(T value) - { - synchronized (lock) { - final Integer id = valueToId.get(value); - return id == null ? -1 : id; - } - } - - public T getValue(int id) - { - synchronized (lock) { - return idToValue.get(id); - } - } - - public boolean contains(T value) - { - synchronized (lock) { - return valueToId.containsKey(value); - } - } - - public int size() - { - synchronized (lock) { - return valueToId.size(); - } - } - - public int add(T value) - { - synchronized (lock) { - Integer prev = valueToId.get(value); - if (prev != null) { - return prev; - } - final int index = size(); - valueToId.put(value, index); - idToValue.add(value); - minValue = minValue == null || minValue.compareTo(value) > 0 ? value : minValue; - maxValue = maxValue == null || maxValue.compareTo(value) < 0 ? value : maxValue; - return index; - } - } - - @Override - public T getMinValue() - { - return minValue; - } - - @Override - public T getMaxValue() - { - return maxValue; - } - - public OnHeapDimLookup sort() - { - synchronized (lock) { - return new OnHeapDimLookup(idToValue, size()); - } - } - } - - static class OnHeapDimLookup> implements SortedDimLookup - { - private final List sortedVals; - private final int[] idToIndex; - private final int[] indexToId; - - public OnHeapDimLookup(List idToValue, int length) - { - Map sortedMap = Maps.newTreeMap(); - for (int id = 0; id < length; id++) { - sortedMap.put(idToValue.get(id), id); - } - this.sortedVals = Lists.newArrayList(sortedMap.keySet()); - this.idToIndex = new int[length]; - this.indexToId = new int[length]; - int index = 0; - for (Integer id : sortedMap.values()) { - idToIndex[id] = index; - indexToId[index] = id; - index++; - } - } - - @Override - public int size() - { - return sortedVals.size(); - } - - @Override - public int getUnsortedIdFromSortedId(int index) - { - return indexToId[index]; - } - - @Override - public T getValueFromSortedId(int index) - { - return sortedVals.get(index); - } - - @Override - public int getSortedIdFromUnsortedId(int id) - { - return idToIndex[id]; - } - } - // Caches references to selector objects for each column instead of creating a new object each time in order to save heap space. // In general the selectorFactory need not to thread-safe. // here its made thread safe to support the special case of groupBy where the multiple threads can add concurrently to the IncrementalIndex. @@ -451,6 +345,12 @@ public DimensionSelector makeDimensionSelector(DimensionSpec dimensionSpec) return delegate.makeDimensionSelector(dimensionSpec); } + @Override + public DimensionSelector makeDictEncodedStringDimensionSelector(DimensionSpec dimensionSpec) + { + throw new UnsupportedOperationException("makeDictEncodedStringDimensionSelector() unsupported"); + } + @Override public FloatColumnSelector makeFloatColumnSelector(String columnName) { diff --git a/processing/src/main/java/io/druid/segment/serde/ComplexMetricColumnSerializer.java b/processing/src/main/java/io/druid/segment/serde/ComplexMetricColumnSerializer.java index d9fe6678d575..824ac8f85152 100644 --- a/processing/src/main/java/io/druid/segment/serde/ComplexMetricColumnSerializer.java +++ b/processing/src/main/java/io/druid/segment/serde/ComplexMetricColumnSerializer.java @@ -73,9 +73,14 @@ public void serialize(Object agg) throws IOException @Override public void close() throws IOException { - writer.close(); - final File outFile = IndexIO.makeMetricFile(outDir, metricName, IndexIO.BYTE_ORDER); + closeFile(outFile); + } + + @Override + public void closeFile(File outFile) throws IOException + { + writer.close(); outFile.delete(); MetricHolder.writeComplexMetric( Files.newOutputStreamSupplier(outFile, true), metricName, serde.getTypeName(), writer diff --git a/processing/src/test/java/io/druid/query/QueryRunnerTestHelper.java b/processing/src/test/java/io/druid/query/QueryRunnerTestHelper.java index 53f0d2c7a9da..596a9a4f40ab 100644 --- a/processing/src/test/java/io/druid/query/QueryRunnerTestHelper.java +++ b/processing/src/test/java/io/druid/query/QueryRunnerTestHelper.java @@ -176,7 +176,10 @@ public TableDataSource apply(@Nullable String input) public static ArithmeticPostAggregator hyperUniqueFinalizingPostAgg = new ArithmeticPostAggregator( hyperUniqueFinalizingPostAggMetric, "+", - Lists.newArrayList(new HyperUniqueFinalizingPostAggregator(uniqueMetric, uniqueMetric), new ConstantPostAggregator(null, 1)) + Lists.newArrayList( + new HyperUniqueFinalizingPostAggregator(uniqueMetric, uniqueMetric), + new ConstantPostAggregator(null, 1) + ) ); public static final List commonAggregators = Arrays.asList( @@ -433,7 +436,7 @@ public static > QueryRunner makeQueryRunner( segmentId, adapter.getDataInterval().getStart(), factory.createRunner(adapter) ), - (QueryToolChest>)factory.getToolchest() + (QueryToolChest>) factory.getToolchest() ); } @@ -501,11 +504,16 @@ public Sequence run(Query query, Map responseContext) public static IntervalChunkingQueryRunnerDecorator NoopIntervalChunkingQueryRunnerDecorator() { - return new IntervalChunkingQueryRunnerDecorator(null, null, null) { + return new IntervalChunkingQueryRunnerDecorator(null, null, null) + { @Override - public QueryRunner decorate(final QueryRunner delegate, - QueryToolChest> toolChest) { - return new QueryRunner() { + public QueryRunner decorate( + final QueryRunner delegate, + QueryToolChest> toolChest + ) + { + return new QueryRunner() + { @Override public Sequence run(Query query, Map responseContext) { diff --git a/processing/src/test/java/io/druid/query/aggregation/FilteredAggregatorTest.java b/processing/src/test/java/io/druid/query/aggregation/FilteredAggregatorTest.java index 7f41e2be25c4..f8c5fadd58a0 100644 --- a/processing/src/test/java/io/druid/query/aggregation/FilteredAggregatorTest.java +++ b/processing/src/test/java/io/druid/query/aggregation/FilteredAggregatorTest.java @@ -32,13 +32,24 @@ import io.druid.segment.FloatColumnSelector; import io.druid.segment.LongColumnSelector; import io.druid.segment.ObjectColumnSelector; +import io.druid.segment.column.ColumnCapabilities; +import io.druid.segment.column.ColumnCapabilitiesImpl; +import io.druid.segment.column.ValueType; import io.druid.segment.data.ArrayBasedIndexedInts; +import io.druid.segment.data.IndexedFloats; import io.druid.segment.data.IndexedInts; +import io.druid.segment.data.IndexedLongs; import org.junit.Assert; import org.junit.Test; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + public class FilteredAggregatorTest { + private static final List VALID_DIMENSIONS = Lists.newArrayList("dim", "dim_long", "dim_float"); + private void aggregate(TestFloatColumnSelector selector, FilteredAggregator agg) { agg.aggregate(); @@ -79,7 +90,7 @@ public DimensionSelector makeDimensionSelector(DimensionSpec dimensionSpec) final String dimensionName = dimensionSpec.getDimension(); final ExtractionFn extractionFn = dimensionSpec.getExtractionFn(); - if (dimensionName.equals("dim")) { + if (VALID_DIMENSIONS.contains(dimensionName)) { return dimensionSpec.decorate( new DimensionSelector() { @@ -124,6 +135,141 @@ public int lookupId(String name) throw new IllegalArgumentException(); } } + + @Override + public IndexedLongs getLongRow() + { + final long rowVal; + if (selector.getIndex() % 3 == 2) { + rowVal = 9001L; + } else { + rowVal = 255L; + } + return new IndexedLongs() + { + @Override + public int size() + { + return 1; + } + + @Override + public long get(int index) + { + return rowVal; + } + + @Override + public void fill(int index, long[] toFill) + { + + } + + @Override + public int binarySearch(long key) + { + return 0; + } + + @Override + public int binarySearch(long key, int from, int to) + { + return 0; + } + + @Override + public void close() throws IOException + { + + } + }; + } + + @Override + public Comparable getExtractedValueLong(long val) + { + return val; + } + + @Override + public IndexedFloats getFloatRow() + { + final float rowVal; + if (selector.getIndex() % 3 == 2) { + rowVal = 55000.0f; + } else { + rowVal = 140.0f; + } + return new IndexedFloats() + { + @Override + public int size() + { + return 1; + } + + @Override + public float get(int index) + { + return rowVal; + } + + @Override + public void fill(int index, float[] toFill) + { + + } + + @Override + public void close() throws IOException + { + + } + }; + } + + @Override + public Comparable getExtractedValueFloat(float val) + { + return val; + } + + @Override + public Comparable getComparableRow() + { + throw new UnsupportedOperationException("getComparableRow() is not supported."); + } + + @Override + public Comparable getExtractedValueComparable(Comparable val) + { + throw new UnsupportedOperationException("getExtractedValueComparable() is not supported."); + + } + + @Override + public ColumnCapabilities getDimCapabilities() + { + ColumnCapabilitiesImpl capabilities = new ColumnCapabilitiesImpl(); + if (dimensionName.equals("dim_float")) { + capabilities.setHasBitmapIndexes(false); + capabilities.setDictionaryEncoded(false); + capabilities.setType(ValueType.FLOAT); + } + if (dimensionName.equals("dim_long")) { + capabilities.setHasBitmapIndexes(false); + capabilities.setDictionaryEncoded(false); + capabilities.setType(ValueType.LONG); + } + if (dimensionName.equals("dim")) { + capabilities.setHasBitmapIndexes(true); + capabilities.setDictionaryEncoded(true); + capabilities.setType(ValueType.STRING); + } + + + return capabilities; + } } ); } else { @@ -131,6 +277,12 @@ public int lookupId(String name) } } + @Override + public DimensionSelector makeDictEncodedStringDimensionSelector(DimensionSpec dimensionSpec) + { + return makeDimensionSelector(dimensionSpec); + } + @Override public LongColumnSelector makeLongColumnSelector(String columnName) { @@ -235,4 +387,32 @@ public void testAggregateWithAndFilter() assertValues(agg, selector, expectedFirst, expectedSecond, expectedThird); } + @Test + public void testAggregateWithLongAndFloatFilter() + { + final float[] values = {0.15f, 0.27f}; + final TestFloatColumnSelector selector = new TestFloatColumnSelector(values); + + FilteredAggregatorFactory factory = new FilteredAggregatorFactory( + new DoubleSumAggregatorFactory("billy", "value"), + new AndDimFilter(Lists.newArrayList( + new NotDimFilter(new SelectorDimFilter("dim_long", "9001")), + new NotDimFilter(new SelectorDimFilter("dim_float", "55000.0")), + new SelectorDimFilter("dim_long", "255"), + new SelectorDimFilter("dim_float", "140.0") + )) + ); + + FilteredAggregator agg = (FilteredAggregator) factory.factorize( + makeColumnSelector(selector) + ); + + Assert.assertEquals("billy", agg.getName()); + + double expectedFirst = new Float(values[0]).doubleValue(); + double expectedSecond = new Float(values[1]).doubleValue() + expectedFirst; + double expectedThird = expectedSecond; + assertValues(agg, selector, expectedFirst, expectedSecond, expectedThird); + } + } diff --git a/processing/src/test/java/io/druid/query/aggregation/cardinality/CardinalityAggregatorBenchmark.java b/processing/src/test/java/io/druid/query/aggregation/cardinality/CardinalityAggregatorBenchmark.java index e42d301a750c..f5d72d50a1e4 100644 --- a/processing/src/test/java/io/druid/query/aggregation/cardinality/CardinalityAggregatorBenchmark.java +++ b/processing/src/test/java/io/druid/query/aggregation/cardinality/CardinalityAggregatorBenchmark.java @@ -29,8 +29,10 @@ import com.google.common.collect.Lists; import com.google.common.collect.Range; import io.druid.segment.DimensionSelector; +import io.druid.segment.column.ValueType; import java.nio.ByteBuffer; +import java.util.Arrays; import java.util.List; public class CardinalityAggregatorBenchmark extends SimpleBenchmark @@ -75,7 +77,7 @@ public String[] apply(Integer input) final CardinalityAggregatorTest.TestDimensionSelector dim1 = - new CardinalityAggregatorTest.TestDimensionSelector(values); + new CardinalityAggregatorTest.TestDimensionSelector(ValueType.STRING, Arrays.asList(values)); selectorList = Lists.newArrayList( (DimensionSelector) dim1 diff --git a/processing/src/test/java/io/druid/query/aggregation/cardinality/CardinalityAggregatorTest.java b/processing/src/test/java/io/druid/query/aggregation/cardinality/CardinalityAggregatorTest.java index 0a4c2eb42dd5..5d8e4cc53a76 100644 --- a/processing/src/test/java/io/druid/query/aggregation/cardinality/CardinalityAggregatorTest.java +++ b/processing/src/test/java/io/druid/query/aggregation/cardinality/CardinalityAggregatorTest.java @@ -31,7 +31,12 @@ import io.druid.query.aggregation.AggregatorFactory; import io.druid.query.aggregation.BufferAggregator; import io.druid.segment.DimensionSelector; +import io.druid.segment.column.ColumnCapabilities; +import io.druid.segment.column.ColumnCapabilitiesImpl; +import io.druid.segment.column.ValueType; +import io.druid.segment.data.IndexedFloats; import io.druid.segment.data.IndexedInts; +import io.druid.segment.data.IndexedLongs; import org.junit.Assert; import org.junit.Test; @@ -47,52 +52,94 @@ public class CardinalityAggregatorTest public static class TestDimensionSelector implements DimensionSelector { private final List column; + private final List unencodedColumn; private final Map ids; private final Map lookup; + private final ValueType type; + private final ColumnCapabilitiesImpl capabilities; + List floatColumn; + List longColumn; private int pos = 0; - public TestDimensionSelector(Iterable values) + public TestDimensionSelector(ValueType type, List values) { this.lookup = Maps.newHashMap(); this.ids = Maps.newHashMap(); - - int index = 0; - for (String[] multiValue : values) { - for (String value : multiValue) { - if (!ids.containsKey(value)) { - ids.put(value, index); - lookup.put(index, value); - index++; + this.capabilities = new ColumnCapabilitiesImpl(); + this.type = type; + + capabilities.setHasBitmapIndexes(type == ValueType.STRING); + capabilities.setDictionaryEncoded(type == ValueType.STRING); + capabilities.setType(type); + + if(type == ValueType.STRING) { + this.unencodedColumn = null; + Iterable strValues = (Iterable) values; + int index = 0; + for (String[] multiValue : strValues) { + for (String value : multiValue) { + if (!ids.containsKey(value)) { + ids.put(value, index); + lookup.put(index, value); + index++; + } } } - } - this.column = Lists.newArrayList( - Iterables.transform( - values, new Function() - { - @Nullable - @Override - public Integer[] apply(@Nullable String[] input) + this.column = Lists.newArrayList( + Iterables.transform( + strValues, new Function() { - return Iterators.toArray( - Iterators.transform( - Iterators.forArray(input), new Function() - { - @Nullable - @Override - public Integer apply(@Nullable String input) + @Nullable + @Override + public Integer[] apply(@Nullable String[] input) + { + return Iterators.toArray( + Iterators.transform( + Iterators.forArray(input), new Function() { - return ids.get(input); + @Nullable + @Override + public Integer apply(@Nullable String input) + { + return ids.get(input); + } } - } - ), Integer.class - ); + ), Integer.class + ); + } } - } - ) - ); + ) + ); + } else { + this.column = null; + this.unencodedColumn = Lists.newArrayList(values); + this.longColumn = Lists.newArrayList(); + this.floatColumn = Lists.newArrayList(); + + if (type == ValueType.LONG) { + for (Object value : values) { + Comparable[] valArr = (Comparable[]) value; + long[] longs = new long[valArr.length]; + for (int i = 0; i < valArr.length; i++) { + longs[i] = (Long) valArr[i]; + } + longColumn.add(longs); + } + } + + if (type == ValueType.FLOAT) { + for (Object value : values) { + Comparable[] valArr = (Comparable[]) value; + float[] floats = new float[valArr.length]; + for (int i = 0; i < valArr.length; i++) { + floats[i] = (Float) valArr[i]; + } + floatColumn.add(floats); + } + } + } } public void increment() @@ -160,6 +207,112 @@ public int lookupId(String s) { return ids.get(s); } + + @Override + public IndexedLongs getLongRow() + { + final long[] longVals = longColumn.get(pos); + return new IndexedLongs() + { + @Override + public int size() + { + return longVals.length; + } + + @Override + public long get(int index) + { + return longVals[index]; + } + + @Override + public void fill(int index, long[] toFill) + { + + } + + @Override + public int binarySearch(long key) + { + return 0; + } + + @Override + public int binarySearch(long key, int from, int to) + { + return 0; + } + + @Override + public void close() throws IOException + { + + } + }; + } + + @Override + public Comparable getExtractedValueLong(long val) + { + return val; + } + + @Override + public IndexedFloats getFloatRow() + { + final float[] floatVals = floatColumn.get(pos); + return new IndexedFloats() + { + @Override + public int size() + { + return floatVals.length; + } + + @Override + public float get(int index) + { + return floatVals[index]; + } + + @Override + public void fill(int index, float[] toFill) + { + + } + + @Override + public void close() throws IOException + { + + } + }; + } + + @Override + public Comparable getExtractedValueFloat(float val) + { + return val; + } + + @Override + public Comparable getComparableRow() + { + throw new UnsupportedOperationException("getComparableRow() is not supported."); + } + + @Override + public Comparable getExtractedValueComparable(Comparable val) + { + return val; + } + + @Override + public ColumnCapabilities getDimCapabilities() + { + return capabilities; + } } /* @@ -167,13 +320,32 @@ public int lookupId(String s) values1: 4 distinct values values2: 8 distinct rows values2: 7 distinct values + valuesLong: 11 distinct rows + valuesLong: 11 distinct values + valuesFloat : 11 distinct rows + valuesFloat : 11 distinct values groupBy(values1, values2): 9 distinct rows groupBy(values1, values2): 7 distinct values + groupBy(valuesLong, valuesFloat): 22 distinct rows + groupBy(valuesLong, valuesFloat): 22 distinct values combine(values1, values2): 8 distinct rows combine(values1, values2): 7 distinct values + combine(valuesLong, valuesFloat): 22 distinct rows + combine(valuesLong, valuesFloat): 22 distinct values */ private static final List values1 = dimensionValues( - "a", "b", "c", "a", "a", null, "b", "b", "b", "b", "a", "a" + "a", + "b", + "c", + "a", + "a", + null, + "b", + "b", + "b", + "b", + "a", + "a" ); private static final List values2 = dimensionValues( "a", @@ -190,6 +362,37 @@ public int lookupId(String s) new String[]{"x", "y", "a"} ); + private static final List valuesLong = Lists.newArrayList( + new Comparable[]{100L}, + new Comparable[]{200L}, + new Comparable[]{300L}, + new Comparable[]{400L}, + new Comparable[]{500L}, + new Comparable[]{600L}, + new Comparable[]{200L}, // second row is identical to seventh row + new Comparable[]{800L}, + new Comparable[]{900L}, + new Comparable[]{1000L}, + new Comparable[]{1100L}, + new Comparable[]{1200L} + ); + + private static final List valuesFloat = Lists.newArrayList( + new Comparable[]{111.0f}, + new Comparable[]{222.0f}, + new Comparable[]{333.0f}, + new Comparable[]{444.0f}, + new Comparable[]{555.0f}, + new Comparable[]{666.0f}, + new Comparable[]{222.0f}, + new Comparable[]{888.0f}, + new Comparable[]{999.0f}, + new Comparable[]{101010.0f}, + new Comparable[]{111111.0f}, + new Comparable[]{121212.0f} + ); + + private static List dimensionValues(Object... values) { return Lists.transform( @@ -237,26 +440,32 @@ private static void bufferAggregate( CardinalityAggregatorFactory valueAggregatorFactory; final TestDimensionSelector dim1; final TestDimensionSelector dim2; + final TestDimensionSelector dimLong; + final TestDimensionSelector dimFloat; public CardinalityAggregatorTest() { - dim1 = new TestDimensionSelector(values1); - dim2 = new TestDimensionSelector(values2); + dim1 = new TestDimensionSelector(ValueType.STRING, values1); + dim2 = new TestDimensionSelector(ValueType.STRING, values2); + dimLong = new TestDimensionSelector(ValueType.LONG, valuesLong); + dimFloat = new TestDimensionSelector(ValueType.FLOAT, valuesFloat); selectorList = Lists.newArrayList( (DimensionSelector) dim1, - dim2 + dim2, + dimLong, + dimFloat ); rowAggregatorFactory = new CardinalityAggregatorFactory( "billy", - Lists.newArrayList("dim1", "dim2"), + Lists.newArrayList("dim1", "dim2", "dimLong", "dimFloat"), true ); valueAggregatorFactory = new CardinalityAggregatorFactory( "billy", - Lists.newArrayList("dim1", "dim2"), + Lists.newArrayList("dim1", "dim2", "dimLong", "dimFloat"), true ); } @@ -274,7 +483,7 @@ public void testAggregateRows() throws Exception for (int i = 0; i < values1.size(); ++i) { aggregate(selectorList, agg); } - Assert.assertEquals(9.0, (Double) rowAggregatorFactory.finalizeComputation(agg.get()), 0.05); + Assert.assertEquals(11.0, (Double) rowAggregatorFactory.finalizeComputation(agg.get()), 0.05); } @Test @@ -289,7 +498,7 @@ public void testAggregateValues() throws Exception for (int i = 0; i < values1.size(); ++i) { aggregate(selectorList, agg); } - Assert.assertEquals(7.0, (Double) valueAggregatorFactory.finalizeComputation(agg.get()), 0.05); + Assert.assertEquals(29.0, (Double) valueAggregatorFactory.finalizeComputation(agg.get()), 0.25); } @Test @@ -310,7 +519,7 @@ public void testBufferAggregateRows() throws Exception for (int i = 0; i < values1.size(); ++i) { bufferAggregate(selectorList, agg, buf, pos); } - Assert.assertEquals(9.0, (Double) rowAggregatorFactory.finalizeComputation(agg.get(buf, pos)), 0.05); + Assert.assertEquals(11.0, (Double) rowAggregatorFactory.finalizeComputation(agg.get(buf, pos)), 0.05); } @Test @@ -331,7 +540,7 @@ public void testBufferAggregateValues() throws Exception for (int i = 0; i < values1.size(); ++i) { bufferAggregate(selectorList, agg, buf, pos); } - Assert.assertEquals(7.0, (Double) valueAggregatorFactory.finalizeComputation(agg.get(buf, pos)), 0.05); + Assert.assertEquals(29.0, (Double) valueAggregatorFactory.finalizeComputation(agg.get(buf, pos)), 0.25); } @Test @@ -339,9 +548,13 @@ public void testCombineRows() { List selector1 = Lists.newArrayList((DimensionSelector) dim1); List selector2 = Lists.newArrayList((DimensionSelector) dim2); + List selectorLong = Lists.newArrayList((DimensionSelector) dimLong); + List selectorFloat = Lists.newArrayList((DimensionSelector) dimFloat); CardinalityAggregator agg1 = new CardinalityAggregator("billy", selector1, true); CardinalityAggregator agg2 = new CardinalityAggregator("billy", selector2, true); + CardinalityAggregator aggLong = new CardinalityAggregator("billy", selectorLong, true); + CardinalityAggregator aggFloat = new CardinalityAggregator("billy", selectorFloat, true); for (int i = 0; i < values1.size(); ++i) { aggregate(selector1, agg1); @@ -349,9 +562,18 @@ public void testCombineRows() for (int i = 0; i < values2.size(); ++i) { aggregate(selector2, agg2); } + for (int i = 0; i < valuesLong.size(); ++i) { + aggregate(selectorLong, aggLong); + } + for (int i = 0; i < valuesFloat.size(); ++i) { + aggregate(selectorFloat, aggFloat); + } Assert.assertEquals(4.0, (Double) rowAggregatorFactory.finalizeComputation(agg1.get()), 0.05); Assert.assertEquals(8.0, (Double) rowAggregatorFactory.finalizeComputation(agg2.get()), 0.05); + Assert.assertEquals(11.0, (Double) rowAggregatorFactory.finalizeComputation(aggLong.get()), 0.05); + Assert.assertEquals(11.0, (Double) rowAggregatorFactory.finalizeComputation(aggFloat.get()), 0.05); + Assert.assertEquals( 9.0, @@ -363,6 +585,34 @@ public void testCombineRows() ), 0.05 ); + + Assert.assertEquals( + 22.0, + (Double) rowAggregatorFactory.finalizeComputation( + rowAggregatorFactory.combine( + aggLong.get(), + aggFloat.get() + ) + ), + 0.15 + ); + + Assert.assertEquals( + 30.0, + (Double) valueAggregatorFactory.finalizeComputation( + valueAggregatorFactory.combine( + valueAggregatorFactory.combine( + agg1.get(), + agg2.get() + ), + valueAggregatorFactory.combine( + aggLong.get(), + aggFloat.get() + ) + ) + ), + 1.25 + ); } @Test @@ -370,9 +620,13 @@ public void testCombineValues() { List selector1 = Lists.newArrayList((DimensionSelector) dim1); List selector2 = Lists.newArrayList((DimensionSelector) dim2); + List selectorLong = Lists.newArrayList((DimensionSelector) dimLong); + List selectorFloat = Lists.newArrayList((DimensionSelector) dimFloat); CardinalityAggregator agg1 = new CardinalityAggregator("billy", selector1, false); CardinalityAggregator agg2 = new CardinalityAggregator("billy", selector2, false); + CardinalityAggregator aggLong = new CardinalityAggregator("billy", selectorLong, true); + CardinalityAggregator aggFloat = new CardinalityAggregator("billy", selectorFloat, true); for (int i = 0; i < values1.size(); ++i) { aggregate(selector1, agg1); @@ -380,20 +634,57 @@ public void testCombineValues() for (int i = 0; i < values2.size(); ++i) { aggregate(selector2, agg2); } + for (int i = 0; i < valuesLong.size(); ++i) { + aggregate(selectorLong, aggLong); + } + for (int i = 0; i < valuesFloat.size(); ++i) { + aggregate(selectorFloat, aggFloat); + } Assert.assertEquals(4.0, (Double) valueAggregatorFactory.finalizeComputation(agg1.get()), 0.05); Assert.assertEquals(7.0, (Double) valueAggregatorFactory.finalizeComputation(agg2.get()), 0.05); + Assert.assertEquals(11.0, (Double) valueAggregatorFactory.finalizeComputation(aggLong.get()), 0.05); + Assert.assertEquals(11.0, (Double) valueAggregatorFactory.finalizeComputation(aggFloat.get()), 0.05); + Assert.assertEquals( 7.0, - (Double) rowAggregatorFactory.finalizeComputation( - rowAggregatorFactory.combine( + (Double) valueAggregatorFactory.finalizeComputation( + valueAggregatorFactory.combine( agg1.get(), agg2.get() ) ), 0.05 ); + + Assert.assertEquals( + 22.0, + (Double) valueAggregatorFactory.finalizeComputation( + valueAggregatorFactory.combine( + aggLong.get(), + aggFloat.get() + ) + ), + 0.15 + ); + + Assert.assertEquals( + 29.0, + (Double) valueAggregatorFactory.finalizeComputation( + valueAggregatorFactory.combine( + valueAggregatorFactory.combine( + agg1.get(), + agg2.get() + ), + valueAggregatorFactory.combine( + aggLong.get(), + aggFloat.get() + ) + ) + ), + 0.25 + ); } @Test diff --git a/processing/src/test/java/io/druid/query/dimension/TestDimensionSelector.java b/processing/src/test/java/io/druid/query/dimension/TestDimensionSelector.java index 89c7d14d7e5e..9f5207847833 100644 --- a/processing/src/test/java/io/druid/query/dimension/TestDimensionSelector.java +++ b/processing/src/test/java/io/druid/query/dimension/TestDimensionSelector.java @@ -20,8 +20,15 @@ package io.druid.query.dimension; import io.druid.segment.DimensionSelector; +import io.druid.segment.column.ColumnCapabilities; +import io.druid.segment.column.ColumnCapabilitiesImpl; +import io.druid.segment.column.ValueType; import io.druid.segment.data.ArrayBasedIndexedInts; +import io.druid.segment.data.IndexedFloats; import io.druid.segment.data.IndexedInts; +import io.druid.segment.data.IndexedLongs; + +import java.util.List; /** * Test dimension selector that has cardinality=26 @@ -61,4 +68,51 @@ public int lookupId(String name) return name.charAt(0) - 'a'; } + @Override + public IndexedLongs getLongRow() + { + throw new UnsupportedOperationException("getLongRow() is not supported."); + } + + @Override + public Comparable getExtractedValueLong(long val) + { + throw new UnsupportedOperationException("getExtractedValueLong() is not supported."); + } + + @Override + public IndexedFloats getFloatRow() + { + throw new UnsupportedOperationException("getFloatRow() is not supported."); + } + + @Override + public Comparable getExtractedValueFloat(float val) + { + throw new UnsupportedOperationException("getExtractedValueFloat() is not supported."); + } + + @Override + public Comparable getComparableRow() + { + throw new UnsupportedOperationException("getComparableRow() is not supported."); + } + + @Override + public Comparable getExtractedValueComparable(Comparable val) + { + throw new UnsupportedOperationException("getExtractedValueComparable() is not supported."); + + } + + @Override + public ColumnCapabilities getDimCapabilities() + { + ColumnCapabilitiesImpl capabilities = new ColumnCapabilitiesImpl(); + capabilities.setDictionaryEncoded(true); + capabilities.setHasBitmapIndexes(true); + capabilities.setType(ValueType.STRING); + return capabilities; + } + } diff --git a/processing/src/test/java/io/druid/query/groupby/GroupByQueryRunnerTest.java b/processing/src/test/java/io/druid/query/groupby/GroupByQueryRunnerTest.java index 0dbdf6d99607..a690c01e731d 100644 --- a/processing/src/test/java/io/druid/query/groupby/GroupByQueryRunnerTest.java +++ b/processing/src/test/java/io/druid/query/groupby/GroupByQueryRunnerTest.java @@ -34,7 +34,15 @@ import com.metamx.common.guava.Sequences; import com.metamx.common.parsers.ParseException; import io.druid.collections.StupidPool; +import io.druid.common.utils.JodaUtils; +import io.druid.data.input.MapBasedInputRow; +import io.druid.data.input.MapBasedRow; import io.druid.data.input.Row; +import io.druid.data.input.impl.DimensionSchema; +import io.druid.data.input.impl.DimensionsSpec; +import io.druid.data.input.impl.FloatDimensionSchema; +import io.druid.data.input.impl.LongDimensionSchema; +import io.druid.data.input.impl.StringDimensionSchema; import io.druid.granularity.PeriodGranularity; import io.druid.granularity.QueryGranularity; import io.druid.jackson.DefaultObjectMapper; @@ -48,7 +56,9 @@ import io.druid.query.QueryToolChest; import io.druid.query.Result; import io.druid.query.TestQueryRunners; +import io.druid.query.aggregation.Aggregator; import io.druid.query.aggregation.AggregatorFactory; +import io.druid.query.aggregation.CountAggregatorFactory; import io.druid.query.aggregation.DoubleMaxAggregatorFactory; import io.druid.query.aggregation.DoubleSumAggregatorFactory; import io.druid.query.aggregation.FilteredAggregatorFactory; @@ -82,9 +92,19 @@ import io.druid.query.groupby.orderby.LimitSpec; import io.druid.query.groupby.orderby.OrderByColumnSpec; import io.druid.query.ordering.StringComparators; +import io.druid.query.select.SelectQueryEngine; +import io.druid.query.select.SelectQueryQueryToolChest; +import io.druid.query.select.SelectQueryRunnerFactory; +import io.druid.query.select.SelectResultValue; import io.druid.query.spec.MultipleIntervalSegmentSpec; +import io.druid.segment.IncrementalIndexSegment; import io.druid.segment.TestHelper; +import io.druid.segment.TestIndex; import io.druid.segment.column.Column; +import io.druid.segment.column.ValueType; +import io.druid.segment.incremental.IncrementalIndex; +import io.druid.segment.incremental.IncrementalIndexSchema; +import io.druid.segment.incremental.OnheapIncrementalIndex; import org.joda.time.DateTime; import org.joda.time.DateTimeZone; import org.joda.time.Interval; @@ -101,6 +121,7 @@ import javax.annotation.Nullable; import java.io.IOException; import java.nio.ByteBuffer; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.Comparator; @@ -117,6 +138,40 @@ public class GroupByQueryRunnerTest private final QueryRunner runner; private GroupByQueryRunnerFactory factory; private Supplier configSupplier; + private ValueType groupingKeyType; + private String qualityName; + private String marketName; + private Map typeHints; + + private static final Map QUALITY_RENAMING_MAP = ImmutableMap.builder() + .put("automotive", "automotive0") + .put("business", "business0") + .put("entertainment", "entertainment0") + .put("health", "health0") + .put("mezzanine", "mezzanine0") + .put("news", "news0") + .put("premium", "premium0") + .put("technology", "technology0") + .put("travel", "travel0") + .put("1111", "automotive0") + .put("2222", "business0") + .put("3333", "entertainment0") + .put("4444", "health0") + .put("5555", "mezzanine0") + .put("6666", "news0") + .put("7777", "premium0") + .put("8888", "technology0") + .put("8899", "travel0") + .put("111.111", "automotive0") + .put("222.222", "business0") + .put("333.333", "entertainment0") + .put("444.444", "health0") + .put("555.555", "mezzanine0") + .put("666.666", "news0") + .put("777.777", "premium0") + .put("888.888", "technology0") + .put("889.999", "travel0") + .build(); @Rule public ExpectedException expectedException = ExpectedException.none(); @@ -183,34 +238,57 @@ public boolean isSingleThreaded() pool ); - - Function function = new Function() + Function> function = new Function>() { @Override - public Object apply(@Nullable Object input) + public List apply(@Nullable Object input) { - return new Object[]{factory, input}; + List params = new ArrayList<>(); + params.add(new Object[]{factory, input, ValueType.STRING}); + params.add(new Object[]{factory, input, ValueType.LONG}); + params.add(new Object[]{factory, input, ValueType.FLOAT}); + return params; } }; return Lists.newArrayList( Iterables.concat( - Iterables.transform( - QueryRunnerTestHelper.makeQueryRunners(factory), - function + Iterables.concat( + Iterables.transform( + QueryRunnerTestHelper.makeQueryRunners(factory), + function + ) ), - Iterables.transform( - QueryRunnerTestHelper.makeQueryRunners(singleThreadFactory), - function + Iterables.concat( + Iterables.transform( + QueryRunnerTestHelper.makeQueryRunners(singleThreadFactory), + function + ) ) ) ); } - public GroupByQueryRunnerTest(GroupByQueryRunnerFactory factory, QueryRunner runner) + private Object getDimMapping(String value) { + return TestIndex.getDimMappingForType(groupingKeyType, value); + } + + private String getDimMappingFirstChar(String value) { + return getDimMapping(value).toString().substring(0,1); + } + + public GroupByQueryRunnerTest(GroupByQueryRunnerFactory factory, QueryRunner runner, ValueType groupingKeyType) { this.factory = factory; this.runner = runner; + this.groupingKeyType = groupingKeyType; + this.qualityName = (String) getDimMapping("quality"); + this.marketName = (String) getDimMapping("market"); + this.typeHints = Maps.newHashMap(); + typeHints.put(qualityName, groupingKeyType); + typeHints.put(marketName, groupingKeyType); + typeHints.put("alias", groupingKeyType); + typeHints.put("marketalias", groupingKeyType); } @Test @@ -220,7 +298,7 @@ public void testGroupBy() .builder() .setDataSource(QueryRunnerTestHelper.dataSource) .setQuerySegmentSpec(QueryRunnerTestHelper.firstToThird) - .setDimensions(Lists.newArrayList(new DefaultDimensionSpec("quality", "alias"))) + .setDimensions(Lists.newArrayList(new DefaultDimensionSpec(qualityName, "alias"))) .setAggregatorSpecs( Arrays.asList( QueryRunnerTestHelper.rowsCount, @@ -228,28 +306,29 @@ public void testGroupBy() ) ) .setGranularity(QueryRunnerTestHelper.dayGran) + .setContext(ImmutableMap.of("typeHints", typeHints)) .build(); List expectedResults = Arrays.asList( - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "automotive", "rows", 1L, "idx", 135L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "business", "rows", 1L, "idx", 118L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "entertainment", "rows", 1L, "idx", 158L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "health", "rows", 1L, "idx", 120L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "mezzanine", "rows", 3L, "idx", 2870L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "news", "rows", 1L, "idx", 121L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "premium", "rows", 3L, "idx", 2900L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "technology", "rows", 1L, "idx", 78L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "travel", "rows", 1L, "idx", 119L), - - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "automotive", "rows", 1L, "idx", 147L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "business", "rows", 1L, "idx", 112L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "entertainment", "rows", 1L, "idx", 166L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "health", "rows", 1L, "idx", 113L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "mezzanine", "rows", 3L, "idx", 2447L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "news", "rows", 1L, "idx", 114L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "premium", "rows", 3L, "idx", 2505L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "technology", "rows", 1L, "idx", 97L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "travel", "rows", 1L, "idx", 126L) + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", getDimMapping("automotive"), "rows", 1L, "idx", 135L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", getDimMapping("business"), "rows", 1L, "idx", 118L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", getDimMapping("entertainment"), "rows", 1L, "idx", 158L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", getDimMapping("health"), "rows", 1L, "idx", 120L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", getDimMapping("mezzanine"), "rows", 3L, "idx", 2870L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", getDimMapping("news"), "rows", 1L, "idx", 121L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", getDimMapping("premium"), "rows", 3L, "idx", 2900L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", getDimMapping("technology"), "rows", 1L, "idx", 78L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", getDimMapping("travel"), "rows", 1L, "idx", 119L), + + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", getDimMapping("automotive"), "rows", 1L, "idx", 147L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", getDimMapping("business"), "rows", 1L, "idx", 112L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", getDimMapping("entertainment"), "rows", 1L, "idx", 166L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", getDimMapping("health"), "rows", 1L, "idx", 113L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", getDimMapping("mezzanine"), "rows", 3L, "idx", 2447L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", getDimMapping("news"), "rows", 1L, "idx", 114L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", getDimMapping("premium"), "rows", 3L, "idx", 2505L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", getDimMapping("technology"), "rows", 1L, "idx", 97L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", getDimMapping("travel"), "rows", 1L, "idx", 126L) ); Iterable results = GroupByQueryRunnerTestHelper.runQuery(factory, runner, query); @@ -281,16 +360,6 @@ public void testGroupByMaxRowsLimitContextOverrid() @Test public void testGroupByWithRebucketRename() { - Map map = new HashMap<>(); - map.put("automotive", "automotive0"); - map.put("business", "business0"); - map.put("entertainment", "entertainment0"); - map.put("health", "health0"); - map.put("mezzanine", "mezzanine0"); - map.put("news", "news0"); - map.put("premium", "premium0"); - map.put("technology", "technology0"); - map.put("travel", "travel0"); GroupByQuery query = GroupByQuery .builder() .setDataSource(QueryRunnerTestHelper.dataSource) @@ -298,7 +367,7 @@ public void testGroupByWithRebucketRename() .setDimensions( Lists.newArrayList( new ExtractionDimensionSpec( - "quality", "alias", new LookupExtractionFn(new MapLookupExtractor(map, false), false, null, false, false), null + qualityName, "alias", new LookupExtractionFn(new MapLookupExtractor(QUALITY_RENAMING_MAP, false), false, null, false, false), null ) ) ) @@ -357,16 +426,6 @@ public void testGroupByWithRebucketRename() @Test public void testGroupByWithSimpleRenameRetainMissingNonInjective() { - Map map = new HashMap<>(); - map.put("automotive", "automotive0"); - map.put("business", "business0"); - map.put("entertainment", "entertainment0"); - map.put("health", "health0"); - map.put("mezzanine", "mezzanine0"); - map.put("news", "news0"); - map.put("premium", "premium0"); - map.put("technology", "technology0"); - map.put("travel", "travel0"); GroupByQuery query = GroupByQuery .builder() .setDataSource(QueryRunnerTestHelper.dataSource) @@ -374,7 +433,7 @@ public void testGroupByWithSimpleRenameRetainMissingNonInjective() .setDimensions( Lists.newArrayList( new ExtractionDimensionSpec( - "quality", "alias", new LookupExtractionFn(new MapLookupExtractor(map, false), true, null, false, false), null + qualityName, "alias", new LookupExtractionFn(new MapLookupExtractor(QUALITY_RENAMING_MAP, false), true, null, false, false), null ) ) ) @@ -433,16 +492,6 @@ public void testGroupByWithSimpleRenameRetainMissingNonInjective() @Test public void testGroupByWithSimpleRenameRetainMissing() { - Map map = new HashMap<>(); - map.put("automotive", "automotive0"); - map.put("business", "business0"); - map.put("entertainment", "entertainment0"); - map.put("health", "health0"); - map.put("mezzanine", "mezzanine0"); - map.put("news", "news0"); - map.put("premium", "premium0"); - map.put("technology", "technology0"); - map.put("travel", "travel0"); GroupByQuery query = GroupByQuery .builder() .setDataSource(QueryRunnerTestHelper.dataSource) @@ -450,7 +499,7 @@ public void testGroupByWithSimpleRenameRetainMissing() .setDimensions( Lists.newArrayList( new ExtractionDimensionSpec( - "quality", "alias", new LookupExtractionFn(new MapLookupExtractor(map, false), true, null, true, false), null + qualityName, "alias", new LookupExtractionFn(new MapLookupExtractor(QUALITY_RENAMING_MAP, false), true, null, true, false), null ) ) ) @@ -509,16 +558,6 @@ public void testGroupByWithSimpleRenameRetainMissing() @Test public void testGroupByWithSimpleRenameAndMissingString() { - Map map = new HashMap<>(); - map.put("automotive", "automotive0"); - map.put("business", "business0"); - map.put("entertainment", "entertainment0"); - map.put("health", "health0"); - map.put("mezzanine", "mezzanine0"); - map.put("news", "news0"); - map.put("premium", "premium0"); - map.put("technology", "technology0"); - map.put("travel", "travel0"); GroupByQuery query = GroupByQuery .builder() .setDataSource(QueryRunnerTestHelper.dataSource) @@ -526,9 +565,9 @@ public void testGroupByWithSimpleRenameAndMissingString() .setDimensions( Lists.newArrayList( new ExtractionDimensionSpec( - "quality", + qualityName, "alias", - new LookupExtractionFn(new MapLookupExtractor(map, false), false, "MISSING", true, false), + new LookupExtractionFn(new MapLookupExtractor(QUALITY_RENAMING_MAP, false), false, "MISSING", true, false), null ) ) @@ -587,16 +626,6 @@ public void testGroupByWithSimpleRenameAndMissingString() @Test public void testGroupByWithSimpleRename() { - Map map = new HashMap<>(); - map.put("automotive", "automotive0"); - map.put("business", "business0"); - map.put("entertainment", "entertainment0"); - map.put("health", "health0"); - map.put("mezzanine", "mezzanine0"); - map.put("news", "news0"); - map.put("premium", "premium0"); - map.put("technology", "technology0"); - map.put("travel", "travel0"); GroupByQuery query = GroupByQuery .builder() .setDataSource(QueryRunnerTestHelper.dataSource) @@ -604,7 +633,7 @@ public void testGroupByWithSimpleRename() .setDimensions( Lists.newArrayList( new ExtractionDimensionSpec( - "quality", "alias", new LookupExtractionFn(new MapLookupExtractor(map, false), false, null, true, false), null + qualityName, "alias", new LookupExtractionFn(new MapLookupExtractor(QUALITY_RENAMING_MAP, false), false, null, true, false), null ) ) ) @@ -771,7 +800,7 @@ public byte[] getCacheKey() @Override public String apply(String dimValue) { - return dimValue.equals("mezzanine") ? null : super.apply(dimValue); + return dimValue.equals(getDimMapping("mezzanine").toString()) ? null : super.apply(dimValue); } }; GroupByQuery query = GroupByQuery @@ -787,29 +816,32 @@ public String apply(String dimValue) .setGranularity(QueryRunnerTestHelper.dayGran) .setDimensions( Lists.newArrayList( - new ExtractionDimensionSpec("quality", "alias", nullExtractionFn, null) + new ExtractionDimensionSpec(qualityName, + "alias", + nullExtractionFn, + null) ) ) .build(); List expectedResults = Arrays.asList( GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", null, "rows", 3L, "idx", 2870L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "a", "rows", 1L, "idx", 135L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "b", "rows", 1L, "idx", 118L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "e", "rows", 1L, "idx", 158L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "h", "rows", 1L, "idx", 120L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "n", "rows", 1L, "idx", 121L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "p", "rows", 3L, "idx", 2900L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "t", "rows", 2L, "idx", 197L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", getDimMappingFirstChar("automotive"), "rows", 1L, "idx", 135L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", getDimMappingFirstChar("business"), "rows", 1L, "idx", 118L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", getDimMappingFirstChar("entertainment"), "rows", 1L, "idx", 158L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", getDimMappingFirstChar("health"), "rows", 1L, "idx", 120L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", getDimMappingFirstChar("news"), "rows", 1L, "idx", 121L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", getDimMappingFirstChar("premium"), "rows", 3L, "idx", 2900L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", getDimMappingFirstChar("technology"), "rows", 2L, "idx", 197L), GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", null, "rows", 3L, "idx", 2447L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "a", "rows", 1L, "idx", 147L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "b", "rows", 1L, "idx", 112L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "e", "rows", 1L, "idx", 166L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "h", "rows", 1L, "idx", 113L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "n", "rows", 1L, "idx", 114L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "p", "rows", 3L, "idx", 2505L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "t", "rows", 2L, "idx", 223L) + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", getDimMappingFirstChar("automotive"), "rows", 1L, "idx", 147L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", getDimMappingFirstChar("business"), "rows", 1L, "idx", 112L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", getDimMappingFirstChar("entertainment"), "rows", 1L, "idx", 166L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", getDimMappingFirstChar("health"), "rows", 1L, "idx", 113L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", getDimMappingFirstChar("news"), "rows", 1L, "idx", 114L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", getDimMappingFirstChar("premium"), "rows", 3L, "idx", 2505L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", getDimMappingFirstChar("technology"), "rows", 2L, "idx", 223L) ); TestHelper.assertExpectedObjects( @@ -838,7 +870,7 @@ public byte[] getCacheKey() @Override public String apply(String dimValue) { - return dimValue.equals("mezzanine") ? "" : super.apply(dimValue); + return dimValue.equals(getDimMapping("mezzanine").toString()) ? "" : super.apply(dimValue); } }; @@ -855,29 +887,29 @@ public String apply(String dimValue) .setGranularity(QueryRunnerTestHelper.dayGran) .setDimensions( Lists.newArrayList( - new ExtractionDimensionSpec("quality", "alias", emptyStringExtractionFn, null) + new ExtractionDimensionSpec(qualityName, "alias", emptyStringExtractionFn, null) ) ) .build(); List expectedResults = Arrays.asList( GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "", "rows", 3L, "idx", 2870L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "a", "rows", 1L, "idx", 135L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "b", "rows", 1L, "idx", 118L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "e", "rows", 1L, "idx", 158L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "h", "rows", 1L, "idx", 120L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "n", "rows", 1L, "idx", 121L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "p", "rows", 3L, "idx", 2900L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "t", "rows", 2L, "idx", 197L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", getDimMappingFirstChar("automotive"), "rows", 1L, "idx", 135L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", getDimMappingFirstChar("business"), "rows", 1L, "idx", 118L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", getDimMappingFirstChar("entertainment"), "rows", 1L, "idx", 158L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", getDimMappingFirstChar("health"), "rows", 1L, "idx", 120L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", getDimMappingFirstChar("news"), "rows", 1L, "idx", 121L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", getDimMappingFirstChar("premium"), "rows", 3L, "idx", 2900L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", getDimMappingFirstChar("technology"), "rows", 2L, "idx", 197L), GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "", "rows", 3L, "idx", 2447L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "a", "rows", 1L, "idx", 147L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "b", "rows", 1L, "idx", 112L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "e", "rows", 1L, "idx", 166L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "h", "rows", 1L, "idx", 113L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "n", "rows", 1L, "idx", 114L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "p", "rows", 3L, "idx", 2505L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "t", "rows", 2L, "idx", 223L) + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", getDimMappingFirstChar("automotive"), "rows", 1L, "idx", 147L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", getDimMappingFirstChar("business"), "rows", 1L, "idx", 112L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", getDimMappingFirstChar("entertainment"), "rows", 1L, "idx", 166L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", getDimMappingFirstChar("health"), "rows", 1L, "idx", 113L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", getDimMappingFirstChar("news"), "rows", 1L, "idx", 114L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", getDimMappingFirstChar("premium"), "rows", 3L, "idx", 2505L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", getDimMappingFirstChar("technology"), "rows", 2L, "idx", 223L) ); TestHelper.assertExpectedObjects( @@ -898,7 +930,7 @@ public void testGroupByWithTimeZone() .setDimensions( Lists.newArrayList( (DimensionSpec) new DefaultDimensionSpec( - "quality", + qualityName, "alias" ) ) @@ -919,13 +951,14 @@ public void testGroupByWithTimeZone() tz ) ) + .setContext(ImmutableMap.of("typeHints", typeHints)) .build(); List expectedResults = Arrays.asList( GroupByQueryRunnerTestHelper.createExpectedRow( new DateTime("2011-03-31", tz), "alias", - "automotive", + getDimMapping("automotive"), "rows", 1L, "idx", @@ -934,7 +967,7 @@ public void testGroupByWithTimeZone() GroupByQueryRunnerTestHelper.createExpectedRow( new DateTime("2011-03-31", tz), "alias", - "business", + getDimMapping("business"), "rows", 1L, "idx", @@ -943,7 +976,7 @@ public void testGroupByWithTimeZone() GroupByQueryRunnerTestHelper.createExpectedRow( new DateTime("2011-03-31", tz), "alias", - "entertainment", + getDimMapping("entertainment"), "rows", 1L, "idx", @@ -952,7 +985,7 @@ public void testGroupByWithTimeZone() GroupByQueryRunnerTestHelper.createExpectedRow( new DateTime("2011-03-31", tz), "alias", - "health", + getDimMapping("health"), "rows", 1L, "idx", @@ -961,7 +994,7 @@ public void testGroupByWithTimeZone() GroupByQueryRunnerTestHelper.createExpectedRow( new DateTime("2011-03-31", tz), "alias", - "mezzanine", + getDimMapping("mezzanine"), "rows", 3L, "idx", @@ -970,7 +1003,7 @@ public void testGroupByWithTimeZone() GroupByQueryRunnerTestHelper.createExpectedRow( new DateTime("2011-03-31", tz), "alias", - "news", + getDimMapping("news"), "rows", 1L, "idx", @@ -979,7 +1012,7 @@ public void testGroupByWithTimeZone() GroupByQueryRunnerTestHelper.createExpectedRow( new DateTime("2011-03-31", tz), "alias", - "premium", + getDimMapping("premium"), "rows", 3L, "idx", @@ -988,7 +1021,7 @@ public void testGroupByWithTimeZone() GroupByQueryRunnerTestHelper.createExpectedRow( new DateTime("2011-03-31", tz), "alias", - "technology", + getDimMapping("technology"), "rows", 1L, "idx", @@ -997,7 +1030,7 @@ public void testGroupByWithTimeZone() GroupByQueryRunnerTestHelper.createExpectedRow( new DateTime("2011-03-31", tz), "alias", - "travel", + getDimMapping("travel"), "rows", 1L, "idx", @@ -1007,7 +1040,7 @@ public void testGroupByWithTimeZone() GroupByQueryRunnerTestHelper.createExpectedRow( new DateTime("2011-04-01", tz), "alias", - "automotive", + getDimMapping("automotive"), "rows", 1L, "idx", @@ -1016,7 +1049,7 @@ public void testGroupByWithTimeZone() GroupByQueryRunnerTestHelper.createExpectedRow( new DateTime("2011-04-01", tz), "alias", - "business", + getDimMapping("business"), "rows", 1L, "idx", @@ -1025,7 +1058,7 @@ public void testGroupByWithTimeZone() GroupByQueryRunnerTestHelper.createExpectedRow( new DateTime("2011-04-01", tz), "alias", - "entertainment", + getDimMapping("entertainment"), "rows", 1L, "idx", @@ -1034,7 +1067,7 @@ public void testGroupByWithTimeZone() GroupByQueryRunnerTestHelper.createExpectedRow( new DateTime("2011-04-01", tz), "alias", - "health", + getDimMapping("health"), "rows", 1L, "idx", @@ -1043,7 +1076,7 @@ public void testGroupByWithTimeZone() GroupByQueryRunnerTestHelper.createExpectedRow( new DateTime("2011-04-01", tz), "alias", - "mezzanine", + getDimMapping("mezzanine"), "rows", 3L, "idx", @@ -1052,7 +1085,7 @@ public void testGroupByWithTimeZone() GroupByQueryRunnerTestHelper.createExpectedRow( new DateTime("2011-04-01", tz), "alias", - "news", + getDimMapping("news"), "rows", 1L, "idx", @@ -1061,7 +1094,7 @@ public void testGroupByWithTimeZone() GroupByQueryRunnerTestHelper.createExpectedRow( new DateTime("2011-04-01", tz), "alias", - "premium", + getDimMapping("premium"), "rows", 3L, "idx", @@ -1070,7 +1103,7 @@ public void testGroupByWithTimeZone() GroupByQueryRunnerTestHelper.createExpectedRow( new DateTime("2011-04-01", tz), "alias", - "technology", + getDimMapping( "technology"), "rows", 1L, "idx", @@ -1079,7 +1112,7 @@ public void testGroupByWithTimeZone() GroupByQueryRunnerTestHelper.createExpectedRow( new DateTime("2011-04-01", tz), "alias", - "travel", + getDimMapping("travel"), "rows", 1L, "idx", @@ -1098,13 +1131,14 @@ public void testMergeResults() .builder() .setDataSource(QueryRunnerTestHelper.dataSource) .setInterval("2011-04-02/2011-04-04") - .setDimensions(Lists.newArrayList(new DefaultDimensionSpec("quality", "alias"))) + .setDimensions(Lists.newArrayList(new DefaultDimensionSpec(qualityName, "alias"))) .setAggregatorSpecs( Arrays.asList( QueryRunnerTestHelper.rowsCount, new LongSumAggregatorFactory("idx", "index") ) ) + .setContext(ImmutableMap.of("typeHints", typeHints)) .setGranularity(new PeriodGranularity(new Period("P1M"), null, null)); final GroupByQuery fullQuery = builder.build(); @@ -1131,15 +1165,15 @@ public Sequence run( ); List expectedResults = Arrays.asList( - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "automotive", "rows", 2L, "idx", 269L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "business", "rows", 2L, "idx", 217L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "entertainment", "rows", 2L, "idx", 319L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "health", "rows", 2L, "idx", 216L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "mezzanine", "rows", 6L, "idx", 4420L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "news", "rows", 2L, "idx", 221L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "premium", "rows", 6L, "idx", 4416L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "technology", "rows", 2L, "idx", 177L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "travel", "rows", 2L, "idx", 243L) + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", getDimMapping("automotive"), "rows", 2L, "idx", 269L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", getDimMapping("business"), "rows", 2L, "idx", 217L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", getDimMapping("entertainment"), "rows", 2L, "idx", 319L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", getDimMapping("health"), "rows", 2L, "idx", 216L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", getDimMapping("mezzanine"), "rows", 6L, "idx", 4420L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", getDimMapping("news"), "rows", 2L, "idx", 221L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", getDimMapping("premium"), "rows", 6L, "idx", 4416L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", getDimMapping("technology"), "rows", 2L, "idx", 177L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", getDimMapping("travel"), "rows", 2L, "idx", 243L) ); Map context = Maps.newHashMap(); @@ -1147,15 +1181,15 @@ public Sequence run( TestHelper.assertExpectedObjects(expectedResults, mergedRunner.run(fullQuery, context), "merged"); List allGranExpectedResults = Arrays.asList( - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "automotive", "rows", 2L, "idx", 269L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "business", "rows", 2L, "idx", 217L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "entertainment", "rows", 2L, "idx", 319L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "health", "rows", 2L, "idx", 216L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "mezzanine", "rows", 6L, "idx", 4420L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "news", "rows", 2L, "idx", 221L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "premium", "rows", 6L, "idx", 4416L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "technology", "rows", 2L, "idx", 177L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "travel", "rows", 2L, "idx", 243L) + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", getDimMapping("automotive"), "rows", 2L, "idx", 269L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", getDimMapping("business"), "rows", 2L, "idx", 217L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", getDimMapping("entertainment"), "rows", 2L, "idx", 319L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", getDimMapping("health"), "rows", 2L, "idx", 216L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", getDimMapping("mezzanine"), "rows", 6L, "idx", 4420L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", getDimMapping("news"), "rows", 2L, "idx", 221L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", getDimMapping("premium"), "rows", 6L, "idx", 4416L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", getDimMapping("technology"), "rows", 2L, "idx", 177L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", getDimMapping("travel"), "rows", 2L, "idx", 243L) ); TestHelper.assertExpectedObjects(allGranExpectedResults, runner.run(allGranQuery, context), "direct"); @@ -1176,7 +1210,7 @@ private void doTestMergeResultsWithValidLimit(final int limit) .builder() .setDataSource(QueryRunnerTestHelper.dataSource) .setInterval("2011-04-02/2011-04-04") - .setDimensions(Lists.newArrayList(new DefaultDimensionSpec("quality", "alias"))) + .setDimensions(Lists.newArrayList(new DefaultDimensionSpec(qualityName, "alias"))) .setAggregatorSpecs( Arrays.asList( QueryRunnerTestHelper.rowsCount, @@ -1184,20 +1218,21 @@ private void doTestMergeResultsWithValidLimit(final int limit) ) ) .setGranularity(new PeriodGranularity(new Period("P1M"), null, null)) + .setContext(ImmutableMap.of("typeHints", typeHints)) .setLimit(Integer.valueOf(limit)); final GroupByQuery fullQuery = builder.build(); List expectedResults = Arrays.asList( - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "automotive", "rows", 2L, "idx", 269L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "business", "rows", 2L, "idx", 217L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "entertainment", "rows", 2L, "idx", 319L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "health", "rows", 2L, "idx", 216L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "mezzanine", "rows", 6L, "idx", 4420L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "news", "rows", 2L, "idx", 221L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "premium", "rows", 6L, "idx", 4416L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "technology", "rows", 2L, "idx", 177L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "travel", "rows", 2L, "idx", 243L) + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", getDimMapping("automotive"), "rows", 2L, "idx", 269L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", getDimMapping("business"), "rows", 2L, "idx", 217L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", getDimMapping("entertainment"), "rows", 2L, "idx", 319L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", getDimMapping("health"), "rows", 2L, "idx", 216L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", getDimMapping("mezzanine"), "rows", 6L, "idx", 4420L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", getDimMapping("news"), "rows", 2L, "idx", 221L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", getDimMapping("premium"), "rows", 6L, "idx", 4416L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", getDimMapping("technology"), "rows", 2L, "idx", 177L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", getDimMapping("travel"), "rows", 2L, "idx", 243L) ); QueryRunner mergeRunner = factory.getToolchest().mergeResults(runner); @@ -1216,7 +1251,7 @@ public void testMergeResultsAcrossMultipleDaysWithLimitAndOrderBy() .builder() .setDataSource(QueryRunnerTestHelper.dataSource) .setInterval(QueryRunnerTestHelper.firstToThird) - .setDimensions(Lists.newArrayList(new DefaultDimensionSpec("quality", "alias"))) + .setDimensions(Lists.newArrayList(new DefaultDimensionSpec(qualityName, "alias"))) .setAggregatorSpecs( Arrays.asList( QueryRunnerTestHelper.rowsCount, @@ -1225,26 +1260,27 @@ public void testMergeResultsAcrossMultipleDaysWithLimitAndOrderBy() ) .setGranularity(QueryGranularity.DAY) .setLimit(limit) + .setContext(ImmutableMap.of("typeHints", typeHints)) .addOrderByColumn("idx", OrderByColumnSpec.Direction.DESCENDING); final GroupByQuery fullQuery = builder.build(); List expectedResults = Arrays.asList( - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "premium", "rows", 3L, "idx", 2900L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "mezzanine", "rows", 3L, "idx", 2870L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "entertainment", "rows", 1L, "idx", 158L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "automotive", "rows", 1L, "idx", 135L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "news", "rows", 1L, "idx", 121L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "health", "rows", 1L, "idx", 120L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "travel", "rows", 1L, "idx", 119L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "business", "rows", 1L, "idx", 118L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "technology", "rows", 1L, "idx", 78L), - - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "premium", "rows", 3L, "idx", 2505L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "mezzanine", "rows", 3L, "idx", 2447L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "entertainment", "rows", 1L, "idx", 166L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "automotive", "rows", 1L, "idx", 147L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "travel", "rows", 1L, "idx", 126L) + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", getDimMapping("premium"), "rows", 3L, "idx", 2900L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", getDimMapping("mezzanine"), "rows", 3L, "idx", 2870L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", getDimMapping("entertainment"), "rows", 1L, "idx", 158L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", getDimMapping("automotive"), "rows", 1L, "idx", 135L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", getDimMapping("news"), "rows", 1L, "idx", 121L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", getDimMapping("health"), "rows", 1L, "idx", 120L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", getDimMapping("travel"), "rows", 1L, "idx", 119L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", getDimMapping("business"), "rows", 1L, "idx", 118L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", getDimMapping("technology"), "rows", 1L, "idx", 78L), + + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", getDimMapping("premium"), "rows", 3L, "idx", 2505L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", getDimMapping("mezzanine"), "rows", 3L, "idx", 2447L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", getDimMapping("entertainment"), "rows", 1L, "idx", 166L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", getDimMapping("automotive"), "rows", 1L, "idx", 147L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", getDimMapping("travel"), "rows", 1L, "idx", 126L) ); QueryRunner mergeRunner = factory.getToolchest().mergeResults(runner); @@ -1263,7 +1299,7 @@ public void testMergeResultsWithNegativeLimit() .builder() .setDataSource(QueryRunnerTestHelper.dataSource) .setInterval("2011-04-02/2011-04-04") - .setDimensions(Lists.newArrayList(new DefaultDimensionSpec("quality", "alias"))) + .setDimensions(Lists.newArrayList(new DefaultDimensionSpec(qualityName, "alias"))) .setAggregatorSpecs( Arrays.asList( QueryRunnerTestHelper.rowsCount, @@ -1283,7 +1319,7 @@ public void testMergeResultsWithOrderBy() new DefaultLimitSpec(OrderByColumnSpec.ascending("idx"), null), new DefaultLimitSpec(OrderByColumnSpec.ascending("rows", "idx"), null), new DefaultLimitSpec(OrderByColumnSpec.descending("idx"), null), - new DefaultLimitSpec(OrderByColumnSpec.descending("rows", "idx"), null), + new DefaultLimitSpec(OrderByColumnSpec.descending("rows", "idx"), null) }; final Comparator idxComparator = @@ -1313,15 +1349,15 @@ public int compare(Row o1, Row o2) }; List allResults = Arrays.asList( - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "automotive", "rows", 2L, "idx", 269L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "business", "rows", 2L, "idx", 217L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "entertainment", "rows", 2L, "idx", 319L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "health", "rows", 2L, "idx", 216L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "mezzanine", "rows", 6L, "idx", 4420L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "news", "rows", 2L, "idx", 221L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "premium", "rows", 6L, "idx", 4416L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "technology", "rows", 2L, "idx", 177L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "travel", "rows", 2L, "idx", 243L) + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", getDimMapping("automotive"), "rows", 2L, "idx", 269L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", getDimMapping("business"), "rows", 2L, "idx", 217L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", getDimMapping("entertainment"), "rows", 2L, "idx", 319L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", getDimMapping("health"), "rows", 2L, "idx", 216L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", getDimMapping("mezzanine"), "rows", 6L, "idx", 4420L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", getDimMapping("news"), "rows", 2L, "idx", 221L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", getDimMapping("premium"), "rows", 6L, "idx", 4416L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", getDimMapping("technology"), "rows", 2L, "idx", 177L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", getDimMapping("travel"), "rows", 2L, "idx", 243L) ); List> expectedResults = Lists.newArrayList( @@ -1342,7 +1378,7 @@ private void doTestMergeResultsWithOrderBy(LimitSpec orderBySpec, List expe .builder() .setDataSource(QueryRunnerTestHelper.dataSource) .setInterval("2011-04-02/2011-04-04") - .setDimensions(Lists.newArrayList(new DefaultDimensionSpec("quality", "alias"))) + .setDimensions(Lists.newArrayList(new DefaultDimensionSpec(qualityName, "alias"))) .setAggregatorSpecs( Arrays.asList( QueryRunnerTestHelper.rowsCount, @@ -1350,6 +1386,7 @@ private void doTestMergeResultsWithOrderBy(LimitSpec orderBySpec, List expe ) ) .setGranularity(new PeriodGranularity(new Period("P1M"), null, null)) + .setContext(ImmutableMap.of("typeHints", typeHints)) .setLimitSpec(orderBySpec); final GroupByQuery fullQuery = builder.build(); @@ -1385,7 +1422,7 @@ public void testGroupByOrderLimit() throws Exception .builder() .setDataSource(QueryRunnerTestHelper.dataSource) .setInterval("2011-04-02/2011-04-04") - .setDimensions(Lists.newArrayList(new DefaultDimensionSpec("quality", "alias"))) + .setDimensions(Lists.newArrayList(new DefaultDimensionSpec(qualityName, "alias"))) .setAggregatorSpecs( Arrays.asList( QueryRunnerTestHelper.rowsCount, @@ -1394,20 +1431,21 @@ public void testGroupByOrderLimit() throws Exception ) .addOrderByColumn("rows") .addOrderByColumn("alias", OrderByColumnSpec.Direction.DESCENDING) + .setContext(ImmutableMap.of("typeHints", typeHints)) .setGranularity(new PeriodGranularity(new Period("P1M"), null, null)); final GroupByQuery query = builder.build(); List expectedResults = Arrays.asList( - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "travel", "rows", 2L, "idx", 243L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "technology", "rows", 2L, "idx", 177L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "news", "rows", 2L, "idx", 221L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "health", "rows", 2L, "idx", 216L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "entertainment", "rows", 2L, "idx", 319L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "business", "rows", 2L, "idx", 217L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "automotive", "rows", 2L, "idx", 269L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "premium", "rows", 6L, "idx", 4416L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "mezzanine", "rows", 6L, "idx", 4420L) + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", getDimMapping("travel"), "rows", 2L, "idx", 243L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", getDimMapping("technology"), "rows", 2L, "idx", 177L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", getDimMapping("news"), "rows", 2L, "idx", 221L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", getDimMapping("health"), "rows", 2L, "idx", 216L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", getDimMapping("entertainment"), "rows", 2L, "idx", 319L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", getDimMapping("business"), "rows", 2L, "idx", 217L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", getDimMapping("automotive"), "rows", 2L, "idx", 269L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", getDimMapping("premium"), "rows", 6L, "idx", 4416L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", getDimMapping("mezzanine"), "rows", 6L, "idx", 4420L) ); Map context = Maps.newHashMap(); @@ -1426,7 +1464,7 @@ public void testGroupByWithOrderLimit2() throws Exception .builder() .setDataSource(QueryRunnerTestHelper.dataSource) .setInterval("2011-04-02/2011-04-04") - .setDimensions(Lists.newArrayList(new DefaultDimensionSpec("quality", "alias"))) + .setDimensions(Lists.newArrayList(new DefaultDimensionSpec(qualityName, "alias"))) .setAggregatorSpecs( Arrays.asList( QueryRunnerTestHelper.rowsCount, @@ -1435,20 +1473,21 @@ public void testGroupByWithOrderLimit2() throws Exception ) .addOrderByColumn("rows", "desc") .addOrderByColumn("alias", "d") + .setContext(ImmutableMap.of("typeHints", typeHints)) .setGranularity(new PeriodGranularity(new Period("P1M"), null, null)); final GroupByQuery query = builder.build(); List expectedResults = Arrays.asList( - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "premium", "rows", 6L, "idx", 4416L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "mezzanine", "rows", 6L, "idx", 4420L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "travel", "rows", 2L, "idx", 243L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "technology", "rows", 2L, "idx", 177L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "news", "rows", 2L, "idx", 221L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "health", "rows", 2L, "idx", 216L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "entertainment", "rows", 2L, "idx", 319L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "business", "rows", 2L, "idx", 217L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "automotive", "rows", 2L, "idx", 269L) + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", getDimMapping("premium"), "rows", 6L, "idx", 4416L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", getDimMapping("mezzanine"), "rows", 6L, "idx", 4420L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", getDimMapping("travel"), "rows", 2L, "idx", 243L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", getDimMapping("technology"), "rows", 2L, "idx", 177L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", getDimMapping("news"), "rows", 2L, "idx", 221L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", getDimMapping("health"), "rows", 2L, "idx", 216L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", getDimMapping("entertainment"), "rows", 2L, "idx", 319L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", getDimMapping("business"), "rows", 2L, "idx", 217L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", getDimMapping("automotive"), "rows", 2L, "idx", 269L) ); Map context = Maps.newHashMap(); @@ -1466,7 +1505,7 @@ public void testGroupByWithOrderLimit3() throws Exception .builder() .setDataSource(QueryRunnerTestHelper.dataSource) .setInterval("2011-04-02/2011-04-04") - .setDimensions(Lists.newArrayList(new DefaultDimensionSpec("quality", "alias"))) + .setDimensions(Lists.newArrayList(new DefaultDimensionSpec(qualityName, "alias"))) .setAggregatorSpecs( Arrays.asList( QueryRunnerTestHelper.rowsCount, @@ -1475,6 +1514,7 @@ public void testGroupByWithOrderLimit3() throws Exception ) .addOrderByColumn("idx", "desc") .addOrderByColumn("alias", "d") + .setContext(ImmutableMap.of("typeHints", typeHints)) .setGranularity(new PeriodGranularity(new Period("P1M"), null, null)); final GroupByQuery query = builder.build(); @@ -1483,7 +1523,7 @@ public void testGroupByWithOrderLimit3() throws Exception GroupByQueryRunnerTestHelper.createExpectedRow( "2011-04-01", "alias", - "mezzanine", + getDimMapping("mezzanine"), "rows", 6L, "idx", @@ -1492,7 +1532,7 @@ public void testGroupByWithOrderLimit3() throws Exception GroupByQueryRunnerTestHelper.createExpectedRow( "2011-04-01", "alias", - "premium", + getDimMapping("premium"), "rows", 6L, "idx", @@ -1501,7 +1541,7 @@ public void testGroupByWithOrderLimit3() throws Exception GroupByQueryRunnerTestHelper.createExpectedRow( "2011-04-01", "alias", - "entertainment", + getDimMapping("entertainment"), "rows", 2L, "idx", @@ -1510,7 +1550,7 @@ public void testGroupByWithOrderLimit3() throws Exception GroupByQueryRunnerTestHelper.createExpectedRow( "2011-04-01", "alias", - "automotive", + getDimMapping("automotive"), "rows", 2L, "idx", @@ -1519,7 +1559,7 @@ public void testGroupByWithOrderLimit3() throws Exception GroupByQueryRunnerTestHelper.createExpectedRow( "2011-04-01", "alias", - "travel", + getDimMapping("travel"), "rows", 2L, "idx", @@ -1528,7 +1568,7 @@ public void testGroupByWithOrderLimit3() throws Exception GroupByQueryRunnerTestHelper.createExpectedRow( "2011-04-01", "alias", - "news", + getDimMapping("news"), "rows", 2L, "idx", @@ -1537,7 +1577,7 @@ public void testGroupByWithOrderLimit3() throws Exception GroupByQueryRunnerTestHelper.createExpectedRow( "2011-04-01", "alias", - "business", + getDimMapping("business"), "rows", 2L, "idx", @@ -1546,7 +1586,7 @@ public void testGroupByWithOrderLimit3() throws Exception GroupByQueryRunnerTestHelper.createExpectedRow( "2011-04-01", "alias", - "health", + getDimMapping("health"), "rows", 2L, "idx", @@ -1555,7 +1595,7 @@ public void testGroupByWithOrderLimit3() throws Exception GroupByQueryRunnerTestHelper.createExpectedRow( "2011-04-01", "alias", - "technology", + getDimMapping("technology"), "rows", 2L, "idx", @@ -1580,7 +1620,7 @@ public void testGroupByWithSameCaseOrdering() .setDimensions( Arrays.asList( new DefaultDimensionSpec( - QueryRunnerTestHelper.marketDimension, + marketName, "marketalias" ) ) @@ -1601,27 +1641,28 @@ public void testGroupByWithSameCaseOrdering() QueryRunnerTestHelper.rowsCount ) ) + .setContext(ImmutableMap.of("typeHints", typeHints)) .build(); List expectedResults = Arrays.asList( GroupByQueryRunnerTestHelper.createExpectedRow( "1970-01-01T00:00:00.000Z", "marketalias", - "upfront", + getDimMapping("upfront"), "rows", 186L ), GroupByQueryRunnerTestHelper.createExpectedRow( "1970-01-01T00:00:00.000Z", "marketalias", - "total_market", + getDimMapping("total_market"), "rows", 186L ), GroupByQueryRunnerTestHelper.createExpectedRow( "1970-01-01T00:00:00.000Z", "marketalias", - "spot", + getDimMapping("spot"), "rows", 837L ) @@ -1640,8 +1681,8 @@ public void testGroupByWithOrderLimit4() .setDimensions( Arrays.asList( new DefaultDimensionSpec( - QueryRunnerTestHelper.marketDimension, - QueryRunnerTestHelper.marketDimension + marketName, + marketName ) ) ) @@ -1650,7 +1691,7 @@ public void testGroupByWithOrderLimit4() new DefaultLimitSpec( Lists.newArrayList( new OrderByColumnSpec( - QueryRunnerTestHelper.marketDimension, + marketName, OrderByColumnSpec.Direction.DESCENDING ) ), 3 @@ -1661,18 +1702,19 @@ public void testGroupByWithOrderLimit4() QueryRunnerTestHelper.rowsCount ) ) + .setContext(ImmutableMap.of("typeHints", typeHints)) .build(); List expectedResults = Arrays.asList( - GroupByQueryRunnerTestHelper.createExpectedRow("1970-01-01T00:00:00.000Z", "market", "upfront", "rows", 186L), + GroupByQueryRunnerTestHelper.createExpectedRow("1970-01-01T00:00:00.000Z", marketName, getDimMapping("upfront"), "rows", 186L), GroupByQueryRunnerTestHelper.createExpectedRow( "1970-01-01T00:00:00.000Z", - "market", - "total_market", + marketName, + getDimMapping("total_market"), "rows", 186L ), - GroupByQueryRunnerTestHelper.createExpectedRow("1970-01-01T00:00:00.000Z", "market", "spot", "rows", 837L) + GroupByQueryRunnerTestHelper.createExpectedRow("1970-01-01T00:00:00.000Z", marketName, getDimMapping("spot"), "rows", 837L) ); Iterable results = GroupByQueryRunnerTestHelper.runQuery(factory, runner, query); @@ -1688,8 +1730,8 @@ public void testGroupByWithOrderOnHyperUnique() .setDimensions( Arrays.asList( new DefaultDimensionSpec( - QueryRunnerTestHelper.marketDimension, - QueryRunnerTestHelper.marketDimension + marketName, + marketName ) ) ) @@ -1717,13 +1759,14 @@ public void testGroupByWithOrderOnHyperUnique() ) ) ) + .setContext(ImmutableMap.of("typeHints", typeHints)) .build(); List expectedResults = Arrays.asList( GroupByQueryRunnerTestHelper.createExpectedRow( "1970-01-01T00:00:00.000Z", - "market", - "spot", + marketName, + getDimMapping("spot"), QueryRunnerTestHelper.uniqueMetric, QueryRunnerTestHelper.UNIQUES_9, QueryRunnerTestHelper.hyperUniqueFinalizingPostAggMetric, @@ -1731,8 +1774,8 @@ public void testGroupByWithOrderOnHyperUnique() ), GroupByQueryRunnerTestHelper.createExpectedRow( "1970-01-01T00:00:00.000Z", - "market", - "upfront", + marketName, + getDimMapping("upfront"), QueryRunnerTestHelper.uniqueMetric, QueryRunnerTestHelper.UNIQUES_2, QueryRunnerTestHelper.hyperUniqueFinalizingPostAggMetric, @@ -1740,8 +1783,8 @@ public void testGroupByWithOrderOnHyperUnique() ), GroupByQueryRunnerTestHelper.createExpectedRow( "1970-01-01T00:00:00.000Z", - "market", - "total_market", + marketName, + getDimMapping("total_market"), QueryRunnerTestHelper.uniqueMetric, QueryRunnerTestHelper.UNIQUES_2, QueryRunnerTestHelper.hyperUniqueFinalizingPostAggMetric, @@ -1959,15 +2002,15 @@ public void testGroupByWithLimitOnFinalizedHyperUnique() public void testGroupByWithAlphaNumericDimensionOrder() { Map map = new HashMap<>(); - map.put("automotive", "health105"); - map.put("business", "health20"); - map.put("entertainment", "travel47"); - map.put("health", "health55"); - map.put("mezzanine", "health09"); - map.put("news", "health0000"); - map.put("premium", "health999"); - map.put("technology", "travel123"); - map.put("travel", "travel555"); + map.put(getDimMapping("automotive").toString(), "health105"); + map.put(getDimMapping("business").toString(), "health20"); + map.put(getDimMapping("entertainment").toString(), "travel47"); + map.put(getDimMapping("health").toString(), "health55"); + map.put(getDimMapping("mezzanine").toString(), "health09"); + map.put(getDimMapping("news").toString(), "health0000"); + map.put(getDimMapping("premium").toString(), "health999"); + map.put(getDimMapping("technology").toString(), "travel123"); + map.put(getDimMapping("travel").toString(), "travel555"); GroupByQuery query = GroupByQuery .builder() @@ -1976,7 +2019,7 @@ public void testGroupByWithAlphaNumericDimensionOrder() .setDimensions( Lists.newArrayList( new ExtractionDimensionSpec( - "quality", "alias", new LookupExtractionFn(new MapLookupExtractor(map, false), false, null, false, false), null + qualityName, "alias", new LookupExtractionFn(new MapLookupExtractor(map, false), false, null, false, false), null ) ) ) @@ -2028,14 +2071,14 @@ public void testLimitPerGrouping() .setDimensions( Arrays.asList( new DefaultDimensionSpec( - QueryRunnerTestHelper.marketDimension, - QueryRunnerTestHelper.marketDimension + marketName, + marketName ) ) ) .setInterval(QueryRunnerTestHelper.firstToThird) - // Using a limitSpec here to achieve a per group limit is incorrect. - // Limit is applied on the overall results. + // Using a limitSpec here to achieve a per group limit is incorrect. + // Limit is applied on the overall results. .setLimitSpec( new DefaultLimitSpec( Lists.newArrayList( @@ -2054,8 +2097,8 @@ public void testLimitPerGrouping() .build(); List expectedResults = Arrays.asList( - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01T00:00:00.000Z", "market", "spot", "rows", 9L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02T00:00:00.000Z", "market", "spot", "rows", 9L) + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01T00:00:00.000Z", marketName, getDimMapping("spot"), "rows", 9L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02T00:00:00.000Z", marketName, getDimMapping("spot"), "rows", 9L) ); Iterable results = GroupByQueryRunnerTestHelper.runQuery(factory, runner, query); @@ -2078,7 +2121,7 @@ public void testPostAggMergedHavingSpec() GroupByQueryRunnerTestHelper.createExpectedRow( "2011-04-01", "alias", - "mezzanine", + getDimMapping("mezzanine"), "rows", 6L, "index", @@ -2089,7 +2132,7 @@ public void testPostAggMergedHavingSpec() GroupByQueryRunnerTestHelper.createExpectedRow( "2011-04-01", "alias", - "premium", + getDimMapping("premium"), "rows", 6L, "index", @@ -2103,7 +2146,7 @@ public void testPostAggMergedHavingSpec() .builder() .setDataSource(QueryRunnerTestHelper.dataSource) .setInterval("2011-04-02/2011-04-04") - .setDimensions(Lists.newArrayList(new DefaultDimensionSpec("quality", "alias"))) + .setDimensions(Lists.newArrayList(new DefaultDimensionSpec(qualityName, "alias"))) .setAggregatorSpecs( Arrays.asList( QueryRunnerTestHelper.rowsCount, @@ -2112,6 +2155,7 @@ public void testPostAggMergedHavingSpec() ) .setPostAggregatorSpecs(ImmutableList.of(QueryRunnerTestHelper.addRowsIndexConstant)) .setGranularity(new PeriodGranularity(new Period("P1M"), null, null)) + .setContext(ImmutableMap.of("typeHints", typeHints)) .setHavingSpec( new OrHavingSpec( ImmutableList.of( @@ -2153,7 +2197,7 @@ public void testGroupByWithOrderLimitHavingSpec() .builder() .setDataSource(QueryRunnerTestHelper.dataSource) .setInterval("2011-01-25/2011-01-28") - .setDimensions(Lists.newArrayList(new DefaultDimensionSpec("quality", "alias"))) + .setDimensions(Lists.newArrayList(new DefaultDimensionSpec(qualityName, "alias"))) .setAggregatorSpecs( Arrays.asList( QueryRunnerTestHelper.rowsCount, @@ -2162,6 +2206,7 @@ public void testGroupByWithOrderLimitHavingSpec() ) .setGranularity(QueryGranularity.ALL) .setHavingSpec(new GreaterThanHavingSpec("index", 310L)) + .setContext(ImmutableMap.of("typeHints", typeHints)) .setLimitSpec( new DefaultLimitSpec( Lists.newArrayList( @@ -2178,7 +2223,7 @@ public void testGroupByWithOrderLimitHavingSpec() GroupByQueryRunnerTestHelper.createExpectedRow( "2011-01-25", "alias", - "business", + getDimMapping("business"), "rows", 3L, "index", @@ -2187,7 +2232,7 @@ public void testGroupByWithOrderLimitHavingSpec() GroupByQueryRunnerTestHelper.createExpectedRow( "2011-01-25", "alias", - "news", + getDimMapping("news"), "rows", 3L, "index", @@ -2196,7 +2241,7 @@ public void testGroupByWithOrderLimitHavingSpec() GroupByQueryRunnerTestHelper.createExpectedRow( "2011-01-25", "alias", - "technology", + getDimMapping("technology"), "rows", 3L, "index", @@ -2205,7 +2250,7 @@ public void testGroupByWithOrderLimitHavingSpec() GroupByQueryRunnerTestHelper.createExpectedRow( "2011-01-25", "alias", - "travel", + getDimMapping("travel"), "rows", 3L, "index", @@ -2214,7 +2259,7 @@ public void testGroupByWithOrderLimitHavingSpec() GroupByQueryRunnerTestHelper.createExpectedRow( "2011-01-25", "alias", - "health", + getDimMapping("health"), "rows", 3L, "index", @@ -2238,7 +2283,7 @@ public void testPostAggHavingSpec() GroupByQueryRunnerTestHelper.createExpectedRow( "2011-04-01", "alias", - "mezzanine", + getDimMapping("mezzanine"), "rows", 6L, "index", @@ -2249,7 +2294,7 @@ public void testPostAggHavingSpec() GroupByQueryRunnerTestHelper.createExpectedRow( "2011-04-01", "alias", - "premium", + getDimMapping("premium"), "rows", 6L, "index", @@ -2263,7 +2308,7 @@ public void testPostAggHavingSpec() .builder() .setDataSource(QueryRunnerTestHelper.dataSource) .setInterval("2011-04-02/2011-04-04") - .setDimensions(Lists.newArrayList(new DefaultDimensionSpec("quality", "alias"))) + .setDimensions(Lists.newArrayList(new DefaultDimensionSpec(qualityName, "alias"))) .setAggregatorSpecs( Arrays.asList( QueryRunnerTestHelper.rowsCount, @@ -2272,6 +2317,7 @@ public void testPostAggHavingSpec() ) .setPostAggregatorSpecs(ImmutableList.of(QueryRunnerTestHelper.addRowsIndexConstant)) .setGranularity(new PeriodGranularity(new Period("P1M"), null, null)) + .setContext(ImmutableMap.of("typeHints", typeHints)) .setHavingSpec( new OrHavingSpec( ImmutableList.of( @@ -2293,16 +2339,16 @@ public void testPostAggHavingSpec() public void testHavingSpec() { List expectedResults = Arrays.asList( - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "business", "rows", 2L, "idx", 217L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "mezzanine", "rows", 6L, "idx", 4420L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "premium", "rows", 6L, "idx", 4416L) + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", getDimMapping("business"), "rows", 2L, "idx", 217L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", getDimMapping("mezzanine"), "rows", 6L, "idx", 4420L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", getDimMapping("premium"), "rows", 6L, "idx", 4416L) ); GroupByQuery.Builder builder = GroupByQuery .builder() .setDataSource(QueryRunnerTestHelper.dataSource) .setInterval("2011-04-02/2011-04-04") - .setDimensions(Lists.newArrayList(new DefaultDimensionSpec("quality", "alias"))) + .setDimensions(Lists.newArrayList(new DefaultDimensionSpec(qualityName, "alias"))) .setAggregatorSpecs( Arrays.asList( QueryRunnerTestHelper.rowsCount, @@ -2310,6 +2356,7 @@ public void testHavingSpec() ) ) .setGranularity(new PeriodGranularity(new Period("P1M"), null, null)) + .setContext(ImmutableMap.of("typeHints", typeHints)) .setHavingSpec( new OrHavingSpec( ImmutableList.of( @@ -2331,16 +2378,16 @@ public void testHavingSpec() public void testMergedHavingSpec() { List expectedResults = Arrays.asList( - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "business", "rows", 2L, "idx", 217L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "mezzanine", "rows", 6L, "idx", 4420L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "premium", "rows", 6L, "idx", 4416L) + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", getDimMapping("business"), "rows", 2L, "idx", 217L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", getDimMapping("mezzanine"), "rows", 6L, "idx", 4420L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", getDimMapping("premium"), "rows", 6L, "idx", 4416L) ); GroupByQuery.Builder builder = GroupByQuery .builder() .setDataSource(QueryRunnerTestHelper.dataSource) .setInterval("2011-04-02/2011-04-04") - .setDimensions(Lists.newArrayList(new DefaultDimensionSpec("quality", "alias"))) + .setDimensions(Lists.newArrayList(new DefaultDimensionSpec(qualityName, "alias"))) .setAggregatorSpecs( Arrays.asList( QueryRunnerTestHelper.rowsCount, @@ -2348,6 +2395,7 @@ public void testMergedHavingSpec() ) ) .setGranularity(new PeriodGranularity(new Period("P1M"), null, null)) + .setContext(ImmutableMap.of("typeHints", typeHints)) .setHavingSpec( new OrHavingSpec( ImmutableList.of( @@ -2390,7 +2438,7 @@ public void testMergedPostAggHavingSpec() GroupByQueryRunnerTestHelper.createExpectedRow( "2011-04-01", "alias", - "business", + getDimMapping("business"), "rows", 2L, "idx", @@ -2401,7 +2449,7 @@ public void testMergedPostAggHavingSpec() GroupByQueryRunnerTestHelper.createExpectedRow( "2011-04-01", "alias", - "mezzanine", + getDimMapping("mezzanine"), "rows", 6L, "idx", @@ -2412,7 +2460,7 @@ public void testMergedPostAggHavingSpec() GroupByQueryRunnerTestHelper.createExpectedRow( "2011-04-01", "alias", - "premium", + getDimMapping("premium"), "rows", 6L, "idx", @@ -2426,7 +2474,7 @@ public void testMergedPostAggHavingSpec() .builder() .setDataSource(QueryRunnerTestHelper.dataSource) .setInterval("2011-04-02/2011-04-04") - .setDimensions(Lists.newArrayList(new DefaultDimensionSpec("quality", "alias"))) + .setDimensions(Lists.newArrayList(new DefaultDimensionSpec(qualityName, "alias"))) .setAggregatorSpecs( Arrays.asList( QueryRunnerTestHelper.rowsCount, @@ -2452,6 +2500,7 @@ public void testMergedPostAggHavingSpec() ) ) .setGranularity(new PeriodGranularity(new Period("P1M"), null, null)) + .setContext(ImmutableMap.of("typeHints", typeHints)) .setHavingSpec( new OrHavingSpec( ImmutableList.of( @@ -2502,23 +2551,25 @@ public Sequence run( @Test public void testGroupByWithRegEx() throws Exception { + String regexPattern = groupingKeyType == ValueType.STRING ? "auto.*" : "11.*"; GroupByQuery.Builder builder = GroupByQuery .builder() .setDataSource(QueryRunnerTestHelper.dataSource) .setInterval("2011-04-02/2011-04-04") - .setDimFilter(new RegexDimFilter("quality", "auto.*")) - .setDimensions(Lists.newArrayList(new DefaultDimensionSpec("quality", "quality"))) + .setDimFilter(new RegexDimFilter(qualityName, regexPattern)) + .setDimensions(Lists.newArrayList(new DefaultDimensionSpec(qualityName, qualityName))) .setAggregatorSpecs( Arrays.asList( QueryRunnerTestHelper.rowsCount ) ) + .setContext(ImmutableMap.of("typeHints", typeHints)) .setGranularity(new PeriodGranularity(new Period("P1M"), null, null)); final GroupByQuery query = builder.build(); List expectedResults = Arrays.asList( - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "quality", "automotive", "rows", 2L) + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", qualityName, getDimMapping("automotive"), "rows", 2L) ); final GroupByQueryEngine engine = new GroupByQueryEngine( @@ -2553,13 +2604,14 @@ public void testGroupByWithMetricColumnDisappears() throws Exception .builder() .setDataSource(QueryRunnerTestHelper.dataSource) .setInterval("2011-04-02/2011-04-04") - .addDimension("quality") + .addDimension(qualityName) .addDimension("index") .setAggregatorSpecs( Arrays.asList( QueryRunnerTestHelper.rowsCount ) ) + .setContext(ImmutableMap.of("typeHints", typeHints)) .setGranularity(new PeriodGranularity(new Period("P1M"), null, null)); final GroupByQuery query = builder.build(); @@ -2569,35 +2621,35 @@ public void testGroupByWithMetricColumnDisappears() throws Exception "2011-04-01", "index", null, - "quality", - "automotive", + qualityName, + getDimMapping("automotive"), "rows", 2L ), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "index", null, "quality", "business", "rows", 2L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "index", null, qualityName, getDimMapping("business"), "rows", 2L), GroupByQueryRunnerTestHelper.createExpectedRow( "2011-04-01", "index", null, - "quality", - "entertainment", + qualityName, + getDimMapping("entertainment"), "rows", 2L ), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "index", null, "quality", "health", "rows", 2L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "index", null, "quality", "mezzanine", "rows", 6L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "index", null, "quality", "news", "rows", 2L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "index", null, "quality", "premium", "rows", 6L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "index", null, qualityName, getDimMapping("health"), "rows", 2L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "index", null, qualityName, getDimMapping("mezzanine"), "rows", 6L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "index", null, qualityName, getDimMapping("news"), "rows", 2L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "index", null, qualityName, getDimMapping("premium"), "rows", 6L), GroupByQueryRunnerTestHelper.createExpectedRow( "2011-04-01", "index", null, - "quality", - "technology", + qualityName, + getDimMapping("technology"), "rows", 2L ), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "index", null, "quality", "travel", "rows", 2L) + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "index", null, qualityName, getDimMapping("travel"), "rows", 2L) ); Map context = Maps.newHashMap(); @@ -2634,12 +2686,13 @@ public void testGroupByWithNonexistentDimension() throws Exception .setDataSource(QueryRunnerTestHelper.dataSource) .setInterval("2011-04-02/2011-04-04") .addDimension("billy") - .addDimension("quality") + .addDimension(qualityName) .setAggregatorSpecs( Arrays.asList( QueryRunnerTestHelper.rowsCount ) ) + .setContext(ImmutableMap.of("typeHints", typeHints)) .setGranularity(new PeriodGranularity(new Period("P1M"), null, null)); final GroupByQuery query = builder.build(); @@ -2649,35 +2702,35 @@ public void testGroupByWithNonexistentDimension() throws Exception "2011-04-01", "billy", null, - "quality", - "automotive", + qualityName, + getDimMapping("automotive"), "rows", 2L ), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "billy", null, "quality", "business", "rows", 2L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "billy", null, qualityName, getDimMapping("business"), "rows", 2L), GroupByQueryRunnerTestHelper.createExpectedRow( "2011-04-01", "billy", null, - "quality", - "entertainment", + qualityName, + getDimMapping("entertainment"), "rows", 2L ), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "billy", null, "quality", "health", "rows", 2L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "billy", null, "quality", "mezzanine", "rows", 6L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "billy", null, "quality", "news", "rows", 2L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "billy", null, "quality", "premium", "rows", 6L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "billy", null, qualityName, getDimMapping("health"), "rows", 2L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "billy", null, qualityName, getDimMapping("mezzanine"), "rows", 6L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "billy", null, qualityName, getDimMapping("news"), "rows", 2L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "billy", null, qualityName, getDimMapping("premium"), "rows", 6L), GroupByQueryRunnerTestHelper.createExpectedRow( "2011-04-01", "billy", null, - "quality", - "technology", + qualityName, + getDimMapping("technology"), "rows", 2L ), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "billy", null, "quality", "travel", "rows", 2L) + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "billy", null, qualityName, getDimMapping("travel"), "rows", 2L) ); Map context = Maps.newHashMap(); @@ -2714,8 +2767,8 @@ public void testIdenticalSubquery() .builder() .setDataSource(QueryRunnerTestHelper.dataSource) .setQuerySegmentSpec(QueryRunnerTestHelper.firstToThird) - .setDimensions(Lists.newArrayList(new DefaultDimensionSpec("quality", "alias"))) - .setDimFilter(new JavaScriptDimFilter("quality", "function(dim){ return true; }")) + .setDimensions(Lists.newArrayList(new DefaultDimensionSpec(qualityName, "alias"))) + .setDimFilter(new JavaScriptDimFilter(qualityName, "function(dim){ return true; }")) .setAggregatorSpecs( Arrays.asList( QueryRunnerTestHelper.rowsCount, @@ -2737,28 +2790,29 @@ public void testIdenticalSubquery() ) ) .setGranularity(QueryRunnerTestHelper.dayGran) + .setContext(ImmutableMap.of("typeHints", typeHints)) .build(); List expectedResults = Arrays.asList( - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "automotive", "rows", 1L, "idx", 135L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "business", "rows", 1L, "idx", 118L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "entertainment", "rows", 1L, "idx", 158L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "health", "rows", 1L, "idx", 120L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "mezzanine", "rows", 3L, "idx", 2870L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "news", "rows", 1L, "idx", 121L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "premium", "rows", 3L, "idx", 2900L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "technology", "rows", 1L, "idx", 78L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "travel", "rows", 1L, "idx", 119L), - - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "automotive", "rows", 1L, "idx", 147L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "business", "rows", 1L, "idx", 112L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "entertainment", "rows", 1L, "idx", 166L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "health", "rows", 1L, "idx", 113L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "mezzanine", "rows", 3L, "idx", 2447L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "news", "rows", 1L, "idx", 114L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "premium", "rows", 3L, "idx", 2505L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "technology", "rows", 1L, "idx", 97L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "travel", "rows", 1L, "idx", 126L) + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", getDimMapping("automotive"), "rows", 1L, "idx", 135L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", getDimMapping("business"), "rows", 1L, "idx", 118L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", getDimMapping("entertainment"), "rows", 1L, "idx", 158L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", getDimMapping("health"), "rows", 1L, "idx", 120L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", getDimMapping("mezzanine"), "rows", 3L, "idx", 2870L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", getDimMapping("news"), "rows", 1L, "idx", 121L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", getDimMapping("premium"), "rows", 3L, "idx", 2900L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", getDimMapping("technology"), "rows", 1L, "idx", 78L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", getDimMapping("travel"), "rows", 1L, "idx", 119L), + + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", getDimMapping("automotive"), "rows", 1L, "idx", 147L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", getDimMapping("business"), "rows", 1L, "idx", 112L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", getDimMapping("entertainment"), "rows", 1L, "idx", 166L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", getDimMapping("health"), "rows", 1L, "idx", 113L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", getDimMapping("mezzanine"), "rows", 3L, "idx", 2447L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", getDimMapping("news"), "rows", 1L, "idx", 114L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", getDimMapping("premium"), "rows", 3L, "idx", 2505L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", getDimMapping("technology"), "rows", 1L, "idx", 97L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", getDimMapping("travel"), "rows", 1L, "idx", 126L) ); // Subqueries are handled by the ToolChest @@ -2773,8 +2827,8 @@ public void testSubqueryWithMultipleIntervalsInOuterQuery() .builder() .setDataSource(QueryRunnerTestHelper.dataSource) .setQuerySegmentSpec(QueryRunnerTestHelper.firstToThird) - .setDimensions(Lists.newArrayList(new DefaultDimensionSpec("quality", "alias"))) - .setDimFilter(new JavaScriptDimFilter("quality", "function(dim){ return true; }")) + .setDimensions(Lists.newArrayList(new DefaultDimensionSpec(qualityName, "alias"))) + .setDimFilter(new JavaScriptDimFilter(qualityName, "function(dim){ return true; }")) .setAggregatorSpecs( Arrays.asList( QueryRunnerTestHelper.rowsCount, @@ -2803,28 +2857,29 @@ public void testSubqueryWithMultipleIntervalsInOuterQuery() ) ) .setGranularity(QueryRunnerTestHelper.dayGran) + .setContext(ImmutableMap.of("typeHints", typeHints)) .build(); List expectedResults = Arrays.asList( - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "automotive", "rows", 1L, "idx", 135L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "business", "rows", 1L, "idx", 118L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "entertainment", "rows", 1L, "idx", 158L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "health", "rows", 1L, "idx", 120L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "mezzanine", "rows", 3L, "idx", 2870L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "news", "rows", 1L, "idx", 121L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "premium", "rows", 3L, "idx", 2900L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "technology", "rows", 1L, "idx", 78L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "travel", "rows", 1L, "idx", 119L), - - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "automotive", "rows", 1L, "idx", 147L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "business", "rows", 1L, "idx", 112L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "entertainment", "rows", 1L, "idx", 166L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "health", "rows", 1L, "idx", 113L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "mezzanine", "rows", 3L, "idx", 2447L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "news", "rows", 1L, "idx", 114L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "premium", "rows", 3L, "idx", 2505L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "technology", "rows", 1L, "idx", 97L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "travel", "rows", 1L, "idx", 126L) + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", getDimMapping("automotive"), "rows", 1L, "idx", 135L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", getDimMapping("business"), "rows", 1L, "idx", 118L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", getDimMapping("entertainment"), "rows", 1L, "idx", 158L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", getDimMapping("health"), "rows", 1L, "idx", 120L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", getDimMapping("mezzanine"), "rows", 3L, "idx", 2870L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", getDimMapping("news"), "rows", 1L, "idx", 121L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", getDimMapping("premium"), "rows", 3L, "idx", 2900L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", getDimMapping("technology"), "rows", 1L, "idx", 78L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", getDimMapping("travel"), "rows", 1L, "idx", 119L), + + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", getDimMapping("automotive"), "rows", 1L, "idx", 147L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", getDimMapping("business"), "rows", 1L, "idx", 112L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", getDimMapping("entertainment"), "rows", 1L, "idx", 166L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", getDimMapping("health"), "rows", 1L, "idx", 113L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", getDimMapping("mezzanine"), "rows", 3L, "idx", 2447L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", getDimMapping("news"), "rows", 1L, "idx", 114L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", getDimMapping("premium"), "rows", 3L, "idx", 2505L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", getDimMapping("technology"), "rows", 1L, "idx", 97L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", getDimMapping("travel"), "rows", 1L, "idx", 126L) ); // Subqueries are handled by the ToolChest @@ -2841,8 +2896,8 @@ public void testSubqueryWithExtractionFnInOuterQuery() .builder() .setDataSource(QueryRunnerTestHelper.dataSource) .setQuerySegmentSpec(QueryRunnerTestHelper.firstToThird) - .setDimensions(Lists.newArrayList(new DefaultDimensionSpec("quality", "alias"))) - .setDimFilter(new JavaScriptDimFilter("quality", "function(dim){ return true; }")) + .setDimensions(Lists.newArrayList(new DefaultDimensionSpec(qualityName, "alias"))) + .setDimFilter(new JavaScriptDimFilter(qualityName, "function(dim){ return true; }")) .setAggregatorSpecs( Arrays.asList( QueryRunnerTestHelper.rowsCount, @@ -2896,7 +2951,7 @@ public void testDifferentGroupingSubquery() .builder() .setDataSource(QueryRunnerTestHelper.dataSource) .setQuerySegmentSpec(QueryRunnerTestHelper.firstToThird) - .setDimensions(Lists.newArrayList(new DefaultDimensionSpec("quality", "alias"))) + .setDimensions(Lists.newArrayList(new DefaultDimensionSpec(qualityName, "alias"))) .setAggregatorSpecs( Arrays.asList( QueryRunnerTestHelper.rowsCount, @@ -2934,7 +2989,7 @@ public void testDifferentGroupingSubqueryMultipleAggregatorsOnSameField() .builder() .setDataSource(QueryRunnerTestHelper.dataSource) .setQuerySegmentSpec(QueryRunnerTestHelper.firstToThird) - .setDimensions(Lists.newArrayList(new DefaultDimensionSpec("quality", "alias"))) + .setDimensions(Lists.newArrayList(new DefaultDimensionSpec(qualityName, "alias"))) .setAggregatorSpecs( Arrays.asList( QueryRunnerTestHelper.rowsCount, @@ -2973,9 +3028,11 @@ public void testDifferentGroupingSubqueryMultipleAggregatorsOnSameField() List expectedResults = Arrays.asList( GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "idx1", 2900.0, "idx2", 2900.0, - "idx3", 5800.0, "idx4", 5800.0), + "idx3", 5800.0, "idx4", 5800.0 + ), GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "idx1", 2505.0, "idx2", 2505.0, - "idx3", 5010.0, "idx4", 5010.0) + "idx3", 5010.0, "idx4", 5010.0 + ) ); Iterable results = GroupByQueryRunnerTestHelper.runQuery(factory, runner, query); @@ -2990,7 +3047,7 @@ public void testDifferentGroupingSubqueryWithFilter() .builder() .setDataSource(QueryRunnerTestHelper.dataSource) .setQuerySegmentSpec(QueryRunnerTestHelper.firstToThird) - .setDimensions(Lists.newArrayList(new DefaultDimensionSpec("quality", "quality"))) + .setDimensions(Lists.newArrayList(new DefaultDimensionSpec(qualityName, qualityName))) .setAggregatorSpecs( Arrays.asList( QueryRunnerTestHelper.rowsCount, @@ -3012,15 +3069,15 @@ public void testDifferentGroupingSubqueryWithFilter() .setDimFilter( new OrDimFilter( Lists.newArrayList( - new SelectorDimFilter("quality", "automotive"), - new SelectorDimFilter("quality", "premium"), - new SelectorDimFilter("quality", "mezzanine"), - new SelectorDimFilter("quality", "business"), - new SelectorDimFilter("quality", "entertainment"), - new SelectorDimFilter("quality", "health"), - new SelectorDimFilter("quality", "news"), - new SelectorDimFilter("quality", "technology"), - new SelectorDimFilter("quality", "travel") + new SelectorDimFilter(qualityName, getDimMapping("automotive").toString()), + new SelectorDimFilter(qualityName, getDimMapping("premium").toString()), + new SelectorDimFilter(qualityName, getDimMapping("mezzanine").toString()), + new SelectorDimFilter(qualityName, getDimMapping("business").toString()), + new SelectorDimFilter(qualityName, getDimMapping("entertainment").toString()), + new SelectorDimFilter(qualityName, getDimMapping("health").toString()), + new SelectorDimFilter(qualityName, getDimMapping("news").toString()), + new SelectorDimFilter(qualityName, getDimMapping("technology").toString()), + new SelectorDimFilter(qualityName, getDimMapping("travel").toString()) ) ) ) @@ -3043,7 +3100,7 @@ public void testDifferentIntervalSubquery() .builder() .setDataSource(QueryRunnerTestHelper.dataSource) .setQuerySegmentSpec(QueryRunnerTestHelper.firstToThird) - .setDimensions(Lists.newArrayList(new DefaultDimensionSpec("quality", "alias"))) + .setDimensions(Lists.newArrayList(new DefaultDimensionSpec(qualityName, "alias"))) .setAggregatorSpecs( Arrays.asList( QueryRunnerTestHelper.rowsCount, @@ -3080,7 +3137,7 @@ public void testEmptySubquery() .builder() .setDataSource(QueryRunnerTestHelper.dataSource) .setQuerySegmentSpec(QueryRunnerTestHelper.emptyInterval) - .setDimensions(Lists.newArrayList(new DefaultDimensionSpec("quality", "alias"))) + .setDimensions(Lists.newArrayList(new DefaultDimensionSpec(qualityName, "alias"))) .setAggregatorSpecs( Arrays.asList( QueryRunnerTestHelper.rowsCount, @@ -3113,8 +3170,8 @@ public void testSubqueryWithPostAggregators() .builder() .setDataSource(QueryRunnerTestHelper.dataSource) .setQuerySegmentSpec(QueryRunnerTestHelper.firstToThird) - .setDimensions(Lists.newArrayList(new DefaultDimensionSpec("quality", "alias"))) - .setDimFilter(new JavaScriptDimFilter("quality", "function(dim){ return true; }")) + .setDimensions(Lists.newArrayList(new DefaultDimensionSpec(qualityName, "alias"))) + .setDimFilter(new JavaScriptDimFilter(qualityName, "function(dim){ return true; }")) .setAggregatorSpecs( Arrays.asList( QueryRunnerTestHelper.rowsCount, @@ -3158,13 +3215,14 @@ public void testSubqueryWithPostAggregators() ) ) .setGranularity(QueryRunnerTestHelper.dayGran) + .setContext(ImmutableMap.of("typeHints", typeHints)) .build(); List expectedResults = Arrays.asList( GroupByQueryRunnerTestHelper.createExpectedRow( "2011-04-01", "alias", - "automotive", + getDimMapping("automotive"), "rows", 1L, "idx_post", @@ -3175,7 +3233,7 @@ public void testSubqueryWithPostAggregators() GroupByQueryRunnerTestHelper.createExpectedRow( "2011-04-01", "alias", - "business", + getDimMapping("business"), "rows", 1L, "idx_post", @@ -3186,7 +3244,7 @@ public void testSubqueryWithPostAggregators() GroupByQueryRunnerTestHelper.createExpectedRow( "2011-04-01", "alias", - "entertainment", + getDimMapping("entertainment"), "rows", 1L, "idx_post", @@ -3197,7 +3255,7 @@ public void testSubqueryWithPostAggregators() GroupByQueryRunnerTestHelper.createExpectedRow( "2011-04-01", "alias", - "health", + getDimMapping("health"), "rows", 1L, "idx_post", @@ -3208,7 +3266,7 @@ public void testSubqueryWithPostAggregators() GroupByQueryRunnerTestHelper.createExpectedRow( "2011-04-01", "alias", - "mezzanine", + getDimMapping("mezzanine"), "rows", 3L, "idx_post", @@ -3219,7 +3277,7 @@ public void testSubqueryWithPostAggregators() GroupByQueryRunnerTestHelper.createExpectedRow( "2011-04-01", "alias", - "news", + getDimMapping("news"), "rows", 1L, "idx_post", @@ -3230,7 +3288,7 @@ public void testSubqueryWithPostAggregators() GroupByQueryRunnerTestHelper.createExpectedRow( "2011-04-01", "alias", - "premium", + getDimMapping("premium"), "rows", 3L, "idx_post", @@ -3241,7 +3299,7 @@ public void testSubqueryWithPostAggregators() GroupByQueryRunnerTestHelper.createExpectedRow( "2011-04-01", "alias", - "technology", + getDimMapping("technology"), "rows", 1L, "idx_post", @@ -3252,7 +3310,7 @@ public void testSubqueryWithPostAggregators() GroupByQueryRunnerTestHelper.createExpectedRow( "2011-04-01", "alias", - "travel", + getDimMapping("travel"), "rows", 1L, "idx_post", @@ -3264,7 +3322,7 @@ public void testSubqueryWithPostAggregators() GroupByQueryRunnerTestHelper.createExpectedRow( "2011-04-02", "alias", - "automotive", + getDimMapping("automotive"), "rows", 1L, "idx_post", @@ -3275,7 +3333,7 @@ public void testSubqueryWithPostAggregators() GroupByQueryRunnerTestHelper.createExpectedRow( "2011-04-02", "alias", - "business", + getDimMapping("business"), "rows", 1L, "idx_post", @@ -3286,7 +3344,7 @@ public void testSubqueryWithPostAggregators() GroupByQueryRunnerTestHelper.createExpectedRow( "2011-04-02", "alias", - "entertainment", + getDimMapping("entertainment"), "rows", 1L, "idx_post", @@ -3297,7 +3355,7 @@ public void testSubqueryWithPostAggregators() GroupByQueryRunnerTestHelper.createExpectedRow( "2011-04-02", "alias", - "health", + getDimMapping("health"), "rows", 1L, "idx_post", @@ -3308,7 +3366,7 @@ public void testSubqueryWithPostAggregators() GroupByQueryRunnerTestHelper.createExpectedRow( "2011-04-02", "alias", - "mezzanine", + getDimMapping("mezzanine"), "rows", 3L, "idx_post", @@ -3319,7 +3377,7 @@ public void testSubqueryWithPostAggregators() GroupByQueryRunnerTestHelper.createExpectedRow( "2011-04-02", "alias", - "news", + getDimMapping("news"), "rows", 1L, "idx_post", @@ -3330,7 +3388,7 @@ public void testSubqueryWithPostAggregators() GroupByQueryRunnerTestHelper.createExpectedRow( "2011-04-02", "alias", - "premium", + getDimMapping("premium"), "rows", 3L, "idx_post", @@ -3341,7 +3399,7 @@ public void testSubqueryWithPostAggregators() GroupByQueryRunnerTestHelper.createExpectedRow( "2011-04-02", "alias", - "technology", + getDimMapping("technology"), "rows", 1L, "idx_post", @@ -3352,7 +3410,7 @@ public void testSubqueryWithPostAggregators() GroupByQueryRunnerTestHelper.createExpectedRow( "2011-04-02", "alias", - "travel", + getDimMapping("travel"), "rows", 1L, "idx_post", @@ -3374,8 +3432,8 @@ public void testSubqueryWithPostAggregatorsAndHaving() .builder() .setDataSource(QueryRunnerTestHelper.dataSource) .setQuerySegmentSpec(QueryRunnerTestHelper.firstToThird) - .setDimensions(Lists.newArrayList(new DefaultDimensionSpec("quality", "alias"))) - .setDimFilter(new JavaScriptDimFilter("quality", "function(dim){ return true; }")) + .setDimensions(Lists.newArrayList(new DefaultDimensionSpec(qualityName, "alias"))) + .setDimFilter(new JavaScriptDimFilter(qualityName, "function(dim){ return true; }")) .setAggregatorSpecs( Arrays.asList( QueryRunnerTestHelper.rowsCount, @@ -3438,13 +3496,14 @@ public byte[] getCacheKey() ) ) .setGranularity(QueryRunnerTestHelper.dayGran) + .setContext(ImmutableMap.of("typeHints", typeHints)) .build(); List expectedResults = Arrays.asList( GroupByQueryRunnerTestHelper.createExpectedRow( "2011-04-01", "alias", - "automotive", + getDimMapping("automotive"), "rows", 1L, "idx_post", @@ -3455,7 +3514,7 @@ public byte[] getCacheKey() GroupByQueryRunnerTestHelper.createExpectedRow( "2011-04-01", "alias", - "business", + getDimMapping("business"), "rows", 1L, "idx_post", @@ -3466,7 +3525,7 @@ public byte[] getCacheKey() GroupByQueryRunnerTestHelper.createExpectedRow( "2011-04-01", "alias", - "entertainment", + getDimMapping("entertainment"), "rows", 1L, "idx_post", @@ -3477,7 +3536,7 @@ public byte[] getCacheKey() GroupByQueryRunnerTestHelper.createExpectedRow( "2011-04-01", "alias", - "health", + getDimMapping("health"), "rows", 1L, "idx_post", @@ -3488,7 +3547,7 @@ public byte[] getCacheKey() GroupByQueryRunnerTestHelper.createExpectedRow( "2011-04-01", "alias", - "news", + getDimMapping("news"), "rows", 1L, "idx_post", @@ -3499,7 +3558,7 @@ public byte[] getCacheKey() GroupByQueryRunnerTestHelper.createExpectedRow( "2011-04-01", "alias", - "technology", + getDimMapping("technology"), "rows", 1L, "idx_post", @@ -3510,7 +3569,7 @@ public byte[] getCacheKey() GroupByQueryRunnerTestHelper.createExpectedRow( "2011-04-01", "alias", - "travel", + getDimMapping("travel"), "rows", 1L, "idx_post", @@ -3522,7 +3581,7 @@ public byte[] getCacheKey() GroupByQueryRunnerTestHelper.createExpectedRow( "2011-04-02", "alias", - "automotive", + getDimMapping("automotive"), "rows", 1L, "idx_post", @@ -3533,7 +3592,7 @@ public byte[] getCacheKey() GroupByQueryRunnerTestHelper.createExpectedRow( "2011-04-02", "alias", - "business", + getDimMapping("business"), "rows", 1L, "idx_post", @@ -3544,7 +3603,7 @@ public byte[] getCacheKey() GroupByQueryRunnerTestHelper.createExpectedRow( "2011-04-02", "alias", - "entertainment", + getDimMapping("entertainment"), "rows", 1L, "idx_post", @@ -3555,7 +3614,7 @@ public byte[] getCacheKey() GroupByQueryRunnerTestHelper.createExpectedRow( "2011-04-02", "alias", - "health", + getDimMapping("health"), "rows", 1L, "idx_post", @@ -3566,7 +3625,7 @@ public byte[] getCacheKey() GroupByQueryRunnerTestHelper.createExpectedRow( "2011-04-02", "alias", - "mezzanine", + getDimMapping("mezzanine"), "rows", 3L, "idx_post", @@ -3577,7 +3636,7 @@ public byte[] getCacheKey() GroupByQueryRunnerTestHelper.createExpectedRow( "2011-04-02", "alias", - "news", + getDimMapping("news"), "rows", 1L, "idx_post", @@ -3588,7 +3647,7 @@ public byte[] getCacheKey() GroupByQueryRunnerTestHelper.createExpectedRow( "2011-04-02", "alias", - "premium", + getDimMapping("premium"), "rows", 3L, "idx_post", @@ -3599,7 +3658,7 @@ public byte[] getCacheKey() GroupByQueryRunnerTestHelper.createExpectedRow( "2011-04-02", "alias", - "technology", + getDimMapping("technology"), "rows", 1L, "idx_post", @@ -3610,7 +3669,7 @@ public byte[] getCacheKey() GroupByQueryRunnerTestHelper.createExpectedRow( "2011-04-02", "alias", - "travel", + getDimMapping("travel"), "rows", 1L, "idx_post", @@ -3632,8 +3691,8 @@ public void testSubqueryWithMultiColumnAggregators() .builder() .setDataSource(QueryRunnerTestHelper.dataSource) .setQuerySegmentSpec(QueryRunnerTestHelper.firstToThird) - .setDimensions(Lists.newArrayList(new DefaultDimensionSpec("quality", "alias"))) - .setDimFilter(new JavaScriptDimFilter("market", "function(dim){ return true; }")) + .setDimensions(Lists.newArrayList(new DefaultDimensionSpec(qualityName, "alias"))) + .setDimFilter(new JavaScriptDimFilter(marketName, "function(dim){ return true; }")) .setAggregatorSpecs( Arrays.asList( QueryRunnerTestHelper.rowsCount, @@ -3714,6 +3773,7 @@ public byte[] getCacheKey() 5 ) ) + .setContext(ImmutableMap.of("typeHints", typeHints)) .setGranularity(QueryRunnerTestHelper.dayGran) .build(); @@ -3721,7 +3781,7 @@ public byte[] getCacheKey() GroupByQueryRunnerTestHelper.createExpectedRow( "2011-04-01", "alias", - "travel", + getDimMapping("travel"), "rows", 1L, "idx_post", @@ -3734,7 +3794,7 @@ public byte[] getCacheKey() GroupByQueryRunnerTestHelper.createExpectedRow( "2011-04-01", "alias", - "technology", + getDimMapping("technology"), "rows", 1L, "idx_post", @@ -3747,7 +3807,7 @@ public byte[] getCacheKey() GroupByQueryRunnerTestHelper.createExpectedRow( "2011-04-01", "alias", - "news", + getDimMapping("news"), "rows", 1L, "idx_post", @@ -3760,7 +3820,7 @@ public byte[] getCacheKey() GroupByQueryRunnerTestHelper.createExpectedRow( "2011-04-01", "alias", - "health", + getDimMapping("health"), "rows", 1L, "idx_post", @@ -3773,7 +3833,7 @@ public byte[] getCacheKey() GroupByQueryRunnerTestHelper.createExpectedRow( "2011-04-01", "alias", - "entertainment", + getDimMapping("entertainment"), "rows", 1L, "idx_post", @@ -3797,7 +3857,7 @@ public void testSubqueryWithHyperUniques() .builder() .setDataSource(QueryRunnerTestHelper.dataSource) .setQuerySegmentSpec(QueryRunnerTestHelper.firstToThird) - .setDimensions(Lists.newArrayList(new DefaultDimensionSpec("quality", "alias"))) + .setDimensions(Lists.newArrayList(new DefaultDimensionSpec(qualityName, "alias"))) .setAggregatorSpecs( Arrays.asList( QueryRunnerTestHelper.rowsCount, @@ -3820,6 +3880,7 @@ public void testSubqueryWithHyperUniques() new HyperUniquesAggregatorFactory("uniq", "quality_uniques") ) ) + .setContext(ImmutableMap.of("typeHints", typeHints)) .setGranularity(QueryRunnerTestHelper.allGran) .build(); @@ -3827,7 +3888,7 @@ public void testSubqueryWithHyperUniques() GroupByQueryRunnerTestHelper.createExpectedRow( "2011-04-01", "alias", - "automotive", + getDimMapping("automotive"), "rows", 2L, "idx", @@ -3838,7 +3899,7 @@ public void testSubqueryWithHyperUniques() GroupByQueryRunnerTestHelper.createExpectedRow( "2011-04-01", "alias", - "business", + getDimMapping("business"), "rows", 2L, "idx", @@ -3849,7 +3910,7 @@ public void testSubqueryWithHyperUniques() GroupByQueryRunnerTestHelper.createExpectedRow( "2011-04-01", "alias", - "entertainment", + getDimMapping("entertainment"), "rows", 2L, "idx", @@ -3860,7 +3921,7 @@ public void testSubqueryWithHyperUniques() GroupByQueryRunnerTestHelper.createExpectedRow( "2011-04-01", "alias", - "health", + getDimMapping("health"), "rows", 2L, "idx", @@ -3871,7 +3932,7 @@ public void testSubqueryWithHyperUniques() GroupByQueryRunnerTestHelper.createExpectedRow( "2011-04-01", "alias", - "mezzanine", + getDimMapping("mezzanine"), "rows", 6L, "idx", @@ -3882,7 +3943,7 @@ public void testSubqueryWithHyperUniques() GroupByQueryRunnerTestHelper.createExpectedRow( "2011-04-01", "alias", - "news", + getDimMapping("news"), "rows", 2L, "idx", @@ -3893,7 +3954,7 @@ public void testSubqueryWithHyperUniques() GroupByQueryRunnerTestHelper.createExpectedRow( "2011-04-01", "alias", - "premium", + getDimMapping("premium"), "rows", 6L, "idx", @@ -3904,7 +3965,7 @@ public void testSubqueryWithHyperUniques() GroupByQueryRunnerTestHelper.createExpectedRow( "2011-04-01", "alias", - "technology", + getDimMapping("technology"), "rows", 2L, "idx", @@ -3915,7 +3976,7 @@ public void testSubqueryWithHyperUniques() GroupByQueryRunnerTestHelper.createExpectedRow( "2011-04-01", "alias", - "travel", + getDimMapping("travel"), "rows", 2L, "idx", @@ -4034,7 +4095,7 @@ public void testGroupByTimeExtraction() .setQuerySegmentSpec(QueryRunnerTestHelper.fullOnInterval) .setDimensions( Lists.newArrayList( - new DefaultDimensionSpec("market", "market"), + new DefaultDimensionSpec(marketName, marketName), new ExtractionDimensionSpec( Column.TIME_COLUMN_NAME, "dayOfWeek", @@ -4051,11 +4112,12 @@ public void testGroupByTimeExtraction() ) .setPostAggregatorSpecs(Arrays.asList(QueryRunnerTestHelper.addRowsIndexConstant)) .setGranularity(QueryRunnerTestHelper.allGran) + .setContext(ImmutableMap.of("typeHints", typeHints)) .setDimFilter( new OrDimFilter( Arrays.asList( - new SelectorDimFilter("market", "spot"), - new SelectorDimFilter("market", "upfront") + new SelectorDimFilter(marketName, getDimMapping("spot").toString()), + new SelectorDimFilter(marketName, getDimMapping("upfront").toString()) ) ) ) @@ -4066,8 +4128,8 @@ public void testGroupByTimeExtraction() "1970-01-01", "dayOfWeek", "Friday", - "market", - "spot", + marketName, + getDimMapping("spot"), "index", 13219.574157714844, "rows", @@ -4079,8 +4141,8 @@ public void testGroupByTimeExtraction() "1970-01-01", "dayOfWeek", "Monday", - "market", - "spot", + marketName, + getDimMapping("spot"), "index", 13557.738830566406, "rows", @@ -4092,8 +4154,8 @@ public void testGroupByTimeExtraction() "1970-01-01", "dayOfWeek", "Saturday", - "market", - "spot", + marketName, + getDimMapping("spot"), "index", 13493.751281738281, "rows", @@ -4105,8 +4167,8 @@ public void testGroupByTimeExtraction() "1970-01-01", "dayOfWeek", "Sunday", - "market", - "spot", + marketName, + getDimMapping("spot"), "index", 13585.541015625, "rows", @@ -4118,8 +4180,8 @@ public void testGroupByTimeExtraction() "1970-01-01", "dayOfWeek", "Thursday", - "market", - "spot", + marketName, + getDimMapping("spot"), "index", 14279.127197265625, "rows", @@ -4131,8 +4193,8 @@ public void testGroupByTimeExtraction() "1970-01-01", "dayOfWeek", "Tuesday", - "market", - "spot", + marketName, + getDimMapping("spot"), "index", 13199.471435546875, "rows", @@ -4144,8 +4206,8 @@ public void testGroupByTimeExtraction() "1970-01-01", "dayOfWeek", "Wednesday", - "market", - "spot", + marketName, + getDimMapping("spot"), "index", 14271.368591308594, "rows", @@ -4157,8 +4219,8 @@ public void testGroupByTimeExtraction() "1970-01-01", "dayOfWeek", "Friday", - "market", - "upfront", + marketName, + getDimMapping("upfront"), "index", 27297.8623046875, "rows", @@ -4170,8 +4232,8 @@ public void testGroupByTimeExtraction() "1970-01-01", "dayOfWeek", "Monday", - "market", - "upfront", + marketName, + getDimMapping("upfront"), "index", 27619.58447265625, "rows", @@ -4183,8 +4245,8 @@ public void testGroupByTimeExtraction() "1970-01-01", "dayOfWeek", "Saturday", - "market", - "upfront", + marketName, + getDimMapping("upfront"), "index", 27820.83154296875, "rows", @@ -4196,8 +4258,8 @@ public void testGroupByTimeExtraction() "1970-01-01", "dayOfWeek", "Sunday", - "market", - "upfront", + marketName, + getDimMapping("upfront"), "index", 24791.223876953125, "rows", @@ -4209,8 +4271,8 @@ public void testGroupByTimeExtraction() "1970-01-01", "dayOfWeek", "Thursday", - "market", - "upfront", + marketName, + getDimMapping("upfront"), "index", 28562.748901367188, "rows", @@ -4222,8 +4284,8 @@ public void testGroupByTimeExtraction() "1970-01-01", "dayOfWeek", "Tuesday", - "market", - "upfront", + marketName, + getDimMapping("upfront"), "index", 26968.280639648438, "rows", @@ -4235,8 +4297,8 @@ public void testGroupByTimeExtraction() "1970-01-01", "dayOfWeek", "Wednesday", - "market", - "upfront", + marketName, + getDimMapping("upfront"), "index", 28985.5751953125, "rows", @@ -4261,7 +4323,7 @@ public void testBySegmentResults() GroupByQueryRunnerTestHelper.createExpectedRow( "2011-04-01", "alias", - "mezzanine", + getDimMapping("mezzanine"), "rows", 6L, "idx", @@ -4278,7 +4340,7 @@ public void testBySegmentResults() .builder() .setDataSource(QueryRunnerTestHelper.dataSource) .setInterval("2011-04-02/2011-04-04") - .setDimensions(Lists.newArrayList(new DefaultDimensionSpec("quality", "alias"))) + .setDimensions(Lists.newArrayList(new DefaultDimensionSpec(qualityName, "alias"))) .setAggregatorSpecs( Arrays.asList( QueryRunnerTestHelper.rowsCount, @@ -4286,7 +4348,7 @@ public void testBySegmentResults() ) ) .setGranularity(new PeriodGranularity(new Period("P1M"), null, null)) - .setDimFilter(new SelectorDimFilter("quality", "mezzanine")) + .setDimFilter(new SelectorDimFilter(qualityName, getDimMapping("mezzanine").toString())) .setContext(ImmutableMap.of("bySegment", true)); final GroupByQuery fullQuery = builder.build(); QueryToolChest toolChest = factory.getToolchest(); @@ -4307,7 +4369,6 @@ public void testBySegmentResults() exec.shutdownNow(); } - @Test public void testBySegmentResultsUnOptimizedDimextraction() { @@ -4339,12 +4400,12 @@ public void testBySegmentResultsUnOptimizedDimextraction() .setDimensions( Lists.newArrayList( new ExtractionDimensionSpec( - "quality", + qualityName, "alias", new LookupExtractionFn( new MapLookupExtractor( ImmutableMap.of( - "mezzanine", + getDimMapping("mezzanine").toString(), "mezzanine0" ), false @@ -4362,7 +4423,7 @@ public void testBySegmentResultsUnOptimizedDimextraction() ) ) .setGranularity(new PeriodGranularity(new Period("P1M"), null, null)) - .setDimFilter(new SelectorDimFilter("quality", "mezzanine")) + .setDimFilter(new SelectorDimFilter(qualityName, getDimMapping("mezzanine").toString())) .setContext(ImmutableMap.of("bySegment", true)); final GroupByQuery fullQuery = builder.build(); QueryToolChest toolChest = factory.getToolchest(); @@ -4379,7 +4440,9 @@ public void testBySegmentResultsUnOptimizedDimextraction() ) ); - TestHelper.assertExpectedObjects(bySegmentResults, theRunner.run(fullQuery, Maps.newHashMap()), ""); + HashMap ctx = Maps.newHashMap(); + + TestHelper.assertExpectedObjects(bySegmentResults, theRunner.run(fullQuery, ctx), ""); exec.shutdownNow(); } @@ -4414,12 +4477,12 @@ public void testBySegmentResultsOptimizedDimextraction() .setDimensions( Lists.newArrayList( new ExtractionDimensionSpec( - "quality", + qualityName, "alias", new LookupExtractionFn( new MapLookupExtractor( ImmutableMap.of( - "mezzanine", + getDimMapping("mezzanine").toString(), "mezzanine0" ), false @@ -4437,7 +4500,7 @@ public void testBySegmentResultsOptimizedDimextraction() ) ) .setGranularity(new PeriodGranularity(new Period("P1M"), null, null)) - .setDimFilter(new SelectorDimFilter("quality", "mezzanine")) + .setDimFilter(new SelectorDimFilter(qualityName, getDimMapping("mezzanine").toString())) .setContext(ImmutableMap.of("bySegment", true)); final GroupByQuery fullQuery = builder.build(); QueryToolChest toolChest = factory.getToolchest(); @@ -4464,21 +4527,21 @@ public void testBySegmentResultsOptimizedDimextraction() public void testGroupByWithExtractionDimFilter() { Map extractionMap = new HashMap<>(); - extractionMap.put("automotive", "automotiveAndBusinessAndNewsAndMezzanine"); - extractionMap.put("business", "automotiveAndBusinessAndNewsAndMezzanine"); - extractionMap.put("mezzanine", "automotiveAndBusinessAndNewsAndMezzanine"); - extractionMap.put("news", "automotiveAndBusinessAndNewsAndMezzanine"); + extractionMap.put(getDimMapping("automotive").toString(), "automotiveAndBusinessAndNewsAndMezzanine"); + extractionMap.put(getDimMapping("business").toString(), "automotiveAndBusinessAndNewsAndMezzanine"); + extractionMap.put(getDimMapping("mezzanine").toString(), "automotiveAndBusinessAndNewsAndMezzanine"); + extractionMap.put(getDimMapping("news").toString(), "automotiveAndBusinessAndNewsAndMezzanine"); MapLookupExtractor mapLookupExtractor = new MapLookupExtractor(extractionMap, false); LookupExtractionFn lookupExtractionFn = new LookupExtractionFn(mapLookupExtractor, false, null, true, false); List dimFilters = Lists.newArrayList( - new ExtractionDimFilter("quality", "automotiveAndBusinessAndNewsAndMezzanine", lookupExtractionFn, null), - new SelectorDimFilter("quality", "entertainment"), - new SelectorDimFilter("quality", "health"), - new SelectorDimFilter("quality", "premium"), - new SelectorDimFilter("quality", "technology"), - new SelectorDimFilter("quality", "travel") + new ExtractionDimFilter(qualityName, "automotiveAndBusinessAndNewsAndMezzanine", lookupExtractionFn, null), + new SelectorDimFilter(qualityName, getDimMapping("entertainment").toString()), + new SelectorDimFilter(qualityName, getDimMapping("health").toString()), + new SelectorDimFilter(qualityName, getDimMapping("premium").toString()), + new SelectorDimFilter(qualityName, getDimMapping("technology").toString()), + new SelectorDimFilter(qualityName, getDimMapping("travel").toString()) ); GroupByQuery query = GroupByQuery.builder().setDataSource(QueryRunnerTestHelper.dataSource) @@ -4486,7 +4549,7 @@ public void testGroupByWithExtractionDimFilter() .setDimensions( Lists.newArrayList( new DefaultDimensionSpec( - "quality", + qualityName, "alias" ) ) @@ -4499,27 +4562,28 @@ public void testGroupByWithExtractionDimFilter() ) .setGranularity(QueryRunnerTestHelper.dayGran) .setDimFilter(Druids.newOrDimFilterBuilder().fields(dimFilters).build()) + .setContext(ImmutableMap.of("typeHints", typeHints)) .build(); List expectedResults = Arrays.asList( - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "automotive", "rows", 1L, "idx", 135L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "business", "rows", 1L, "idx", 118L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "entertainment", "rows", 1L, "idx", 158L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "health", "rows", 1L, "idx", 120L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "mezzanine", "rows", 3L, "idx", 2870L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "news", "rows", 1L, "idx", 121L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "premium", "rows", 3L, "idx", 2900L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "technology", "rows", 1L, "idx", 78L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "travel", "rows", 1L, "idx", 119L), - - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "automotive", "rows", 1L, "idx", 147L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "business", "rows", 1L, "idx", 112L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "entertainment", "rows", 1L, "idx", 166L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "health", "rows", 1L, "idx", 113L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "mezzanine", "rows", 3L, "idx", 2447L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "news", "rows", 1L, "idx", 114L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "premium", "rows", 3L, "idx", 2505L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "technology", "rows", 1L, "idx", 97L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "travel", "rows", 1L, "idx", 126L) + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", getDimMapping("automotive"), "rows", 1L, "idx", 135L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", getDimMapping("business"), "rows", 1L, "idx", 118L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", getDimMapping("entertainment"), "rows", 1L, "idx", 158L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", getDimMapping("health"), "rows", 1L, "idx", 120L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", getDimMapping("mezzanine"), "rows", 3L, "idx", 2870L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", getDimMapping("news"), "rows", 1L, "idx", 121L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", getDimMapping("premium"), "rows", 3L, "idx", 2900L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", getDimMapping("technology"), "rows", 1L, "idx", 78L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", getDimMapping("travel"), "rows", 1L, "idx", 119L), + + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", getDimMapping("automotive"), "rows", 1L, "idx", 147L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", getDimMapping("business"), "rows", 1L, "idx", 112L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", getDimMapping("entertainment"), "rows", 1L, "idx", 166L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", getDimMapping("health"), "rows", 1L, "idx", 113L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", getDimMapping("mezzanine"), "rows", 3L, "idx", 2447L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", getDimMapping("news"), "rows", 1L, "idx", 114L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", getDimMapping("premium"), "rows", 3L, "idx", 2505L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", getDimMapping("technology"), "rows", 1L, "idx", 97L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", getDimMapping("travel"), "rows", 1L, "idx", 126L) ); Iterable results = GroupByQueryRunnerTestHelper.runQuery(factory, runner, query); @@ -4530,16 +4594,9 @@ public void testGroupByWithExtractionDimFilter() @Test public void testGroupByWithExtractionDimFilterCaseMappingValueIsNullOrEmpty() { - Map extractionMap = new HashMap<>(); - extractionMap.put("automotive", "automotive0"); - extractionMap.put("business", "business0"); - extractionMap.put("entertainment", "entertainment0"); - extractionMap.put("health", "health0"); - extractionMap.put("mezzanine", null); - extractionMap.put("news", ""); - extractionMap.put("premium", "premium0"); - extractionMap.put("technology", "technology0"); - extractionMap.put("travel", "travel0"); + Map extractionMap = Maps.newHashMap(QUALITY_RENAMING_MAP); + extractionMap.put(getDimMapping("mezzanine").toString(), null); + extractionMap.put(getDimMapping("news").toString(), ""); MapLookupExtractor mapLookupExtractor = new MapLookupExtractor(extractionMap, false); LookupExtractionFn lookupExtractionFn = new LookupExtractionFn(mapLookupExtractor, false, null, true, false); @@ -4548,7 +4605,7 @@ public void testGroupByWithExtractionDimFilterCaseMappingValueIsNullOrEmpty() .setDimensions( Lists.newArrayList( new DefaultDimensionSpec( - "quality", + qualityName, "alias" ) ) @@ -4560,13 +4617,14 @@ public void testGroupByWithExtractionDimFilterCaseMappingValueIsNullOrEmpty() ) ) .setGranularity(QueryRunnerTestHelper.dayGran) - .setDimFilter(new ExtractionDimFilter("quality", "", lookupExtractionFn, null)) + .setDimFilter(new ExtractionDimFilter(qualityName, "", lookupExtractionFn, null)) + .setContext(ImmutableMap.of("typeHints", typeHints)) .build(); List expectedResults = Arrays.asList( - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "mezzanine", "rows", 3L, "idx", 2870L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "news", "rows", 1L, "idx", 121L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "mezzanine", "rows", 3L, "idx", 2447L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "news", "rows", 1L, "idx", 114L) + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", getDimMapping("mezzanine"), "rows", 3L, "idx", 2870L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", getDimMapping("news"), "rows", 1L, "idx", 121L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", getDimMapping("mezzanine"), "rows", 3L, "idx", 2447L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", getDimMapping("news"), "rows", 1L, "idx", 114L) ); Iterable results = GroupByQueryRunnerTestHelper.runQuery(factory, runner, query); @@ -4585,7 +4643,7 @@ public void testGroupByWithExtractionDimFilterWhenSearchValueNotInTheMap() .setDimensions( Lists.newArrayList( new DefaultDimensionSpec( - "quality", + qualityName, "alias" ) ) @@ -4599,7 +4657,7 @@ public void testGroupByWithExtractionDimFilterWhenSearchValueNotInTheMap() .setGranularity(QueryRunnerTestHelper.dayGran) .setDimFilter( new ExtractionDimFilter( - "quality", + qualityName, "NOT_THERE", lookupExtractionFn, null @@ -4659,26 +4717,19 @@ public void testGroupByWithExtractionDimFilterKeyisNull() @Test public void testGroupByWithAggregatorFilterAndExtractionFunction() { - Map extractionMap = new HashMap<>(); - extractionMap.put("automotive", "automotive0"); - extractionMap.put("business", "business0"); - extractionMap.put("entertainment", "entertainment0"); - extractionMap.put("health", "health0"); - extractionMap.put("mezzanine", "mezzanineANDnews"); - extractionMap.put("news", "mezzanineANDnews"); - extractionMap.put("premium", "premium0"); - extractionMap.put("technology", "technology0"); - extractionMap.put("travel", "travel0"); + Map extractionMap = Maps.newHashMap(QUALITY_RENAMING_MAP); + extractionMap.put(getDimMapping("mezzanine").toString(), "mezzanineANDnews"); + extractionMap.put(getDimMapping("news").toString(), "mezzanineANDnews"); MapLookupExtractor mapLookupExtractor = new MapLookupExtractor(extractionMap, false); LookupExtractionFn lookupExtractionFn = new LookupExtractionFn(mapLookupExtractor, false, "missing", true, false); - DimFilter filter = new ExtractionDimFilter("quality","mezzanineANDnews",lookupExtractionFn,null); + DimFilter filter = new ExtractionDimFilter(qualityName,"mezzanineANDnews",lookupExtractionFn,null); GroupByQuery query = GroupByQuery.builder().setDataSource(QueryRunnerTestHelper.dataSource) .setQuerySegmentSpec(QueryRunnerTestHelper.firstToThird) .setDimensions( Lists.newArrayList( new DefaultDimensionSpec( - "quality", + qualityName, "alias" ) ) @@ -4695,27 +4746,28 @@ public void testGroupByWithAggregatorFilterAndExtractionFunction() ) ) .setGranularity(QueryRunnerTestHelper.dayGran) + .setContext(ImmutableMap.of("typeHints", typeHints)) .build(); List expectedResults = Arrays.asList( - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "automotive", "rows", 0L, "idx", 0L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "business", "rows", 0L, "idx", 0L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "entertainment", "rows", 0L, "idx", 0L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "health", "rows", 0L, "idx", 0L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "mezzanine", "rows", 3L, "idx", 2870L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "news", "rows", 1L, "idx", 121L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "premium", "rows", 0L, "idx", 0L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "technology", "rows", 0L, "idx", 0L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "travel", "rows", 0L, "idx", 0L), - - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "automotive", "rows", 0L, "idx", 0L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "business", "rows", 0L, "idx", 0L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "entertainment", "rows", 0L, "idx", 0L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "health", "rows", 0L, "idx", 0L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "mezzanine", "rows", 3L, "idx", 2447L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "news", "rows", 1L, "idx", 114L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "premium", "rows", 0L, "idx", 0L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "technology", "rows", 0L, "idx", 0L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "travel", "rows", 0L, "idx", 0L) + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", getDimMapping("automotive"), "rows", 0L, "idx", 0L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", getDimMapping("business"), "rows", 0L, "idx", 0L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", getDimMapping("entertainment"), "rows", 0L, "idx", 0L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", getDimMapping("health"), "rows", 0L, "idx", 0L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", getDimMapping("mezzanine"), "rows", 3L, "idx", 2870L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", getDimMapping("news"), "rows", 1L, "idx", 121L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", getDimMapping("premium"), "rows", 0L, "idx", 0L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", getDimMapping("technology"), "rows", 0L, "idx", 0L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", getDimMapping("travel"), "rows", 0L, "idx", 0L), + + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", getDimMapping("automotive"), "rows", 0L, "idx", 0L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", getDimMapping("business"), "rows", 0L, "idx", 0L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", getDimMapping("entertainment"), "rows", 0L, "idx", 0L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", getDimMapping("health"), "rows", 0L, "idx", 0L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", getDimMapping("mezzanine"), "rows", 3L, "idx", 2447L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", getDimMapping("news"), "rows", 1L, "idx", 114L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", getDimMapping("premium"), "rows", 0L, "idx", 0L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", getDimMapping("technology"), "rows", 0L, "idx", 0L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", getDimMapping("travel"), "rows", 0L, "idx", 0L) ); Iterable results = GroupByQueryRunnerTestHelper.runQuery(factory, runner, query); @@ -4727,24 +4779,25 @@ public void testGroupByWithAggregatorFilterAndExtractionFunction() public void testGroupByWithExtractionDimFilterOptimazitionManyToOne() { Map extractionMap = new HashMap<>(); - extractionMap.put("mezzanine", "newsANDmezzanine"); - extractionMap.put("news", "newsANDmezzanine"); + extractionMap.put(getDimMapping("mezzanine").toString(), "newsANDmezzanine"); + extractionMap.put(getDimMapping("news").toString(), "newsANDmezzanine"); MapLookupExtractor mapLookupExtractor = new MapLookupExtractor(extractionMap, false); LookupExtractionFn lookupExtractionFn = new LookupExtractionFn(mapLookupExtractor, false, null, true, true); GroupByQuery query = GroupByQuery.builder().setDataSource(QueryRunnerTestHelper.dataSource) .setQuerySegmentSpec(QueryRunnerTestHelper.firstToThird) - .setDimensions(Lists.newArrayList(new DefaultDimensionSpec("quality", "alias"))) + .setDimensions(Lists.newArrayList(new DefaultDimensionSpec(qualityName, "alias"))) .setAggregatorSpecs( Arrays.asList(QueryRunnerTestHelper.rowsCount, new LongSumAggregatorFactory("idx", "index"))) .setGranularity(QueryRunnerTestHelper.dayGran) - .setDimFilter(new ExtractionDimFilter("quality", "newsANDmezzanine", lookupExtractionFn, null)) + .setDimFilter(new ExtractionDimFilter(qualityName, "newsANDmezzanine", lookupExtractionFn, null)) + .setContext(ImmutableMap.of("typeHints", typeHints)) .build(); List expectedResults = Arrays.asList( - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "mezzanine", "rows", 3L, "idx", 2870L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "news", "rows", 1L, "idx", 121L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "mezzanine", "rows", 3L, "idx", 2447L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "news", "rows", 1L, "idx", 114L)); + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", getDimMapping("mezzanine"), "rows", 3L, "idx", 2870L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", getDimMapping("news"), "rows", 1L, "idx", 121L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", getDimMapping("mezzanine"), "rows", 3L, "idx", 2447L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", getDimMapping("news"), "rows", 1L, "idx", 114L)); Iterable results = GroupByQueryRunnerTestHelper.runQuery(factory, runner, query); TestHelper.assertExpectedObjects(expectedResults, results, ""); @@ -4761,14 +4814,27 @@ public void testGroupByWithExtractionDimFilterOptimazitionManyToOne() GroupByQuery query = GroupByQuery.builder().setDataSource(QueryRunnerTestHelper.dataSource) .setQuerySegmentSpec(QueryRunnerTestHelper.firstToThird) - .setDimensions(Lists.newArrayList(new DefaultDimensionSpec("null_column", "alias"))) + .setDimensions(Lists.newArrayList(new DefaultDimensionSpec( + "null_column", + "alias" + ))) .setAggregatorSpecs( - Arrays.asList(QueryRunnerTestHelper.rowsCount, new LongSumAggregatorFactory("idx", "index"))) + Arrays.asList( + QueryRunnerTestHelper.rowsCount, + new LongSumAggregatorFactory("idx", "index") + )) .setGranularity(QueryRunnerTestHelper.dayGran) - .setDimFilter(new ExtractionDimFilter("null_column", "EMPTY", lookupExtractionFn, null)).build(); + .setDimFilter(new ExtractionDimFilter( + "null_column", + "EMPTY", + lookupExtractionFn, + null + )).build(); List expectedResults = Arrays - .asList(GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", null, "rows", 13L, "idx", 6619L), - GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", null, "rows", 13L, "idx", 5827L)); + .asList( + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", null, "rows", 13L, "idx", 6619L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", null, "rows", 13L, "idx", 5827L) + ); Iterable results = GroupByQueryRunnerTestHelper.runQuery(factory, runner, query); TestHelper.assertExpectedObjects(expectedResults, results, ""); diff --git a/processing/src/test/java/io/druid/query/metadata/SegmentAnalyzerTest.java b/processing/src/test/java/io/druid/query/metadata/SegmentAnalyzerTest.java index b7beeebeed44..c66e2bc120bb 100644 --- a/processing/src/test/java/io/druid/query/metadata/SegmentAnalyzerTest.java +++ b/processing/src/test/java/io/druid/query/metadata/SegmentAnalyzerTest.java @@ -75,16 +75,22 @@ private void testIncrementalWorksHelper(EnumSet 0); + if (typeName.equals(ValueType.STRING.name())) { + Assert.assertTrue(dimension, columnAnalysis.getCardinality() > 0); + } } else { - Assert.assertEquals(dimension, 0, columnAnalysis.getCardinality().longValue()); + if (typeName.equals(ValueType.STRING.name())) { + Assert.assertEquals(dimension, 0, columnAnalysis.getCardinality().longValue()); + } Assert.assertEquals(dimension, 0, columnAnalysis.getSize()); } + dimIndex++; } for (String metric : TestIndex.METRICS) { @@ -125,20 +131,27 @@ private void testMappedWorksHelper(EnumSet an columns.size() ); // All columns including time and excluding empty/null column + int dimIndex = 0; for (String dimension : TestIndex.DIMENSIONS) { final ColumnAnalysis columnAnalysis = columns.get(dimension); if (dimension.equals("null_column")) { Assert.assertNull(columnAnalysis); } else { - Assert.assertEquals(dimension, ValueType.STRING.name(), columnAnalysis.getType()); + String typeName = TestIndex.DIMENSION_SCHEMAS[dimIndex].getValueType().toString(); + Assert.assertEquals(dimension, typeName, columnAnalysis.getType()); if (analyses == null) { Assert.assertTrue(dimension, columnAnalysis.getSize() > 0); - Assert.assertTrue(dimension, columnAnalysis.getCardinality() > 0); + if (typeName.equals(ValueType.STRING.name())) { + Assert.assertTrue(dimension, columnAnalysis.getCardinality() > 0); + } } else { - Assert.assertEquals(dimension, 0, columnAnalysis.getCardinality().longValue()); + if (typeName.equals(ValueType.STRING.name())) { + Assert.assertEquals(dimension, 0, columnAnalysis.getCardinality().longValue()); + } Assert.assertEquals(dimension, 0, columnAnalysis.getSize()); } } + dimIndex++; } for (String metric : TestIndex.METRICS) { diff --git a/processing/src/test/java/io/druid/query/metadata/SegmentMetadataQueryTest.java b/processing/src/test/java/io/druid/query/metadata/SegmentMetadataQueryTest.java index 628fcfa6a207..733c0a99b2c8 100644 --- a/processing/src/test/java/io/druid/query/metadata/SegmentMetadataQueryTest.java +++ b/processing/src/test/java/io/druid/query/metadata/SegmentMetadataQueryTest.java @@ -177,7 +177,7 @@ public SegmentMetadataQueryTest( null, null ) - ), mmap1 ? 71982 : 72755, + ), mmap1 ? 115506 : 116279, 1209, null ); @@ -218,7 +218,7 @@ public SegmentMetadataQueryTest( null ) // null_column will be included only for incremental index, which makes a little bigger result than expected - ), mmap2 ? 71982 : 72755, + ), mmap2 ? 115506 : 116279, 1209, null ); diff --git a/processing/src/test/java/io/druid/query/search/SearchQueryRunnerTest.java b/processing/src/test/java/io/druid/query/search/SearchQueryRunnerTest.java index d1456a720fcc..4b3fc35c5470 100644 --- a/processing/src/test/java/io/druid/query/search/SearchQueryRunnerTest.java +++ b/processing/src/test/java/io/druid/query/search/SearchQueryRunnerTest.java @@ -22,6 +22,8 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import com.google.common.collect.Sets; import com.metamx.common.guava.Sequence; import com.metamx.common.guava.Sequences; import com.metamx.common.logger.Logger; @@ -30,6 +32,7 @@ import io.druid.query.QueryRunner; import io.druid.query.QueryRunnerTestHelper; import io.druid.query.Result; +import io.druid.query.dimension.DimensionSpec; import io.druid.query.dimension.ExtractionDimensionSpec; import io.druid.query.lookup.LookupExtractionFn; import io.druid.query.extraction.MapLookupExtractor; @@ -53,9 +56,12 @@ import org.junit.runners.Parameterized; import java.io.IOException; +import java.util.ArrayList; import java.util.Arrays; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; /** */ @@ -548,6 +554,97 @@ public void testSearchNonExistingDimension() ); } + + @Test + public void testSearchLongFloatInMultiDims() + { + SearchQuery searchQuery = Druids.newSearchQueryBuilder() + .dataSource(QueryRunnerTestHelper.dataSource) + .granularity(QueryRunnerTestHelper.allGran) + .intervals(QueryRunnerTestHelper.fullOnInterval) + .dimensions( + Arrays.asList( + "quality_long", + "quality_float", + "market_long", + "market_float" + ) + ) + .query("22") + .build(); + + List expectedHits = Lists.newLinkedList(); + expectedHits.add(new SearchHit("market_float", "222.222", 186)); + expectedHits.add(new SearchHit("quality_float", "222.222", 93)); + expectedHits.add(new SearchHit("market_long", "2222", 186)); + expectedHits.add(new SearchHit("quality_long", "2222", 93)); + + checkSearchQuery(searchQuery, expectedHits); + } + + @Test + public void testSearchLongFloatWithExtractionFilter() + { + final String automotiveSnowman = "automotive☃"; + Map> expectedResults = Maps.newTreeMap(String.CASE_INSENSITIVE_ORDER); + expectedResults.put( + "quality_long", new HashSet(Arrays.asList(automotiveSnowman)) + ); + expectedResults.put( + "quality_float", new HashSet(Arrays.asList(automotiveSnowman)) + ); + + List expectedHits = Lists.newLinkedList(); + expectedHits.add(new SearchHit("quality_float", automotiveSnowman, 93)); + expectedHits.add(new SearchHit("quality_long", automotiveSnowman, 93)); + + final LookupExtractionFn lookupExtractionFn = new LookupExtractionFn( + new MapLookupExtractor(ImmutableMap.of("1111", automotiveSnowman, + "111.111", automotiveSnowman + ), false), + true, + null, + true, + false + ); + + DimFilter filter = Druids.newAndDimFilterBuilder() + .fields( + Arrays.asList( + new ExtractionDimFilter( + "quality_long", + automotiveSnowman, + lookupExtractionFn, + null + ), + new ExtractionDimFilter( + "quality_float", + automotiveSnowman, + lookupExtractionFn, + null + ) + ) + ) + .build(); + + List dimensionSpecs = new ArrayList<>(); + dimensionSpecs.add(new ExtractionDimensionSpec("quality_long", null, lookupExtractionFn, null)); + dimensionSpecs.add(new ExtractionDimensionSpec("quality_float", null, lookupExtractionFn, null)); + + checkSearchQuery( + Druids.newSearchQueryBuilder() + .dataSource(QueryRunnerTestHelper.dataSource) + .granularity(QueryRunnerTestHelper.allGran) + .filters(filter) + .intervals(QueryRunnerTestHelper.fullOnInterval) + .dimensions(dimensionSpecs) + .query("☃") + .build(), + expectedHits + ); + } + + private void checkSearchQuery(Query searchQuery, List expectedResults) { checkSearchQuery(searchQuery, runner, expectedResults); diff --git a/processing/src/test/java/io/druid/query/search/SearchQueryRunnerWithCaseTest.java b/processing/src/test/java/io/druid/query/search/SearchQueryRunnerWithCaseTest.java index c9332adf536f..201b5efa4376 100644 --- a/processing/src/test/java/io/druid/query/search/SearchQueryRunnerWithCaseTest.java +++ b/processing/src/test/java/io/druid/query/search/SearchQueryRunnerWithCaseTest.java @@ -77,10 +77,10 @@ public static Iterable constructorFeeder() throws IOException ); CharSource input = CharSource.wrap( - "2011-01-12T00:00:00.000Z\tspot\tAutoMotive\tPREFERRED\ta\u0001preferred\t100.000000\n" + - "2011-01-12T00:00:00.000Z\tSPot\tbusiness\tpreferred\tb\u0001Preferred\t100.000000\n" + - "2011-01-12T00:00:00.000Z\tspot\tentertainment\tPREFERRed\te\u0001preferred\t100.000000\n" + - "2011-01-13T00:00:00.000Z\tspot\tautomotive\tpreferred\ta\u0001preferred\t94.874713" + "2011-01-12T00:00:00.000Z\tspot\t11\t11.0\tAutoMotive\t111\t111.0\tPREFERRED\ta\u0001preferred\t100.000000\n" + + "2011-01-12T00:00:00.000Z\tSPot\t11\t11.0\tbusiness\t222\t222.0\tpreferred\tb\u0001Preferred\t100.000000\n" + + "2011-01-12T00:00:00.000Z\tspot\t11\t11.0\tentertainment\t333\t333.0\tPREFERRed\te\u0001preferred\t100.000000\n" + + "2011-01-13T00:00:00.000Z\tspot\t11\t11.0\tautomotive\t111\t111.0\tpreferred\ta\u0001preferred\t94.874713" ); IncrementalIndex index1 = TestIndex.makeRealtimeIndex(input); diff --git a/processing/src/test/java/io/druid/query/select/MultiSegmentSelectQueryTest.java b/processing/src/test/java/io/druid/query/select/MultiSegmentSelectQueryTest.java index ee7193f5b232..abf0bd462aec 100644 --- a/processing/src/test/java/io/druid/query/select/MultiSegmentSelectQueryTest.java +++ b/processing/src/test/java/io/druid/query/select/MultiSegmentSelectQueryTest.java @@ -191,7 +191,7 @@ public static void clear() private final Druids.SelectQueryBuilder builder = Druids.newSelectQueryBuilder() .dataSource(new TableDataSource(QueryRunnerTestHelper.dataSource)) - .intervals(SelectQueryRunnerTest.I_0112_0114) + .intervals(SelectQueryRunnerTest.INTERVAL_0112_0114) .granularity(QueryRunnerTestHelper.allGran) .dimensionSpecs(DefaultDimensionSpec.toSpec(QueryRunnerTestHelper.dimensions)) .pagingSpec(new PagingSpec(null, 3)); diff --git a/processing/src/test/java/io/druid/query/select/SelectQueryRunnerTest.java b/processing/src/test/java/io/druid/query/select/SelectQueryRunnerTest.java index 4831d68fed5d..73d9bffdf2c7 100644 --- a/processing/src/test/java/io/druid/query/select/SelectQueryRunnerTest.java +++ b/processing/src/test/java/io/druid/query/select/SelectQueryRunnerTest.java @@ -63,41 +63,41 @@ public class SelectQueryRunnerTest { // copied from druid.sample.tsv - public static final String[] V_0112 = { - "2011-01-12T00:00:00.000Z spot automotive preferred apreferred 100.000000", - "2011-01-12T00:00:00.000Z spot business preferred bpreferred 100.000000", - "2011-01-12T00:00:00.000Z spot entertainment preferred epreferred 100.000000", - "2011-01-12T00:00:00.000Z spot health preferred hpreferred 100.000000", - "2011-01-12T00:00:00.000Z spot mezzanine preferred mpreferred 100.000000", - "2011-01-12T00:00:00.000Z spot news preferred npreferred 100.000000", - "2011-01-12T00:00:00.000Z spot premium preferred ppreferred 100.000000", - "2011-01-12T00:00:00.000Z spot technology preferred tpreferred 100.000000", - "2011-01-12T00:00:00.000Z spot travel preferred tpreferred 100.000000", - "2011-01-12T00:00:00.000Z total_market mezzanine preferred mpreferred 1000.000000", - "2011-01-12T00:00:00.000Z total_market premium preferred ppreferred 1000.000000", - "2011-01-12T00:00:00.000Z upfront mezzanine preferred mpreferred 800.000000 value", - "2011-01-12T00:00:00.000Z upfront premium preferred ppreferred 800.000000 value" + public static final String[] EVENTS_0112 = { + "2011-01-12T00:00:00.000Z\tspot\t1111\t111.111\tautomotive\t1111\t111.111\tpreferred\tapreferred\t100.000000", + "2011-01-12T00:00:00.000Z\tspot\t1111\t111.111\tbusiness\t2222\t222.222\tpreferred\tbpreferred\t100.000000", + "2011-01-12T00:00:00.000Z\tspot\t1111\t111.111\tentertainment\t3333\t333.333\tpreferred\tepreferred\t100.000000", + "2011-01-12T00:00:00.000Z\tspot\t1111\t111.111\thealth\t4444\t444.444\tpreferred\thpreferred\t100.000000", + "2011-01-12T00:00:00.000Z\tspot\t1111\t111.111\tmezzanine\t5555\t555.555\tpreferred\tmpreferred\t100.000000", + "2011-01-12T00:00:00.000Z\tspot\t1111\t111.111\tnews\t6666\t666.666\tpreferred\tnpreferred\t100.000000", + "2011-01-12T00:00:00.000Z\tspot\t1111\t111.111\tpremium\t7777\t777.777\tpreferred\tppreferred\t100.000000", + "2011-01-12T00:00:00.000Z\tspot\t1111\t111.111\ttechnology\t8888\t888.888\tpreferred\ttpreferred\t100.000000", + "2011-01-12T00:00:00.000Z\tspot\t1111\t111.111\ttravel\t8899\t889.999\tpreferred\ttpreferred\t100.000000", + "2011-01-12T00:00:00.000Z\ttotal_market\t2222\t222.222\tmezzanine\t5555\t555.555\tpreferred\tmpreferred\t1000.000000", + "2011-01-12T00:00:00.000Z\ttotal_market\t2222\t222.222\tpremium\t7777\t777.777\tpreferred\tppreferred\t1000.000000", + "2011-01-12T00:00:00.000Z\tupfront\t3333\t333.333\tmezzanine\t5555\t555.555\tpreferred\tmpreferred\t800.000000\tvalue", + "2011-01-12T00:00:00.000Z\tupfront\t3333\t333.333\tpremium\t7777\t777.777\tpreferred\tppreferred\t800.000000\tvalue" }; - public static final String[] V_0113 = { - "2011-01-13T00:00:00.000Z spot automotive preferred apreferred 94.874713", - "2011-01-13T00:00:00.000Z spot business preferred bpreferred 103.629399", - "2011-01-13T00:00:00.000Z spot entertainment preferred epreferred 110.087299", - "2011-01-13T00:00:00.000Z spot health preferred hpreferred 114.947403", - "2011-01-13T00:00:00.000Z spot mezzanine preferred mpreferred 104.465767", - "2011-01-13T00:00:00.000Z spot news preferred npreferred 102.851683", - "2011-01-13T00:00:00.000Z spot premium preferred ppreferred 108.863011", - "2011-01-13T00:00:00.000Z spot technology preferred tpreferred 111.356672", - "2011-01-13T00:00:00.000Z spot travel preferred tpreferred 106.236928", - "2011-01-13T00:00:00.000Z total_market mezzanine preferred mpreferred 1040.945505", - "2011-01-13T00:00:00.000Z total_market premium preferred ppreferred 1689.012875", - "2011-01-13T00:00:00.000Z upfront mezzanine preferred mpreferred 826.060182 value", - "2011-01-13T00:00:00.000Z upfront premium preferred ppreferred 1564.617729 value" + public static final String[] EVENTS_0113 = { + "2011-01-13T00:00:00.000Z\tspot\t1111\t111.111\tautomotive\t1111\t111.111\tpreferred\tapreferred\t94.874713", + "2011-01-13T00:00:00.000Z\tspot\t1111\t111.111\tbusiness\t2222\t222.222\tpreferred\tbpreferred\t103.629399", + "2011-01-13T00:00:00.000Z\tspot\t1111\t111.111\tentertainment\t3333\t333.333\tpreferred\tepreferred\t110.087299", + "2011-01-13T00:00:00.000Z\tspot\t1111\t111.111\thealth\t4444\t444.444\tpreferred\thpreferred\t114.947403", + "2011-01-13T00:00:00.000Z\tspot\t1111\t111.111\tmezzanine\t5555\t555.555\tpreferred\tmpreferred\t104.465767", + "2011-01-13T00:00:00.000Z\tspot\t1111\t111.111\tnews\t6666\t666.666\tpreferred\tnpreferred\t102.851683", + "2011-01-13T00:00:00.000Z\tspot\t1111\t111.111\tpremium\t7777\t777.777\tpreferred\tppreferred\t108.863011", + "2011-01-13T00:00:00.000Z\tspot\t1111\t111.111\ttechnology\t8888\t888.888\tpreferred\ttpreferred\t111.356672", + "2011-01-13T00:00:00.000Z\tspot\t1111\t111.111\ttravel\t8899\t889.999\tpreferred\ttpreferred\t106.236928", + "2011-01-13T00:00:00.000Z\ttotal_market\t2222\t222.222\tmezzanine\t5555\t555.555\tpreferred\tmpreferred\t1040.945505", + "2011-01-13T00:00:00.000Z\ttotal_market\t2222\t222.222\tpremium\t7777\t777.777\tpreferred\tppreferred\t1689.012875", + "2011-01-13T00:00:00.000Z\tupfront\t3333\t333.333\tmezzanine\t5555\t555.555\tpreferred\tmpreferred\t826.060182\tvalue", + "2011-01-13T00:00:00.000Z\tupfront\t3333\t333.333\tpremium\t7777\t777.777\tpreferred\tppreferred\t1564.617729\tvalue" }; - public static final QuerySegmentSpec I_0112_0114 = new LegacySegmentSpec( + public static final QuerySegmentSpec INTERVAL_0112_0114 = new LegacySegmentSpec( new Interval("2011-01-12/2011-01-14") ); - public static final String[] V_0112_0114 = ObjectArrays.concat(V_0112, V_0113, String.class); + public static final String[] EVENTS_0112_0114 = ObjectArrays.concat(EVENTS_0112, EVENTS_0113, String.class); @Parameterized.Parameters(name = "{0}:descending={1}") public static Iterable constructorFeeder() throws IOException @@ -131,7 +131,7 @@ public void testFullOnSelect() { SelectQuery query = new SelectQuery( new TableDataSource(QueryRunnerTestHelper.dataSource), - I_0112_0114, + INTERVAL_0112_0114, descending, null, QueryRunnerTestHelper.allGran, @@ -148,7 +148,7 @@ public void testFullOnSelect() PagingOffset offset = query.getPagingOffset(QueryRunnerTestHelper.segmentId); List> expectedResults = toExpected( - toEvents(new String[]{EventHolder.timestampKey + ":TIME"}, V_0112_0114), + toEvents(new String[]{EventHolder.timestampKey + ":TIME"}, EVENTS_0112_0114), offset.startOffset(), offset.threshold() ); @@ -177,11 +177,15 @@ public void testFullOnSelectWithDimensionSpec() QueryRunnerTestHelper.allGran, Arrays.asList( new DefaultDimensionSpec(QueryRunnerTestHelper.marketDimension, "mar"), + new DefaultDimensionSpec("market_long", null), + new DefaultDimensionSpec("market_float", null), new ExtractionDimensionSpec( QueryRunnerTestHelper.qualityDimension, "qual", new LookupExtractionFn(new MapLookupExtractor(map, true), false, null, true, false) ), + new DefaultDimensionSpec("quality_long", null), + new DefaultDimensionSpec("quality_float", null), new DefaultDimensionSpec(QueryRunnerTestHelper.placementDimension, "place") ), Lists.newArrayList(), new PagingSpec(null, 3), null @@ -288,11 +292,12 @@ public void testSelectWithDimsAndMets() { SelectQuery query = new SelectQuery( new TableDataSource(QueryRunnerTestHelper.dataSource), - I_0112_0114, + INTERVAL_0112_0114, descending, null, QueryRunnerTestHelper.allGran, - DefaultDimensionSpec.toSpec(Arrays.asList(QueryRunnerTestHelper.marketDimension)), + DefaultDimensionSpec.toSpec(Arrays.asList(QueryRunnerTestHelper.marketDimension, "market_long", "market_float", + QueryRunnerTestHelper.qualityDimension, "quality_long", "quality_float")), Arrays.asList(QueryRunnerTestHelper.indexMetric), new PagingSpec(null, 3), null @@ -309,12 +314,16 @@ public void testSelectWithDimsAndMets() new String[]{ EventHolder.timestampKey + ":TIME", QueryRunnerTestHelper.marketDimension + ":STRING", - null, + "market_long" + ":LONG", + "market_float" + ":FLOAT", + QueryRunnerTestHelper.qualityDimension + ":STRING", + "quality_long" + ":LONG", + "quality_float" + ":FLOAT", null, null, QueryRunnerTestHelper.indexMetric + ":FLOAT" }, - V_0112_0114 + EVENTS_0112_0114 ), offset.startOffset(), offset.threshold() @@ -327,7 +336,7 @@ public void testSelectPagination() { SelectQuery query = new SelectQuery( new TableDataSource(QueryRunnerTestHelper.dataSource), - I_0112_0114, + INTERVAL_0112_0114, descending, null, QueryRunnerTestHelper.allGran, @@ -350,7 +359,7 @@ public void testSelectPagination() "foo:NULL", "foo2:NULL" }, - V_0112_0114 + EVENTS_0112_0114 ), offset.startOffset(), offset.threshold() @@ -365,11 +374,18 @@ public void testFullOnSelectWithFilter() for (int[] param : new int[][]{{3, 3}, {0, 1}, {5, 5}, {2, 7}, {3, 0}}) { SelectQuery query = new SelectQuery( new TableDataSource(QueryRunnerTestHelper.dataSource), - I_0112_0114, + INTERVAL_0112_0114, descending, new SelectorDimFilter(QueryRunnerTestHelper.marketDimension, "spot"), QueryRunnerTestHelper.dayGran, - DefaultDimensionSpec.toSpec(Lists.newArrayList(QueryRunnerTestHelper.qualityDimension)), + DefaultDimensionSpec.toSpec(Lists.newArrayList( + QueryRunnerTestHelper.marketDimension, + "market_long", + "market_float", + QueryRunnerTestHelper.qualityDimension, + "quality_long", + "quality_float") + ), Lists.newArrayList(QueryRunnerTestHelper.indexMetric), new PagingSpec(toPagingIdentifier(param[0], descending), param[1]), null @@ -383,34 +399,38 @@ public void testFullOnSelectWithFilter() final List>> events = toEvents( new String[]{ EventHolder.timestampKey + ":TIME", - null, + QueryRunnerTestHelper.marketDimension + ":STRING", + "market_long" + ":LONG", + "market_float" + ":FLOAT", QueryRunnerTestHelper.qualityDimension + ":STRING", + "quality_long" + ":LONG", + "quality_float" + ":FLOAT", null, null, QueryRunnerTestHelper.indexMetric + ":FLOAT" }, // filtered values with day granularity new String[]{ - "2011-01-12T00:00:00.000Z spot automotive preferred apreferred 100.000000", - "2011-01-12T00:00:00.000Z spot business preferred bpreferred 100.000000", - "2011-01-12T00:00:00.000Z spot entertainment preferred epreferred 100.000000", - "2011-01-12T00:00:00.000Z spot health preferred hpreferred 100.000000", - "2011-01-12T00:00:00.000Z spot mezzanine preferred mpreferred 100.000000", - "2011-01-12T00:00:00.000Z spot news preferred npreferred 100.000000", - "2011-01-12T00:00:00.000Z spot premium preferred ppreferred 100.000000", - "2011-01-12T00:00:00.000Z spot technology preferred tpreferred 100.000000", - "2011-01-12T00:00:00.000Z spot travel preferred tpreferred 100.000000" + "2011-01-12T00:00:00.000Z\tspot\t1111\t111.111\tautomotive\t1111\t111.111\tpreferred\tapreferred\t100.000000", + "2011-01-12T00:00:00.000Z\tspot\t1111\t111.111\tbusiness\t2222\t222.222\tpreferred\tbpreferred\t100.000000", + "2011-01-12T00:00:00.000Z\tspot\t1111\t111.111\tentertainment\t3333\t333.333\tpreferred\tepreferred\t100.000000", + "2011-01-12T00:00:00.000Z\tspot\t1111\t111.111\thealth\t4444\t444.444\tpreferred\thpreferred\t100.000000", + "2011-01-12T00:00:00.000Z\tspot\t1111\t111.111\tmezzanine\t5555\t555.555\tpreferred\tmpreferred\t100.000000", + "2011-01-12T00:00:00.000Z\tspot\t1111\t111.111\tnews\t6666\t666.666\tpreferred\tnpreferred\t100.000000", + "2011-01-12T00:00:00.000Z\tspot\t1111\t111.111\tpremium\t7777\t777.777\tpreferred\tppreferred\t100.000000", + "2011-01-12T00:00:00.000Z\tspot\t1111\t111.111\ttechnology\t8888\t888.888\tpreferred\ttpreferred\t100.000000", + "2011-01-12T00:00:00.000Z\tspot\t1111\t111.111\ttravel\t8899\t889.999\tpreferred\ttpreferred\t100.000000" }, new String[]{ - "2011-01-13T00:00:00.000Z spot automotive preferred apreferred 94.874713", - "2011-01-13T00:00:00.000Z spot business preferred bpreferred 103.629399", - "2011-01-13T00:00:00.000Z spot entertainment preferred epreferred 110.087299", - "2011-01-13T00:00:00.000Z spot health preferred hpreferred 114.947403", - "2011-01-13T00:00:00.000Z spot mezzanine preferred mpreferred 104.465767", - "2011-01-13T00:00:00.000Z spot news preferred npreferred 102.851683", - "2011-01-13T00:00:00.000Z spot premium preferred ppreferred 108.863011", - "2011-01-13T00:00:00.000Z spot technology preferred tpreferred 111.356672", - "2011-01-13T00:00:00.000Z spot travel preferred tpreferred 106.236928" + "2011-01-13T00:00:00.000Z\tspot\t1111\t111.111\tautomotive\t1111\t111.111\tpreferred\tapreferred\t94.874713", + "2011-01-13T00:00:00.000Z\tspot\t1111\t111.111\tbusiness\t2222\t222.222\tpreferred\tbpreferred\t103.629399", + "2011-01-13T00:00:00.000Z\tspot\t1111\t111.111\tentertainment\t3333\t333.333\tpreferred\tepreferred\t110.087299", + "2011-01-13T00:00:00.000Z\tspot\t1111\t111.111\thealth\t4444\t444.444\tpreferred\thpreferred\t114.947403", + "2011-01-13T00:00:00.000Z\tspot\t1111\t111.111\tmezzanine\t5555\t555.555\tpreferred\tmpreferred\t104.465767", + "2011-01-13T00:00:00.000Z\tspot\t1111\t111.111\tnews\t6666\t666.666\tpreferred\tnpreferred\t102.851683", + "2011-01-13T00:00:00.000Z\tspot\t1111\t111.111\tpremium\t7777\t777.777\tpreferred\tppreferred\t108.863011", + "2011-01-13T00:00:00.000Z\tspot\t1111\t111.111\ttechnology\t8888\t888.888\tpreferred\ttpreferred\t111.356672", + "2011-01-13T00:00:00.000Z\tspot\t1111\t111.111\ttravel\t8899\t889.999\tpreferred\ttpreferred\t106.236928" } ); @@ -429,12 +449,14 @@ public void testFullSelectNoResults() { SelectQuery query = new SelectQuery( new TableDataSource(QueryRunnerTestHelper.dataSource), - I_0112_0114, + INTERVAL_0112_0114, descending, new AndDimFilter( Arrays.asList( new SelectorDimFilter(QueryRunnerTestHelper.marketDimension, "spot"), - new SelectorDimFilter(QueryRunnerTestHelper.marketDimension, "foo") + new SelectorDimFilter(QueryRunnerTestHelper.marketDimension, "foo"), + new SelectorDimFilter("market_long", "1111"), + new SelectorDimFilter("market_float", "111.111") ) ), QueryRunnerTestHelper.allGran, @@ -465,7 +487,7 @@ public void testFullSelectNoDimensionAndMetric() { SelectQuery query = new SelectQuery( new TableDataSource(QueryRunnerTestHelper.dataSource), - I_0112_0114, + INTERVAL_0112_0114, descending, null, QueryRunnerTestHelper.allGran, @@ -486,7 +508,7 @@ public void testFullSelectNoDimensionAndMetric() "foo:NULL", "foo2:NULL" }, - V_0112_0114 + EVENTS_0112_0114 ); PagingOffset offset = query.getPagingOffset(QueryRunnerTestHelper.segmentId); @@ -595,6 +617,7 @@ private static void verify( Result expected = expectedIter.next(); Result actual = actualIter.next(); + String resultsStr = "\nExpected: " + expected + "\nActual: " + actual + "\nField: "; Assert.assertEquals(expected.getTimestamp(), actual.getTimestamp()); for (Map.Entry entry : expected.getValue().getPagingIdentifiers().entrySet()) { @@ -618,7 +641,7 @@ private static void verify( if (acHolder.getEvent().get(ex.getKey()) instanceof Double) { actVal = ((Double) actVal).floatValue(); } - Assert.assertEquals(ex.getValue(), actVal); + Assert.assertEquals(resultsStr + ex.getKey(), ex.getValue(), actVal); } } diff --git a/processing/src/test/java/io/druid/query/timeseries/TimeseriesQueryRunnerTest.java b/processing/src/test/java/io/druid/query/timeseries/TimeseriesQueryRunnerTest.java index c3821630647b..e58f4ad0de93 100644 --- a/processing/src/test/java/io/druid/query/timeseries/TimeseriesQueryRunnerTest.java +++ b/processing/src/test/java/io/druid/query/timeseries/TimeseriesQueryRunnerTest.java @@ -40,6 +40,7 @@ import io.druid.query.filter.AndDimFilter; import io.druid.query.filter.BoundDimFilter; import io.druid.query.filter.DimFilter; +import io.druid.query.filter.ExtractionDimFilter; import io.druid.query.filter.InDimFilter; import io.druid.query.filter.NotDimFilter; import io.druid.query.filter.RegexDimFilter; @@ -2261,4 +2262,135 @@ public void testTimeseriesWithBetweenFilter1() ); TestHelper.assertExpectedResults(expectedResults, results); } + + @Test + public void testFullOnTimeseriesWithLongFloatFilters() + { + + DimFilter filter = Druids.newAndDimFilterBuilder() + .fields( + Arrays.asList( + new SelectorDimFilter("market_long", "3333"), + new SelectorDimFilter("market_float", "333.333") + ) + ) + .build(); + TimeseriesQuery query = Druids.newTimeseriesQueryBuilder() + .dataSource(QueryRunnerTestHelper.dataSource) + .granularity(QueryRunnerTestHelper.dayGran) + .filters(filter) + .intervals(QueryRunnerTestHelper.fullOnInterval) + .aggregators( + Arrays.asList( + QueryRunnerTestHelper.rowsCount, + QueryRunnerTestHelper.qualityUniques + ) + ) + .descending(descending) + .build(); + + final DateTime expectedLast = descending ? + QueryRunnerTestHelper.earliest : + QueryRunnerTestHelper.last; + + Iterable> results = Sequences.toList( + runner.run(query, CONTEXT), + Lists.>newArrayList() + ); + + for (Result result : results) { + DateTime current = result.getTimestamp(); + Assert.assertFalse( + String.format("Timestamp[%s] > expectedLast[%s]", current, expectedLast), + descending ? current.isBefore(expectedLast) : current.isAfter(expectedLast) + ); + + final TimeseriesResultValue value = result.getValue(); + + Assert.assertEquals( + result.toString(), + QueryRunnerTestHelper.skippedDay.equals(result.getTimestamp()) ? 0L : 2L, + value.getLongMetric("rows").longValue() + ); + Assert.assertEquals( + result.toString(), + QueryRunnerTestHelper.skippedDay.equals(result.getTimestamp()) ? 0.0d : 2.0d, + value.getDoubleMetric( + "uniques" + ), + 0.01 + ); + } + } + + @Test + public void testTimeseriesWithLongFloatBoundFilters() + { + TimeseriesQuery query = Druids.newTimeseriesQueryBuilder() + .dataSource(QueryRunnerTestHelper.dataSource) + .granularity(QueryRunnerTestHelper.dayGran) + .filters( + new AndDimFilter( + Arrays.asList( + new BoundDimFilter( + "market_long", + "1000", + "2000", + true, + null, + null + ), + (DimFilter) new BoundDimFilter( + "market_float", + "100.567", + "200.789", + null, + null, + null + ) + ) + ) + ) + .intervals(QueryRunnerTestHelper.firstToThird) + .aggregators( + Arrays.asList( + QueryRunnerTestHelper.rowsCount, + QueryRunnerTestHelper.indexLongSum, + QueryRunnerTestHelper.qualityUniques + ) + ) + .postAggregators(Arrays.asList(QueryRunnerTestHelper.addRowsIndexConstant)) + .build(); + + List> expectedResults = Arrays.asList( + new Result<>( + new DateTime("2011-04-01"), + new TimeseriesResultValue( + ImmutableMap.of( + "rows", 9L, + "index", 1102L, + "addRowsIndexConstant", 1112.0, + "uniques", QueryRunnerTestHelper.UNIQUES_9 + ) + ) + ), + new Result<>( + new DateTime("2011-04-02"), + new TimeseriesResultValue( + ImmutableMap.of( + "rows", 9L, + "index", 1120L, + "addRowsIndexConstant", 1130.0, + "uniques", QueryRunnerTestHelper.UNIQUES_9 + ) + ) + ) + ); + + Iterable> results = Sequences.toList( + runner.run(query, CONTEXT), + Lists.>newArrayList() + ); + TestHelper.assertExpectedResults(expectedResults, results); + } } diff --git a/processing/src/test/java/io/druid/query/topn/AlphaNumericTopNMetricSpecTest.java b/processing/src/test/java/io/druid/query/topn/AlphaNumericTopNMetricSpecTest.java index 85fb0b577f1a..54f7e9038823 100644 --- a/processing/src/test/java/io/druid/query/topn/AlphaNumericTopNMetricSpecTest.java +++ b/processing/src/test/java/io/druid/query/topn/AlphaNumericTopNMetricSpecTest.java @@ -37,7 +37,7 @@ public class AlphaNumericTopNMetricSpecTest @Test public void testComparator() throws Exception { - final Comparator comparator = AlphaNumericTopNMetricSpec.comparator; + final Comparator comparator = AlphaNumericTopNMetricSpec.comparator; // equality Assert.assertEquals(0, comparator.compare("", "")); diff --git a/processing/src/test/java/io/druid/query/topn/ColumnTypeMergeTest.java b/processing/src/test/java/io/druid/query/topn/ColumnTypeMergeTest.java new file mode 100644 index 000000000000..d0aef95e1b0b --- /dev/null +++ b/processing/src/test/java/io/druid/query/topn/ColumnTypeMergeTest.java @@ -0,0 +1,343 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.query.topn; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.base.Supplier; +import com.google.common.base.Suppliers; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import com.metamx.common.guava.Sequences; +import io.druid.collections.StupidPool; +import io.druid.common.utils.JodaUtils; +import io.druid.data.input.MapBasedInputRow; +import io.druid.data.input.Row; +import io.druid.data.input.impl.DimensionSchema; +import io.druid.data.input.impl.DimensionsSpec; +import io.druid.data.input.impl.FloatDimensionSchema; +import io.druid.data.input.impl.LongDimensionSchema; +import io.druid.data.input.impl.StringDimensionSchema; +import io.druid.granularity.PeriodGranularity; +import io.druid.granularity.QueryGranularity; +import io.druid.jackson.DefaultObjectMapper; +import io.druid.query.FinalizeResultsQueryRunner; +import io.druid.query.QueryRunner; +import io.druid.query.QueryRunnerFactory; +import io.druid.query.QueryRunnerTestHelper; +import io.druid.query.QueryToolChest; +import io.druid.query.Result; +import io.druid.query.TestQueryRunners; +import io.druid.query.aggregation.AggregatorFactory; +import io.druid.query.aggregation.CountAggregatorFactory; +import io.druid.query.dimension.DefaultDimensionSpec; +import io.druid.query.dimension.DimensionSpec; +import io.druid.query.groupby.GroupByQuery; +import io.druid.query.groupby.GroupByQueryConfig; +import io.druid.query.groupby.GroupByQueryEngine; +import io.druid.query.groupby.GroupByQueryQueryToolChest; +import io.druid.query.groupby.GroupByQueryRunnerFactory; +import io.druid.query.groupby.GroupByQueryRunnerTestHelper; +import io.druid.segment.IncrementalIndexSegment; +import io.druid.segment.TestHelper; +import io.druid.segment.column.ValueType; +import io.druid.segment.incremental.IncrementalIndex; +import io.druid.segment.incremental.IncrementalIndexSchema; +import io.druid.segment.incremental.OnheapIncrementalIndex; +import org.joda.time.DateTime; +import org.joda.time.Interval; +import org.joda.time.Period; +import org.junit.Test; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.Executors; + +public class ColumnTypeMergeTest +{ + private String dimName; + private String dimName2; + private IncrementalIndex stringIndex; + private IncrementalIndex numericIndex; + + public ColumnTypeMergeTest() throws Exception + { + init(); + } + + private void init() throws Exception + { + dimName = "foo"; + dimName2 = "bar"; + List stringDimSchemas = Arrays.asList((DimensionSchema) new StringDimensionSchema(dimName), new StringDimensionSchema(dimName2)); + List numericDimSchemas = Arrays.asList((DimensionSchema) new FloatDimensionSchema(dimName), new LongDimensionSchema(dimName2)); + + List floatStrValues = Arrays.asList("1.0", "2.0", "3.0", "4.0"); + List floatValues = Arrays.asList(1.0f, 2.0f, 3.0f, 4.0f); + List longStrValues = Arrays.asList("1", "2", "3", "4-abc"); + List longValues = Arrays.asList(1L, 2L, 3L, 4L); + IncrementalIndexSchema stringIndexSchema = new IncrementalIndexSchema( + 0L, + QueryGranularity.NONE, + new DimensionsSpec(stringDimSchemas, null, null), + new AggregatorFactory[]{new CountAggregatorFactory("count")} + ); + + IncrementalIndexSchema numericIndexSchema = new IncrementalIndexSchema( + 0L, + QueryGranularity.NONE, + new DimensionsSpec(numericDimSchemas, null, null), + new AggregatorFactory[]{new CountAggregatorFactory("count")} + ); + + stringIndex = new OnheapIncrementalIndex(stringIndexSchema, true, 1000); + numericIndex = new OnheapIncrementalIndex(numericIndexSchema, true, 1000); + + for (int i = 0; i < floatStrValues.size(); i++) { + MapBasedInputRow row = new MapBasedInputRow( + 1, + Arrays.asList(dimName, dimName2), + ImmutableMap.of(dimName, floatStrValues.get(i), dimName2, longStrValues.get(i)) + ); + stringIndex.add(row); + } + + for (int i = 0; i < floatValues.size(); i++) { + MapBasedInputRow row = new MapBasedInputRow( + 1, + Arrays.asList(dimName, dimName2), + ImmutableMap.of(dimName, floatValues.get(i), dimName2, longValues.get(i)) + ); + numericIndex.add(row); + } + } + + @Test + public void testGroupByMergeWithColumnTypeMismatch() throws Exception + { + + boolean bySegment = false; + + final ObjectMapper mapper = new DefaultObjectMapper(); + final StupidPool pool = new StupidPool<>( + new Supplier() + { + @Override + public ByteBuffer get() + { + return ByteBuffer.allocate(1024 * 1024); + } + } + ); + + final GroupByQueryConfig config = new GroupByQueryConfig(); + config.setMaxIntermediateRows(10000); + + final Supplier configSupplier = Suppliers.ofInstance(config); + final GroupByQueryEngine engine = new GroupByQueryEngine(configSupplier, pool); + final GroupByQueryRunnerFactory factory = new GroupByQueryRunnerFactory( + engine, + QueryRunnerTestHelper.NOOP_QUERYWATCHER, + configSupplier, + new GroupByQueryQueryToolChest( + configSupplier, mapper, engine, TestQueryRunners.pool, + QueryRunnerTestHelper.NoopIntervalChunkingQueryRunnerDecorator() + ), + TestQueryRunners.pool + ); + + QueryRunner stringRunner = QueryRunnerTestHelper.makeQueryRunner(factory, "string-segment", new IncrementalIndexSegment(stringIndex, "string-segment")); + QueryRunner numericRunner = QueryRunnerTestHelper.makeQueryRunner(factory, "float-segment", new IncrementalIndexSegment(numericIndex, "float-segment")); + + List> runners = Arrays.asList(stringRunner, numericRunner); + + QueryToolChest toolChest = factory.getToolchest(); + QueryRunner theRunner = toolChest.postMergeQueryDecoration( + new FinalizeResultsQueryRunner<>( + toolChest.mergeResults(factory.mergeRunners(Executors.newCachedThreadPool(), runners)), + toolChest + ) + ); + + List expectedResults = Arrays + .asList( + GroupByQueryRunnerTestHelper.createExpectedRow("1970-01-01T00:00:00.000Z", "foo", 1.0f, "bar", "1", "rows", 2L), + GroupByQueryRunnerTestHelper.createExpectedRow("1970-01-01T00:00:00.000Z", "foo", 2.0f, "bar", "2", "rows", 2L), + GroupByQueryRunnerTestHelper.createExpectedRow("1970-01-01T00:00:00.000Z", "foo", 3.0f, "bar", "3", "rows", 2L), + GroupByQueryRunnerTestHelper.createExpectedRow("1970-01-01T00:00:00.000Z", "foo", 4.0f, "bar", "4", "rows", 1L), + GroupByQueryRunnerTestHelper.createExpectedRow("1970-01-01T00:00:00.000Z", "foo", 4.0f, "bar", "4-abc", "rows", 1L) + ); + + Map typeHints = ImmutableMap.of("foo", ValueType.FLOAT); + + Interval allTime = new Interval(0, JodaUtils.MAX_INSTANT); + GroupByQuery.Builder builder = GroupByQuery + .builder() + .setDataSource("type-test") + .setInterval(allTime) + .setDimensions( + Lists.newArrayList( + new DefaultDimensionSpec(dimName, dimName), + new DefaultDimensionSpec(dimName2, dimName2) + ) + ) + .setAggregatorSpecs( + Arrays.asList(new CountAggregatorFactory("rows"))) + .setGranularity(new PeriodGranularity(new Period("P1M"), null, null)) + .setContext(ImmutableMap.of("bySegment", bySegment, "typeHints", typeHints)); + + final GroupByQuery fullQuery = builder.build(); + + HashMap ctx = Maps.newHashMap(); + List results = Sequences.toList(theRunner.run(fullQuery, ctx), Lists.newArrayList()); + TestHelper.assertExpectedObjects(expectedResults, results, ""); + } + + + @Test + public void testTopNMergeWithColumnTypeMismatch() throws Exception + { + QueryRunnerFactory factory = new TopNQueryRunnerFactory( + TestQueryRunners.getPool(), + new TopNQueryQueryToolChest( + new TopNQueryConfig(), + QueryRunnerTestHelper.NoopIntervalChunkingQueryRunnerDecorator() + ), + QueryRunnerTestHelper.NOOP_QUERYWATCHER + ); + + QueryRunner stringRunner = QueryRunnerTestHelper.makeQueryRunner(factory, "string-segment", new IncrementalIndexSegment(stringIndex, "string-segment")); + QueryRunner numericRunner = QueryRunnerTestHelper.makeQueryRunner(factory, "float-segment", new IncrementalIndexSegment(numericIndex, "float-segment")); + + List> runners = Arrays.asList(stringRunner, numericRunner); + + final TopNQueryQueryToolChest chest = new TopNQueryQueryToolChest( + new TopNQueryConfig(), + QueryRunnerTestHelper.NoopIntervalChunkingQueryRunnerDecorator() + ); + + QueryRunner mergedRunner = factory.mergeRunners(Executors.newCachedThreadPool(), runners); + + final QueryRunner> theRunner = chest.mergeResults(chest.preMergeQueryDecoration(mergedRunner)); + + Map typeHints = ImmutableMap.of("foo", ValueType.FLOAT); + + Interval allTime = new Interval(0, JodaUtils.MAX_INSTANT); + TopNQuery query = new TopNQueryBuilder() + .dataSource("type-test") + .granularity(QueryRunnerTestHelper.allGran) + .dimension(dimName) + .metric("rows") + .threshold(10) + .intervals(Arrays.asList(allTime)) + .aggregators( + Lists.newArrayList( + new CountAggregatorFactory("rows") + ) + ) + .context(ImmutableMap.of("typeHints", typeHints)) + .build(); + + List> expectedResults = Arrays.asList( + new Result<>( + new DateTime("1970-01-01T00:00:00.001Z"), + new TopNResultValue( + Arrays.asList( + ImmutableMap.of( + "foo", 1.0f, + "rows", 2L + ), + ImmutableMap.of( + "foo", 2.0f, + "rows", 2L + ), + ImmutableMap.of( + "foo", 3.0f, + "rows", 2L + ), + ImmutableMap.of( + "foo", 4.0f, + "rows", 2L + ) + ) + ) + ) + ); + + HashMap ctx = Maps.newHashMap(); + ArrayList> results = Sequences., ArrayList>toList(theRunner.run(query, ctx), + Lists.>newArrayList()); + TestHelper.assertExpectedObjects(expectedResults, results, ""); + + TopNQuery query2 = new TopNQueryBuilder() + .dataSource("type-test") + .granularity(QueryRunnerTestHelper.allGran) + .dimension(dimName2) + .metric("rows") + .threshold(10) + .intervals(Arrays.asList(allTime)) + .aggregators( + Lists.newArrayList( + new CountAggregatorFactory("rows") + ) + ) + .context(ImmutableMap.of("typeHints", typeHints)) + .build(); + + List> expectedResults2 = Arrays.asList( + new Result<>( + new DateTime("1970-01-01T00:00:00.001Z"), + new TopNResultValue( + Arrays.asList( + ImmutableMap.of( + "bar", "1", + "rows", 2L + ), + ImmutableMap.of( + "bar", "2", + "rows", 2L + ), + ImmutableMap.of( + "bar", "3", + "rows", 2L + ), + ImmutableMap.of( + "bar", "4", + "rows", 1L + ), + ImmutableMap.of( + "bar", "4-abc", + "rows", 1L + ) + ) + ) + ) + ); + + ctx = Maps.newHashMap(); + ArrayList> results2 = Sequences., ArrayList>toList(theRunner.run(query2, ctx), + Lists.>newArrayList()); + TestHelper.assertExpectedObjects(expectedResults2, results2, ""); + } +} diff --git a/processing/src/test/java/io/druid/query/topn/TopNBinaryFnBenchmark.java b/processing/src/test/java/io/druid/query/topn/TopNBinaryFnBenchmark.java index fcb1ff484add..8c0f40f11606 100644 --- a/processing/src/test/java/io/druid/query/topn/TopNBinaryFnBenchmark.java +++ b/processing/src/test/java/io/druid/query/topn/TopNBinaryFnBenchmark.java @@ -121,7 +121,8 @@ protected void setUp() throws Exception new NumericTopNMetricSpec("index"), 100, aggregatorFactories, - postAggregators + postAggregators, + null ); } diff --git a/processing/src/test/java/io/druid/query/topn/TopNBinaryFnTest.java b/processing/src/test/java/io/druid/query/topn/TopNBinaryFnTest.java index 9500bdc511f1..7cae635e2682 100644 --- a/processing/src/test/java/io/druid/query/topn/TopNBinaryFnTest.java +++ b/processing/src/test/java/io/druid/query/topn/TopNBinaryFnTest.java @@ -148,7 +148,8 @@ public void testMerge() new NumericTopNMetricSpec("index"), 2, aggregatorFactories, - postAggregators + postAggregators, + null ).apply( result1, result2 @@ -230,7 +231,8 @@ public void testMergeDay() new NumericTopNMetricSpec("index"), 2, aggregatorFactories, - postAggregators + postAggregators, + null ).apply( result1, result2 @@ -275,7 +277,8 @@ public void testMergeOneResultNull() new NumericTopNMetricSpec("index"), 2, aggregatorFactories, - postAggregators + postAggregators, + null ).apply( result1, result2 @@ -371,7 +374,8 @@ public void testMergeByPostAgg() new NumericTopNMetricSpec("addrowsindexconstant"), 3, aggregatorFactories, - postAggregators + postAggregators, + null ).apply( result1, result2 @@ -453,7 +457,8 @@ public void testMergeShiftedTimestamp() new NumericTopNMetricSpec("index"), 2, aggregatorFactories, - postAggregators + postAggregators, + null ).apply( result1, result2 diff --git a/processing/src/test/java/io/druid/query/topn/TopNQueryRunnerTest.java b/processing/src/test/java/io/druid/query/topn/TopNQueryRunnerTest.java index f822c18ec218..554514fdd006 100644 --- a/processing/src/test/java/io/druid/query/topn/TopNQueryRunnerTest.java +++ b/processing/src/test/java/io/druid/query/topn/TopNQueryRunnerTest.java @@ -32,15 +32,25 @@ import com.metamx.common.guava.Sequence; import com.metamx.common.guava.Sequences; import io.druid.collections.StupidPool; +import io.druid.common.utils.JodaUtils; +import io.druid.data.input.MapBasedInputRow; +import io.druid.data.input.Row; +import io.druid.data.input.impl.DimensionSchema; +import io.druid.data.input.impl.DimensionsSpec; +import io.druid.data.input.impl.FloatDimensionSchema; +import io.druid.data.input.impl.LongDimensionSchema; +import io.druid.data.input.impl.StringDimensionSchema; import io.druid.granularity.QueryGranularity; import io.druid.query.BySegmentResultValue; import io.druid.query.BySegmentResultValueClass; import io.druid.query.Druids; import io.druid.query.QueryRunner; +import io.druid.query.QueryRunnerFactory; import io.druid.query.QueryRunnerTestHelper; import io.druid.query.Result; import io.druid.query.TestQueryRunners; import io.druid.query.aggregation.AggregatorFactory; +import io.druid.query.aggregation.CountAggregatorFactory; import io.druid.query.aggregation.DoubleMaxAggregatorFactory; import io.druid.query.aggregation.DoubleMinAggregatorFactory; import io.druid.query.aggregation.FilteredAggregatorFactory; @@ -62,8 +72,14 @@ import io.druid.query.filter.SelectorDimFilter; import io.druid.query.spec.MultipleIntervalSegmentSpec; import io.druid.query.timeseries.TimeseriesQuery; +import io.druid.segment.IncrementalIndexSegment; import io.druid.segment.TestHelper; +import io.druid.segment.TestIndex; import io.druid.segment.column.Column; +import io.druid.segment.column.ValueType; +import io.druid.segment.incremental.IncrementalIndex; +import io.druid.segment.incremental.IncrementalIndexSchema; +import io.druid.segment.incremental.OnheapIncrementalIndex; import org.joda.time.DateTime; import org.joda.time.Interval; import org.junit.Assert; @@ -81,12 +97,19 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.concurrent.Executors; /** */ @RunWith(Parameterized.class) public class TopNQueryRunnerTest { + private final ValueType keyType; + private final String marketStr; + private final Object spotVal; + private final Object totalMarketVal; + private final Object upfrontVal; + @Parameterized.Parameters public static Iterable constructorFeeder() throws IOException { @@ -125,16 +148,30 @@ public ByteBuffer get() ) ); - return QueryRunnerTestHelper.transformToConstructionFeeder(retVal); + ValueType[] types = new ValueType[]{ValueType.STRING, + ValueType.LONG, + ValueType.FLOAT}; + + return QueryRunnerTestHelper.cartesian(retVal, Lists.newArrayList(types)); + } + + private Object getDimMapping(String value) { + return TestIndex.getDimMappingForType(keyType, value); } private final QueryRunner> runner; public TopNQueryRunnerTest( - QueryRunner> runner + QueryRunner> runner, + ValueType keyType ) { this.runner = runner; + this.keyType = keyType; + this.marketStr = (String) getDimMapping(QueryRunnerTestHelper.marketDimension); + this.spotVal = getDimMapping("spot"); + this.totalMarketVal = getDimMapping("total_market"); + this.upfrontVal = getDimMapping("upfront"); } private Sequence> assertExpectedResults( @@ -172,7 +209,7 @@ public void testFullOnTopN() TopNQuery query = new TopNQueryBuilder() .dataSource(QueryRunnerTestHelper.dataSource) .granularity(QueryRunnerTestHelper.allGran) - .dimension(QueryRunnerTestHelper.marketDimension) + .dimension(marketStr) .metric(QueryRunnerTestHelper.indexMetric) .threshold(4) .intervals(QueryRunnerTestHelper.fullOnInterval) @@ -196,7 +233,7 @@ public void testFullOnTopN() new TopNResultValue( Arrays.>asList( ImmutableMap.builder() - .put(QueryRunnerTestHelper.marketDimension, "total_market") + .put(marketStr, totalMarketVal) .put("rows", 186L) .put("index", 215679.82879638672D) .put("addRowsIndexConstant", 215866.82879638672D) @@ -205,7 +242,7 @@ public void testFullOnTopN() .put("minIndex", 792.3260498046875D) .build(), ImmutableMap.builder() - .put(QueryRunnerTestHelper.marketDimension, "upfront") + .put(marketStr, upfrontVal) .put("rows", 186L) .put("index", 192046.1060180664D) .put("addRowsIndexConstant", 192233.1060180664D) @@ -214,7 +251,7 @@ public void testFullOnTopN() .put("minIndex", 545.9906005859375D) .build(), ImmutableMap.builder() - .put(QueryRunnerTestHelper.marketDimension, "spot") + .put(marketStr, spotVal) .put("rows", 837L) .put("index", 95606.57232284546D) .put("addRowsIndexConstant", 96444.57232284546D) @@ -235,7 +272,7 @@ public void testFullOnTopNOverPostAggs() TopNQuery query = new TopNQueryBuilder() .dataSource(QueryRunnerTestHelper.dataSource) .granularity(QueryRunnerTestHelper.allGran) - .dimension(QueryRunnerTestHelper.marketDimension) + .dimension(marketStr) .metric(QueryRunnerTestHelper.addRowsIndexConstantMetric) .threshold(4) .intervals(QueryRunnerTestHelper.fullOnInterval) @@ -259,7 +296,7 @@ public void testFullOnTopNOverPostAggs() new TopNResultValue( Arrays.>asList( ImmutableMap.builder() - .put(QueryRunnerTestHelper.marketDimension, "total_market") + .put(marketStr, totalMarketVal) .put("rows", 186L) .put("index", 215679.82879638672D) .put("addRowsIndexConstant", 215866.82879638672D) @@ -268,7 +305,7 @@ public void testFullOnTopNOverPostAggs() .put("minIndex", 792.3260498046875D) .build(), ImmutableMap.builder() - .put(QueryRunnerTestHelper.marketDimension, "upfront") + .put(marketStr, upfrontVal) .put("rows", 186L) .put("index", 192046.1060180664D) .put("addRowsIndexConstant", 192233.1060180664D) @@ -277,7 +314,7 @@ public void testFullOnTopNOverPostAggs() .put("minIndex", 545.9906005859375D) .build(), ImmutableMap.builder() - .put(QueryRunnerTestHelper.marketDimension, "spot") + .put(marketStr, spotVal) .put("rows", 837L) .put("index", 95606.57232284546D) .put("addRowsIndexConstant", 96444.57232284546D) @@ -299,7 +336,7 @@ public void testFullOnTopNOverUniques() TopNQuery query = new TopNQueryBuilder() .dataSource(QueryRunnerTestHelper.dataSource) .granularity(QueryRunnerTestHelper.allGran) - .dimension(QueryRunnerTestHelper.marketDimension) + .dimension(marketStr) .metric(QueryRunnerTestHelper.uniqueMetric) .threshold(3) .intervals(QueryRunnerTestHelper.fullOnInterval) @@ -323,7 +360,7 @@ public void testFullOnTopNOverUniques() new TopNResultValue( Arrays.>asList( ImmutableMap.builder() - .put("market", "spot") + .put(marketStr, spotVal) .put("rows", 837L) .put("index", 95606.57232284546D) .put("addRowsIndexConstant", 96444.57232284546D) @@ -332,7 +369,7 @@ public void testFullOnTopNOverUniques() .put("minIndex", 59.02102279663086D) .build(), ImmutableMap.builder() - .put("market", "total_market") + .put(marketStr, totalMarketVal) .put("rows", 186L) .put("index", 215679.82879638672D) .put("addRowsIndexConstant", 215866.82879638672D) @@ -341,7 +378,7 @@ public void testFullOnTopNOverUniques() .put("minIndex", 792.3260498046875D) .build(), ImmutableMap.builder() - .put("market", "upfront") + .put(marketStr, upfrontVal) .put("rows", 186L) .put("index", 192046.1060180664D) .put("addRowsIndexConstant", 192233.1060180664D) @@ -362,7 +399,7 @@ public void testTopNOverMissingUniques() TopNQuery query = new TopNQueryBuilder() .dataSource(QueryRunnerTestHelper.dataSource) .granularity(QueryRunnerTestHelper.allGran) - .dimension(QueryRunnerTestHelper.marketDimension) + .dimension(marketStr) .metric(QueryRunnerTestHelper.uniqueMetric) .threshold(3) .intervals(QueryRunnerTestHelper.fullOnInterval) @@ -370,26 +407,26 @@ public void testTopNOverMissingUniques() Arrays.asList(new HyperUniquesAggregatorFactory("uniques", "missingUniques")) ) .build(); + + List> expectedList = Arrays.>asList( + ImmutableMap.builder() + .put(marketStr, spotVal) + .put("uniques", 0) + .build(), + ImmutableMap.builder() + .put(marketStr, totalMarketVal) + .put("uniques", 0) + .build(), + ImmutableMap.builder() + .put(marketStr, upfrontVal) + .put("uniques", 0) + .build() + ); List> expectedResults = Arrays.asList( new Result( new DateTime("2011-01-12T00:00:00.000Z"), - new TopNResultValue( - Arrays.>asList( - ImmutableMap.builder() - .put("market", "total_market") - .put("uniques", 0) - .build(), - ImmutableMap.builder() - .put("market", "spot") - .put("uniques", 0) - .build(), - ImmutableMap.builder() - .put("market", "upfront") - .put("uniques", 0) - .build() - ) - ) + new TopNResultValue(expectedList) ) ); assertExpectedResults(expectedResults, query); @@ -401,7 +438,7 @@ public void testTopNOverHyperUniqueFinalizingPostAggregator() TopNQuery query = new TopNQueryBuilder() .dataSource(QueryRunnerTestHelper.dataSource) .granularity(QueryRunnerTestHelper.allGran) - .dimension(QueryRunnerTestHelper.marketDimension) + .dimension(marketStr) .metric(QueryRunnerTestHelper.hyperUniqueFinalizingPostAggMetric) .threshold(3) .intervals(QueryRunnerTestHelper.fullOnInterval) @@ -422,17 +459,17 @@ public void testTopNOverHyperUniqueFinalizingPostAggregator() new TopNResultValue( Arrays.>asList( ImmutableMap.builder() - .put("market", "spot") + .put(marketStr, spotVal) .put(QueryRunnerTestHelper.uniqueMetric, QueryRunnerTestHelper.UNIQUES_9) .put(QueryRunnerTestHelper.hyperUniqueFinalizingPostAggMetric, QueryRunnerTestHelper.UNIQUES_9) .build(), ImmutableMap.builder() - .put("market", "total_market") + .put(marketStr, totalMarketVal) .put(QueryRunnerTestHelper.uniqueMetric, QueryRunnerTestHelper.UNIQUES_2) .put(QueryRunnerTestHelper.hyperUniqueFinalizingPostAggMetric, QueryRunnerTestHelper.UNIQUES_2) .build(), ImmutableMap.builder() - .put("market", "upfront") + .put(marketStr, upfrontVal) .put(QueryRunnerTestHelper.uniqueMetric, QueryRunnerTestHelper.UNIQUES_2) .put(QueryRunnerTestHelper.hyperUniqueFinalizingPostAggMetric, QueryRunnerTestHelper.UNIQUES_2) .build() @@ -452,7 +489,7 @@ public void testTopNBySegment() TopNQuery query = new TopNQueryBuilder() .dataSource(QueryRunnerTestHelper.dataSource) .granularity(QueryRunnerTestHelper.allGran) - .dimension(QueryRunnerTestHelper.marketDimension) + .dimension(marketStr) .metric(QueryRunnerTestHelper.indexMetric) .threshold(4) .intervals(QueryRunnerTestHelper.firstToThird) @@ -470,21 +507,21 @@ public void testTopNBySegment() ImmutableMap.of( "addRowsIndexConstant", 5356.814697265625D, "index", 5351.814697265625D, - QueryRunnerTestHelper.marketDimension, "total_market", + marketStr, totalMarketVal, "uniques", QueryRunnerTestHelper.UNIQUES_2, "rows", 4L ), ImmutableMap.of( "addRowsIndexConstant", 4880.669677734375D, "index", 4875.669677734375D, - QueryRunnerTestHelper.marketDimension, "upfront", + marketStr, upfrontVal, "uniques", QueryRunnerTestHelper.UNIQUES_2, "rows", 4L ), ImmutableMap.of( "addRowsIndexConstant", 2250.8768157958984D, "index", 2231.8768157958984D, - QueryRunnerTestHelper.marketDimension, "spot", + marketStr, spotVal, "uniques", QueryRunnerTestHelper.UNIQUES_9, "rows", 18L ) @@ -558,7 +595,7 @@ public void testTopN() TopNQuery query = new TopNQueryBuilder() .dataSource(QueryRunnerTestHelper.dataSource) .granularity(QueryRunnerTestHelper.allGran) - .dimension(QueryRunnerTestHelper.marketDimension) + .dimension(marketStr) .metric(QueryRunnerTestHelper.indexMetric) .threshold(4) .intervals(QueryRunnerTestHelper.firstToThird) @@ -573,21 +610,21 @@ public void testTopN() new TopNResultValue( Arrays.>asList( ImmutableMap.of( - QueryRunnerTestHelper.marketDimension, "total_market", + marketStr, totalMarketVal, "rows", 4L, "index", 5351.814697265625D, "addRowsIndexConstant", 5356.814697265625D, "uniques", QueryRunnerTestHelper.UNIQUES_2 ), ImmutableMap.of( - QueryRunnerTestHelper.marketDimension, "upfront", + marketStr, upfrontVal, "rows", 4L, "index", 4875.669677734375D, "addRowsIndexConstant", 4880.669677734375D, "uniques", QueryRunnerTestHelper.UNIQUES_2 ), ImmutableMap.of( - QueryRunnerTestHelper.marketDimension, "spot", + marketStr, spotVal, "rows", 18L, "index", 2231.8768157958984D, "addRowsIndexConstant", 2250.8768157958984D, @@ -606,7 +643,7 @@ public void testTopNByUniques() TopNQuery query = new TopNQueryBuilder() .dataSource(QueryRunnerTestHelper.dataSource) .granularity(QueryRunnerTestHelper.allGran) - .dimension(QueryRunnerTestHelper.marketDimension) + .dimension(marketStr) .metric(new NumericTopNMetricSpec("uniques")) .threshold(4) .intervals(QueryRunnerTestHelper.firstToThird) @@ -621,21 +658,21 @@ public void testTopNByUniques() new TopNResultValue( Arrays.>asList( ImmutableMap.of( - "market", "spot", + marketStr, spotVal, "rows", 18L, "index", 2231.8768157958984D, "addRowsIndexConstant", 2250.8768157958984D, "uniques", QueryRunnerTestHelper.UNIQUES_9 ), ImmutableMap.of( - "market", "total_market", + marketStr, totalMarketVal, "rows", 4L, "index", 5351.814697265625D, "addRowsIndexConstant", 5356.814697265625D, "uniques", QueryRunnerTestHelper.UNIQUES_2 ), ImmutableMap.of( - "market", "upfront", + marketStr, upfrontVal, "rows", 4L, "index", 4875.669677734375D, "addRowsIndexConstant", 4880.669677734375D, @@ -654,8 +691,8 @@ public void testTopNWithOrFilter1() TopNQuery query = new TopNQueryBuilder() .dataSource(QueryRunnerTestHelper.dataSource) .granularity(QueryRunnerTestHelper.allGran) - .filters(QueryRunnerTestHelper.marketDimension, "total_market", "upfront", "spot") - .dimension(QueryRunnerTestHelper.marketDimension) + .filters(marketStr, totalMarketVal.toString(), upfrontVal.toString(), spotVal.toString()) + .dimension(marketStr) .metric(QueryRunnerTestHelper.indexMetric) .threshold(4) .intervals(QueryRunnerTestHelper.firstToThird) @@ -669,21 +706,21 @@ public void testTopNWithOrFilter1() new TopNResultValue( Arrays.>asList( ImmutableMap.of( - QueryRunnerTestHelper.marketDimension, "total_market", + marketStr, totalMarketVal, "rows", 4L, "index", 5351.814697265625D, "addRowsIndexConstant", 5356.814697265625D, "uniques", QueryRunnerTestHelper.UNIQUES_2 ), ImmutableMap.of( - QueryRunnerTestHelper.marketDimension, "upfront", + marketStr, upfrontVal, "rows", 4L, "index", 4875.669677734375D, "addRowsIndexConstant", 4880.669677734375D, "uniques", QueryRunnerTestHelper.UNIQUES_2 ), ImmutableMap.of( - QueryRunnerTestHelper.marketDimension, "spot", + marketStr, spotVal, "rows", 18L, "index", 2231.8768157958984D, "addRowsIndexConstant", 2250.8768157958984D, @@ -702,8 +739,8 @@ public void testTopNWithOrFilter2() TopNQuery query = new TopNQueryBuilder() .dataSource(QueryRunnerTestHelper.dataSource) .granularity(QueryRunnerTestHelper.allGran) - .filters(QueryRunnerTestHelper.marketDimension, "total_market", "upfront") - .dimension(QueryRunnerTestHelper.marketDimension) + .filters(marketStr, totalMarketVal.toString(), upfrontVal.toString()) + .dimension(marketStr) .metric(QueryRunnerTestHelper.indexMetric) .threshold(4) .intervals(QueryRunnerTestHelper.firstToThird) @@ -717,14 +754,14 @@ public void testTopNWithOrFilter2() new TopNResultValue( Arrays.>asList( ImmutableMap.of( - QueryRunnerTestHelper.marketDimension, "total_market", + marketStr, totalMarketVal, "rows", 4L, "index", 5351.814697265625D, "addRowsIndexConstant", 5356.814697265625D, "uniques", QueryRunnerTestHelper.UNIQUES_2 ), ImmutableMap.of( - QueryRunnerTestHelper.marketDimension, "upfront", + marketStr, upfrontVal, "rows", 4L, "index", 4875.669677734375D, "addRowsIndexConstant", 4880.669677734375D, @@ -743,8 +780,8 @@ public void testTopNWithFilter1() TopNQuery query = new TopNQueryBuilder() .dataSource(QueryRunnerTestHelper.dataSource) .granularity(QueryRunnerTestHelper.allGran) - .filters(QueryRunnerTestHelper.marketDimension, "upfront") - .dimension(QueryRunnerTestHelper.marketDimension) + .filters(marketStr, upfrontVal.toString()) + .dimension(marketStr) .metric(QueryRunnerTestHelper.indexMetric) .threshold(4) .intervals(QueryRunnerTestHelper.firstToThird) @@ -758,7 +795,7 @@ public void testTopNWithFilter1() new TopNResultValue( Arrays.>asList( ImmutableMap.of( - QueryRunnerTestHelper.marketDimension, "upfront", + marketStr, upfrontVal, "rows", 4L, "index", 4875.669677734375D, "addRowsIndexConstant", 4880.669677734375D, @@ -778,7 +815,7 @@ public void testTopNWithFilter2() .dataSource(QueryRunnerTestHelper.dataSource) .granularity(QueryRunnerTestHelper.allGran) .filters(QueryRunnerTestHelper.qualityDimension, "mezzanine") - .dimension(QueryRunnerTestHelper.marketDimension) + .dimension(marketStr) .metric(QueryRunnerTestHelper.indexMetric) .threshold(4) .intervals(QueryRunnerTestHelper.firstToThird) @@ -792,21 +829,21 @@ public void testTopNWithFilter2() new TopNResultValue( Arrays.>asList( ImmutableMap.of( - QueryRunnerTestHelper.marketDimension, "upfront", + marketStr, upfrontVal, "rows", 2L, "index", 2591.68359375D, "addRowsIndexConstant", 2594.68359375D, "uniques", QueryRunnerTestHelper.UNIQUES_1 ), ImmutableMap.of( - QueryRunnerTestHelper.marketDimension, "total_market", + marketStr, totalMarketVal, "rows", 2L, "index", 2508.39599609375D, "addRowsIndexConstant", 2511.39599609375D, "uniques", QueryRunnerTestHelper.UNIQUES_1 ), ImmutableMap.of( - QueryRunnerTestHelper.marketDimension, "spot", + marketStr, spotVal, "rows", 2L, "index", 220.63774871826172D, "addRowsIndexConstant", 223.63774871826172D, @@ -826,7 +863,7 @@ public void testTopNWithFilter2OneDay() .dataSource(QueryRunnerTestHelper.dataSource) .granularity(QueryRunnerTestHelper.allGran) .filters(QueryRunnerTestHelper.qualityDimension, "mezzanine") - .dimension(QueryRunnerTestHelper.marketDimension) + .dimension(marketStr) .metric(QueryRunnerTestHelper.indexMetric) .threshold(4) .intervals( @@ -844,21 +881,21 @@ public void testTopNWithFilter2OneDay() new TopNResultValue( Arrays.>asList( ImmutableMap.of( - QueryRunnerTestHelper.marketDimension, "upfront", + marketStr, upfrontVal, "rows", 1L, "index", new Float(1447.341160).doubleValue(), "addRowsIndexConstant", new Float(1449.341160).doubleValue(), "uniques", QueryRunnerTestHelper.UNIQUES_1 ), ImmutableMap.of( - QueryRunnerTestHelper.marketDimension, "total_market", + marketStr, totalMarketVal, "rows", 1L, "index", new Float(1314.839715).doubleValue(), "addRowsIndexConstant", new Float(1316.839715).doubleValue(), "uniques", QueryRunnerTestHelper.UNIQUES_1 ), ImmutableMap.of( - QueryRunnerTestHelper.marketDimension, "spot", + marketStr, spotVal, "rows", 1L, "index", new Float(109.705815).doubleValue(), "addRowsIndexConstant", new Float(111.705815).doubleValue(), @@ -877,8 +914,8 @@ public void testTopNWithNonExistentFilterInOr() TopNQuery query = new TopNQueryBuilder() .dataSource(QueryRunnerTestHelper.dataSource) .granularity(QueryRunnerTestHelper.allGran) - .filters(QueryRunnerTestHelper.marketDimension, "total_market", "upfront", "billyblank") - .dimension(QueryRunnerTestHelper.marketDimension) + .filters(marketStr, totalMarketVal.toString(), upfrontVal.toString(), getDimMapping("billyblank").toString()) + .dimension(marketStr) .metric(QueryRunnerTestHelper.indexMetric) .threshold(4) .intervals(QueryRunnerTestHelper.firstToThird) @@ -892,14 +929,14 @@ public void testTopNWithNonExistentFilterInOr() new TopNResultValue( Arrays.>asList( ImmutableMap.of( - QueryRunnerTestHelper.marketDimension, "total_market", + marketStr, totalMarketVal, "rows", 4L, "index", 5351.814697265625D, "addRowsIndexConstant", 5356.814697265625D, "uniques", QueryRunnerTestHelper.UNIQUES_2 ), ImmutableMap.of( - QueryRunnerTestHelper.marketDimension, "upfront", + marketStr, upfrontVal, "rows", 4L, "index", 4875.669677734375D, "addRowsIndexConstant", 4880.669677734375D, @@ -918,8 +955,8 @@ public void testTopNWithNonExistentFilter() TopNQuery query = new TopNQueryBuilder() .dataSource(QueryRunnerTestHelper.dataSource) .granularity(QueryRunnerTestHelper.allGran) - .filters(QueryRunnerTestHelper.marketDimension, "billyblank") - .dimension(QueryRunnerTestHelper.marketDimension) + .filters(marketStr, getDimMapping("billyblank").toString()) + .dimension(marketStr) .metric(QueryRunnerTestHelper.indexMetric) .threshold(4) .intervals(QueryRunnerTestHelper.firstToThird) @@ -944,8 +981,8 @@ public void testTopNWithNonExistentFilterMultiDim() .fields( Lists.newArrayList( Druids.newSelectorDimFilterBuilder() - .dimension(QueryRunnerTestHelper.marketDimension) - .value("billyblank") + .dimension(marketStr) + .value(getDimMapping("billyblank").toString()) .build(), Druids.newSelectorDimFilterBuilder() .dimension(QueryRunnerTestHelper.qualityDimension) @@ -957,7 +994,7 @@ public void testTopNWithNonExistentFilterMultiDim() .dataSource(QueryRunnerTestHelper.dataSource) .granularity(QueryRunnerTestHelper.allGran) .filters(andDimFilter) - .dimension(QueryRunnerTestHelper.marketDimension) + .dimension(marketStr) .metric(QueryRunnerTestHelper.indexMetric) .threshold(4) .intervals(QueryRunnerTestHelper.firstToThird) @@ -981,7 +1018,7 @@ public void testTopNWithMultiValueDimFilter1() .dataSource(QueryRunnerTestHelper.dataSource) .granularity(QueryRunnerTestHelper.allGran) .filters(QueryRunnerTestHelper.placementishDimension, "m") - .dimension(QueryRunnerTestHelper.marketDimension) + .dimension(marketStr) .metric(QueryRunnerTestHelper.indexMetric) .threshold(4) .intervals(QueryRunnerTestHelper.firstToThird) @@ -996,7 +1033,7 @@ public void testTopNWithMultiValueDimFilter1() .dataSource(QueryRunnerTestHelper.dataSource) .granularity(QueryRunnerTestHelper.allGran) .filters(QueryRunnerTestHelper.qualityDimension, "mezzanine") - .dimension(QueryRunnerTestHelper.marketDimension) + .dimension(marketStr) .metric(QueryRunnerTestHelper.indexMetric) .threshold(4) .intervals(QueryRunnerTestHelper.firstToThird) @@ -1226,7 +1263,7 @@ public void testTopNWithNonExistentDimensionAndActualFilter() TopNQuery query = new TopNQueryBuilder() .dataSource(QueryRunnerTestHelper.dataSource) .granularity(QueryRunnerTestHelper.allGran) - .filters(QueryRunnerTestHelper.marketDimension, "upfront") + .filters(marketStr, upfrontVal.toString()) .dimension("doesn't exist") .metric(QueryRunnerTestHelper.indexMetric) .threshold(4) @@ -1296,7 +1333,7 @@ public void testTopNLexicographic() TopNQuery query = new TopNQueryBuilder() .dataSource(QueryRunnerTestHelper.dataSource) .granularity(QueryRunnerTestHelper.allGran) - .dimension(QueryRunnerTestHelper.marketDimension) + .dimension(marketStr) .metric(new LexicographicTopNMetricSpec("")) .threshold(4) .intervals(QueryRunnerTestHelper.firstToThird) @@ -1310,21 +1347,21 @@ public void testTopNLexicographic() new TopNResultValue( Arrays.>asList( ImmutableMap.of( - QueryRunnerTestHelper.marketDimension, "spot", + marketStr, spotVal, "rows", 18L, "index", 2231.8768157958984D, "addRowsIndexConstant", 2250.8768157958984D, "uniques", QueryRunnerTestHelper.UNIQUES_9 ), ImmutableMap.of( - QueryRunnerTestHelper.marketDimension, "total_market", + marketStr, totalMarketVal, "rows", 4L, "index", 5351.814697265625D, "addRowsIndexConstant", 5356.814697265625D, "uniques", QueryRunnerTestHelper.UNIQUES_2 ), ImmutableMap.of( - QueryRunnerTestHelper.marketDimension, "upfront", + marketStr, upfrontVal, "rows", 4L, "index", 4875.669677734375D, "addRowsIndexConstant", 4880.669677734375D, @@ -1343,8 +1380,8 @@ public void testTopNLexicographicWithPreviousStop() TopNQuery query = new TopNQueryBuilder() .dataSource(QueryRunnerTestHelper.dataSource) .granularity(QueryRunnerTestHelper.allGran) - .dimension(QueryRunnerTestHelper.marketDimension) - .metric(new LexicographicTopNMetricSpec("spot")) + .dimension(marketStr) + .metric(new LexicographicTopNMetricSpec(spotVal.toString())) .threshold(4) .intervals(QueryRunnerTestHelper.firstToThird) .aggregators(QueryRunnerTestHelper.commonAggregators) @@ -1357,14 +1394,14 @@ public void testTopNLexicographicWithPreviousStop() new TopNResultValue( Arrays.>asList( ImmutableMap.of( - QueryRunnerTestHelper.marketDimension, "total_market", + marketStr, totalMarketVal, "rows", 4L, "index", 5351.814697265625D, "addRowsIndexConstant", 5356.814697265625D, "uniques", QueryRunnerTestHelper.UNIQUES_2 ), ImmutableMap.of( - QueryRunnerTestHelper.marketDimension, "upfront", + marketStr, upfrontVal, "rows", 4L, "index", 4875.669677734375D, "addRowsIndexConstant", 4880.669677734375D, @@ -1380,11 +1417,12 @@ public void testTopNLexicographicWithPreviousStop() @Test public void testTopNLexicographicWithNonExistingPreviousStop() { + String stopVal = keyType == ValueType.STRING ? "t" : "1112"; TopNQuery query = new TopNQueryBuilder() .dataSource(QueryRunnerTestHelper.dataSource) .granularity(QueryRunnerTestHelper.allGran) - .dimension(QueryRunnerTestHelper.marketDimension) - .metric(new LexicographicTopNMetricSpec("t")) + .dimension(marketStr) + .metric(new LexicographicTopNMetricSpec(stopVal)) .threshold(4) .intervals(QueryRunnerTestHelper.firstToThird) .aggregators(QueryRunnerTestHelper.commonAggregators) @@ -1397,14 +1435,14 @@ public void testTopNLexicographicWithNonExistingPreviousStop() new TopNResultValue( Arrays.>asList( ImmutableMap.of( - QueryRunnerTestHelper.marketDimension, "total_market", + marketStr, totalMarketVal, "rows", 4L, "index", 5351.814697265625D, "addRowsIndexConstant", 5356.814697265625D, "uniques", QueryRunnerTestHelper.UNIQUES_2 ), ImmutableMap.of( - QueryRunnerTestHelper.marketDimension, "upfront", + marketStr, upfrontVal, "rows", 4L, "index", 4875.669677734375D, "addRowsIndexConstant", 4880.669677734375D, @@ -1423,8 +1461,8 @@ public void testTopNInvertedLexicographicWithPreviousStop() TopNQuery query = new TopNQueryBuilder() .dataSource(QueryRunnerTestHelper.dataSource) .granularity(QueryRunnerTestHelper.allGran) - .dimension(QueryRunnerTestHelper.marketDimension) - .metric(new InvertedTopNMetricSpec(new LexicographicTopNMetricSpec("upfront"))) + .dimension(marketStr) + .metric(new InvertedTopNMetricSpec(new LexicographicTopNMetricSpec(upfrontVal.toString()))) .threshold(4) .intervals(QueryRunnerTestHelper.firstToThird) .aggregators(QueryRunnerTestHelper.commonAggregators) @@ -1437,14 +1475,14 @@ public void testTopNInvertedLexicographicWithPreviousStop() new TopNResultValue( Arrays.>asList( ImmutableMap.of( - QueryRunnerTestHelper.marketDimension, "total_market", + marketStr, totalMarketVal, "rows", 4L, "index", 5351.814697265625D, "addRowsIndexConstant", 5356.814697265625D, "uniques", QueryRunnerTestHelper.UNIQUES_2 ), ImmutableMap.of( - QueryRunnerTestHelper.marketDimension, "spot", + marketStr, spotVal, "rows", 18L, "index", 2231.8768157958984D, "addRowsIndexConstant", 2250.8768157958984D, @@ -1460,11 +1498,12 @@ public void testTopNInvertedLexicographicWithPreviousStop() @Test public void testTopNInvertedLexicographicWithNonExistingPreviousStop() { + String stopVal = keyType == ValueType.STRING ? "u" : "333.1"; TopNQuery query = new TopNQueryBuilder() .dataSource(QueryRunnerTestHelper.dataSource) .granularity(QueryRunnerTestHelper.allGran) - .dimension(QueryRunnerTestHelper.marketDimension) - .metric(new InvertedTopNMetricSpec(new LexicographicTopNMetricSpec("u"))) + .dimension(marketStr) + .metric(new InvertedTopNMetricSpec(new LexicographicTopNMetricSpec(stopVal))) .threshold(4) .intervals(QueryRunnerTestHelper.firstToThird) .aggregators(QueryRunnerTestHelper.commonAggregators) @@ -1477,14 +1516,14 @@ public void testTopNInvertedLexicographicWithNonExistingPreviousStop() new TopNResultValue( Arrays.>asList( ImmutableMap.of( - QueryRunnerTestHelper.marketDimension, "total_market", + marketStr, totalMarketVal, "rows", 4L, "index", 5351.814697265625D, "addRowsIndexConstant", 5356.814697265625D, "uniques", QueryRunnerTestHelper.UNIQUES_2 ), ImmutableMap.of( - QueryRunnerTestHelper.marketDimension, "spot", + marketStr, spotVal, "rows", 18L, "index", 2231.8768157958984D, "addRowsIndexConstant", 2250.8768157958984D, @@ -1506,8 +1545,8 @@ public void testTopNDimExtractionToOne() throws IOException .granularity(QueryRunnerTestHelper.allGran) .dimension( new ExtractionDimensionSpec( - QueryRunnerTestHelper.marketDimension, - QueryRunnerTestHelper.marketDimension, + marketStr, + marketStr, new JavaScriptExtractionFn("function(f) { return \"POTATO\"; }", false), null ) @@ -1541,7 +1580,7 @@ public void testTopNDimExtractionToOne() throws IOException ImmutableMap.of( "addRowsIndexConstant", 504542.5071372986D, "index", 503332.5071372986D, - QueryRunnerTestHelper.marketDimension, "POTATO", + marketStr, "POTATO", "uniques", QueryRunnerTestHelper.UNIQUES_9, "rows", 1209L ) @@ -1614,8 +1653,8 @@ public void testTopNDimExtraction() .granularity(QueryRunnerTestHelper.allGran) .dimension( new ExtractionDimensionSpec( - QueryRunnerTestHelper.marketDimension, - QueryRunnerTestHelper.marketDimension, + marketStr, + marketStr, new RegexDimExtractionFn("(.)", false, null), null ) @@ -1633,21 +1672,21 @@ public void testTopNDimExtraction() new TopNResultValue( Arrays.>asList( ImmutableMap.of( - QueryRunnerTestHelper.marketDimension, "s", + marketStr, spotVal.toString().substring(0,1), "rows", 18L, "index", 2231.8768157958984D, "addRowsIndexConstant", 2250.8768157958984D, "uniques", QueryRunnerTestHelper.UNIQUES_9 ), ImmutableMap.of( - QueryRunnerTestHelper.marketDimension, "t", + marketStr, totalMarketVal.toString().substring(0,1), "rows", 4L, "index", 5351.814697265625D, "addRowsIndexConstant", 5356.814697265625D, "uniques", QueryRunnerTestHelper.UNIQUES_2 ), ImmutableMap.of( - QueryRunnerTestHelper.marketDimension, "u", + marketStr, upfrontVal.toString().substring(0,1), "rows", 4L, "index", 4875.669677734375D, "addRowsIndexConstant", 4880.669677734375D, @@ -1669,14 +1708,14 @@ public void testTopNDimExtractionFastTopNOptimalWithReplaceMissing() .granularity(QueryRunnerTestHelper.allGran) .dimension( new ExtractionDimensionSpec( - QueryRunnerTestHelper.marketDimension, - QueryRunnerTestHelper.marketDimension, + marketStr, + marketStr, new LookupExtractionFn( new MapLookupExtractor( ImmutableMap.of( - "spot", "2spot0", - "total_market", "1total_market0", - "upfront", "3upfront0" + spotVal.toString(), "2spot0", + totalMarketVal.toString(), "1total_market0", + upfrontVal.toString(), "3upfront0" ), false ), false, "MISSING", true, @@ -1698,21 +1737,21 @@ public void testTopNDimExtractionFastTopNOptimalWithReplaceMissing() new TopNResultValue( Arrays.>asList( ImmutableMap.of( - QueryRunnerTestHelper.marketDimension, "2spot0", + marketStr, "2spot0", "rows", 18L, "index", 2231.8768157958984D, "addRowsIndexConstant", 2250.8768157958984D, "uniques", QueryRunnerTestHelper.UNIQUES_9 ), ImmutableMap.of( - QueryRunnerTestHelper.marketDimension, "1total_market0", + marketStr, "1total_market0", "rows", 4L, "index", 5351.814697265625D, "addRowsIndexConstant", 5356.814697265625D, "uniques", QueryRunnerTestHelper.UNIQUES_2 ), ImmutableMap.of( - QueryRunnerTestHelper.marketDimension, "3upfront0", + marketStr, "3upfront0", "rows", 4L, "index", 4875.669677734375D, "addRowsIndexConstant", 4880.669677734375D, @@ -1734,14 +1773,14 @@ public void testTopNDimExtractionFastTopNUnOptimalWithReplaceMissing() .granularity(QueryRunnerTestHelper.allGran) .dimension( new ExtractionDimensionSpec( - QueryRunnerTestHelper.marketDimension, - QueryRunnerTestHelper.marketDimension, + marketStr, + marketStr, new LookupExtractionFn( new MapLookupExtractor( ImmutableMap.of( - "spot", "2spot0", - "total_market", "1total_market0", - "upfront", "3upfront0" + spotVal.toString(), "2spot0", + totalMarketVal.toString(), "1total_market0", + upfrontVal.toString(), "3upfront0" ), false ), false, "MISSING", false, @@ -1763,21 +1802,21 @@ public void testTopNDimExtractionFastTopNUnOptimalWithReplaceMissing() new TopNResultValue( Arrays.>asList( ImmutableMap.of( - QueryRunnerTestHelper.marketDimension, "2spot0", + marketStr, "2spot0", "rows", 18L, "index", 2231.8768157958984D, "addRowsIndexConstant", 2250.8768157958984D, "uniques", QueryRunnerTestHelper.UNIQUES_9 ), ImmutableMap.of( - QueryRunnerTestHelper.marketDimension, "1total_market0", + marketStr, "1total_market0", "rows", 4L, "index", 5351.814697265625D, "addRowsIndexConstant", 5356.814697265625D, "uniques", QueryRunnerTestHelper.UNIQUES_2 ), ImmutableMap.of( - QueryRunnerTestHelper.marketDimension, "3upfront0", + marketStr, "3upfront0", "rows", 4L, "index", 4875.669677734375D, "addRowsIndexConstant", 4880.669677734375D, @@ -1800,14 +1839,14 @@ public void testTopNDimExtractionFastTopNOptimal() .granularity(QueryRunnerTestHelper.allGran) .dimension( new ExtractionDimensionSpec( - QueryRunnerTestHelper.marketDimension, - QueryRunnerTestHelper.marketDimension, + marketStr, + marketStr, new LookupExtractionFn( new MapLookupExtractor( ImmutableMap.of( - "spot", "2spot0", - "total_market", "1total_market0", - "upfront", "3upfront0" + spotVal.toString(), "2spot0", + totalMarketVal.toString(), "1total_market0", + upfrontVal.toString(), "3upfront0" ), false ), true, null, true, @@ -1829,21 +1868,21 @@ public void testTopNDimExtractionFastTopNOptimal() new TopNResultValue( Arrays.>asList( ImmutableMap.of( - QueryRunnerTestHelper.marketDimension, "2spot0", + marketStr, "2spot0", "rows", 18L, "index", 2231.8768157958984D, "addRowsIndexConstant", 2250.8768157958984D, "uniques", QueryRunnerTestHelper.UNIQUES_9 ), ImmutableMap.of( - QueryRunnerTestHelper.marketDimension, "1total_market0", + marketStr, "1total_market0", "rows", 4L, "index", 5351.814697265625D, "addRowsIndexConstant", 5356.814697265625D, "uniques", QueryRunnerTestHelper.UNIQUES_2 ), ImmutableMap.of( - QueryRunnerTestHelper.marketDimension, "3upfront0", + marketStr, "3upfront0", "rows", 4L, "index", 4875.669677734375D, "addRowsIndexConstant", 4880.669677734375D, @@ -1865,16 +1904,16 @@ public void testTopNDimExtractionFastTopNUnOptimal() .granularity(QueryRunnerTestHelper.allGran) .dimension( new ExtractionDimensionSpec( - QueryRunnerTestHelper.marketDimension, - QueryRunnerTestHelper.marketDimension, + marketStr, + marketStr, new LookupExtractionFn( new MapLookupExtractor( ImmutableMap.of( - "spot", + spotVal.toString(), "spot0", - "total_market", + totalMarketVal.toString(), "total_market0", - "upfront", + upfrontVal.toString(), "upfront0" ), false @@ -1897,21 +1936,21 @@ public void testTopNDimExtractionFastTopNUnOptimal() new TopNResultValue( Arrays.>asList( ImmutableMap.of( - QueryRunnerTestHelper.marketDimension, "spot0", + marketStr, "spot0", "rows", 18L, "index", 2231.8768157958984D, "addRowsIndexConstant", 2250.8768157958984D, "uniques", QueryRunnerTestHelper.UNIQUES_9 ), ImmutableMap.of( - QueryRunnerTestHelper.marketDimension, "total_market0", + marketStr, "total_market0", "rows", 4L, "index", 5351.814697265625D, "addRowsIndexConstant", 5356.814697265625D, "uniques", QueryRunnerTestHelper.UNIQUES_2 ), ImmutableMap.of( - QueryRunnerTestHelper.marketDimension, "upfront0", + marketStr, "upfront0", "rows", 4L, "index", 4875.669677734375D, "addRowsIndexConstant", 4880.669677734375D, @@ -1932,16 +1971,16 @@ public void testTopNLexicographicDimExtractionOptimalNamespace() .granularity(QueryRunnerTestHelper.allGran) .dimension( new ExtractionDimensionSpec( - QueryRunnerTestHelper.marketDimension, - QueryRunnerTestHelper.marketDimension, + marketStr, + marketStr, new LookupExtractionFn( new MapLookupExtractor( ImmutableMap.of( - "spot", + spotVal.toString(), "2spot", - "total_market", + totalMarketVal.toString(), "3total_market", - "upfront", + upfrontVal.toString(), "1upfront" ), false @@ -1964,21 +2003,21 @@ public void testTopNLexicographicDimExtractionOptimalNamespace() new TopNResultValue( Arrays.>asList( ImmutableMap.of( - QueryRunnerTestHelper.marketDimension, "1upfront", + marketStr, "1upfront", "rows", 4L, "index", 4875.669677734375D, "addRowsIndexConstant", 4880.669677734375D, "uniques", QueryRunnerTestHelper.UNIQUES_2 ), ImmutableMap.of( - QueryRunnerTestHelper.marketDimension, "2spot", + marketStr, "2spot", "rows", 18L, "index", 2231.8768157958984D, "addRowsIndexConstant", 2250.8768157958984D, "uniques", QueryRunnerTestHelper.UNIQUES_9 ), ImmutableMap.of( - QueryRunnerTestHelper.marketDimension, "3total_market", + marketStr, "3total_market", "rows", 4L, "index", 5351.814697265625D, "addRowsIndexConstant", 5356.814697265625D, @@ -1999,16 +2038,16 @@ public void testTopNLexicographicDimExtractionUnOptimalNamespace() .granularity(QueryRunnerTestHelper.allGran) .dimension( new ExtractionDimensionSpec( - QueryRunnerTestHelper.marketDimension, - QueryRunnerTestHelper.marketDimension, + marketStr, + marketStr, new LookupExtractionFn( new MapLookupExtractor( ImmutableMap.of( - "spot", + spotVal.toString(), "2spot", - "total_market", + totalMarketVal.toString(), "3total_market", - "upfront", + upfrontVal.toString(), "1upfront" ), false @@ -2031,21 +2070,21 @@ public void testTopNLexicographicDimExtractionUnOptimalNamespace() new TopNResultValue( Arrays.>asList( ImmutableMap.of( - QueryRunnerTestHelper.marketDimension, "1upfront", + marketStr, "1upfront", "rows", 4L, "index", 4875.669677734375D, "addRowsIndexConstant", 4880.669677734375D, "uniques", QueryRunnerTestHelper.UNIQUES_2 ), ImmutableMap.of( - QueryRunnerTestHelper.marketDimension, "2spot", + marketStr, "2spot", "rows", 18L, "index", 2231.8768157958984D, "addRowsIndexConstant", 2250.8768157958984D, "uniques", QueryRunnerTestHelper.UNIQUES_9 ), ImmutableMap.of( - QueryRunnerTestHelper.marketDimension, "3total_market", + marketStr, "3total_market", "rows", 4L, "index", 5351.814697265625D, "addRowsIndexConstant", 5356.814697265625D, @@ -2067,16 +2106,16 @@ public void testTopNLexicographicDimExtractionOptimalNamespaceWithRunner() .granularity(QueryRunnerTestHelper.allGran) .dimension( new ExtractionDimensionSpec( - QueryRunnerTestHelper.marketDimension, - QueryRunnerTestHelper.marketDimension, + marketStr, + marketStr, new LookupExtractionFn( new MapLookupExtractor( ImmutableMap.of( - "spot", + spotVal.toString(), "2spot", - "total_market", + totalMarketVal.toString(), "3total_market", - "upfront", + upfrontVal.toString(), "1upfront" ), false @@ -2099,21 +2138,21 @@ public void testTopNLexicographicDimExtractionOptimalNamespaceWithRunner() new TopNResultValue( Arrays.>asList( ImmutableMap.of( - QueryRunnerTestHelper.marketDimension, "1upfront", + marketStr, "1upfront", "rows", 4L, "index", 4875.669677734375D, "addRowsIndexConstant", 4880.669677734375D, "uniques", QueryRunnerTestHelper.UNIQUES_2 ), ImmutableMap.of( - QueryRunnerTestHelper.marketDimension, "2spot", + marketStr, "2spot", "rows", 18L, "index", 2231.8768157958984D, "addRowsIndexConstant", 2250.8768157958984D, "uniques", QueryRunnerTestHelper.UNIQUES_9 ), ImmutableMap.of( - QueryRunnerTestHelper.marketDimension, "3total_market", + marketStr, "3total_market", "rows", 4L, "index", 5351.814697265625D, "addRowsIndexConstant", 5356.814697265625D, @@ -2134,8 +2173,8 @@ public void testTopNLexicographicDimExtraction() .granularity(QueryRunnerTestHelper.allGran) .dimension( new ExtractionDimensionSpec( - QueryRunnerTestHelper.marketDimension, - QueryRunnerTestHelper.marketDimension, + marketStr, + marketStr, new RegexDimExtractionFn("(.)", false, null), null ) @@ -2153,21 +2192,21 @@ public void testTopNLexicographicDimExtraction() new TopNResultValue( Arrays.>asList( ImmutableMap.of( - QueryRunnerTestHelper.marketDimension, "s", + marketStr, spotVal.toString().substring(0, 1), "rows", 18L, "index", 2231.8768157958984D, "addRowsIndexConstant", 2250.8768157958984D, "uniques", QueryRunnerTestHelper.UNIQUES_9 ), ImmutableMap.of( - QueryRunnerTestHelper.marketDimension, "t", + marketStr, totalMarketVal.toString().substring(0,1), "rows", 4L, "index", 5351.814697265625D, "addRowsIndexConstant", 5356.814697265625D, "uniques", QueryRunnerTestHelper.UNIQUES_2 ), ImmutableMap.of( - QueryRunnerTestHelper.marketDimension, "u", + marketStr, upfrontVal.toString().substring(0,1), "rows", 4L, "index", 4875.669677734375D, "addRowsIndexConstant", 4880.669677734375D, @@ -2188,8 +2227,8 @@ public void testInvertedTopNLexicographicDimExtraction2() .granularity(QueryRunnerTestHelper.allGran) .dimension( new ExtractionDimensionSpec( - QueryRunnerTestHelper.marketDimension, - QueryRunnerTestHelper.marketDimension, + marketStr, + marketStr, new RegexDimExtractionFn("..(.)", false, null), null ) @@ -2201,34 +2240,61 @@ public void testInvertedTopNLexicographicDimExtraction2() .postAggregators(Arrays.asList(QueryRunnerTestHelper.addRowsIndexConstant)) .build(); + List> expectedList; + if (keyType == ValueType.STRING) { + expectedList = Arrays.>asList( + ImmutableMap.of( + marketStr, "t", + "rows", 4L, + "index", 5351.814697265625D, + "addRowsIndexConstant", 5356.814697265625D, + "uniques", QueryRunnerTestHelper.UNIQUES_2 + ), + ImmutableMap.of( + marketStr, "o", + "rows", 18L, + "index", 2231.8768157958984D, + "addRowsIndexConstant", 2250.8768157958984D, + "uniques", QueryRunnerTestHelper.UNIQUES_9 + ), + ImmutableMap.of( + marketStr, "f", + "rows", 4L, + "index", 4875.669677734375D, + "addRowsIndexConstant", 4880.669677734375D, + "uniques", QueryRunnerTestHelper.UNIQUES_2 + ) + ); + } else { + expectedList = Arrays.>asList( + ImmutableMap.of( + marketStr, "3", + "rows", 4L, + "index", 4875.669677734375D, + "addRowsIndexConstant", 4880.669677734375D, + "uniques", QueryRunnerTestHelper.UNIQUES_2 + ), + ImmutableMap.of( + marketStr, "2", + "rows", 4L, + "index", 5351.814697265625D, + "addRowsIndexConstant", 5356.814697265625D, + "uniques", QueryRunnerTestHelper.UNIQUES_2 + ), + ImmutableMap.of( + marketStr, "1", + "rows", 18L, + "index", 2231.8768157958984D, + "addRowsIndexConstant", 2250.8768157958984D, + "uniques", QueryRunnerTestHelper.UNIQUES_9 + ) + ); + } + List> expectedResults = Arrays.asList( new Result( new DateTime("2011-04-01T00:00:00.000Z"), - new TopNResultValue( - Arrays.>asList( - ImmutableMap.of( - QueryRunnerTestHelper.marketDimension, "t", - "rows", 4L, - "index", 5351.814697265625D, - "addRowsIndexConstant", 5356.814697265625D, - "uniques", QueryRunnerTestHelper.UNIQUES_2 - ), - ImmutableMap.of( - QueryRunnerTestHelper.marketDimension, "o", - "rows", 18L, - "index", 2231.8768157958984D, - "addRowsIndexConstant", 2250.8768157958984D, - "uniques", QueryRunnerTestHelper.UNIQUES_9 - ), - ImmutableMap.of( - QueryRunnerTestHelper.marketDimension, "f", - "rows", 4L, - "index", 4875.669677734375D, - "addRowsIndexConstant", 4880.669677734375D, - "uniques", QueryRunnerTestHelper.UNIQUES_2 - ) - ) - ) + new TopNResultValue(expectedList) ) ); assertExpectedResults(expectedResults, query); @@ -2237,18 +2303,19 @@ public void testInvertedTopNLexicographicDimExtraction2() @Test public void testTopNLexicographicDimExtractionWithPreviousStop() { + String stopValue = keyType == ValueType.STRING ? "s" : "1"; TopNQuery query = new TopNQueryBuilder() .dataSource(QueryRunnerTestHelper.dataSource) .granularity(QueryRunnerTestHelper.allGran) .dimension( new ExtractionDimensionSpec( - QueryRunnerTestHelper.marketDimension, - QueryRunnerTestHelper.marketDimension, + marketStr, + marketStr, new RegexDimExtractionFn("(.)", false, null), null ) ) - .metric(new LexicographicTopNMetricSpec("s")) + .metric(new LexicographicTopNMetricSpec(stopValue)) .threshold(4) .intervals(QueryRunnerTestHelper.firstToThird) .aggregators(QueryRunnerTestHelper.commonAggregators) @@ -2261,14 +2328,14 @@ public void testTopNLexicographicDimExtractionWithPreviousStop() new TopNResultValue( Arrays.>asList( ImmutableMap.of( - QueryRunnerTestHelper.marketDimension, "t", + marketStr, totalMarketVal.toString().substring(0,1), "rows", 4L, "index", 5351.814697265625D, "addRowsIndexConstant", 5356.814697265625D, "uniques", QueryRunnerTestHelper.UNIQUES_2 ), ImmutableMap.of( - QueryRunnerTestHelper.marketDimension, "u", + marketStr, upfrontVal.toString().substring(0,1), "rows", 4L, "index", 4875.669677734375D, "addRowsIndexConstant", 4880.669677734375D, @@ -2284,12 +2351,13 @@ public void testTopNLexicographicDimExtractionWithPreviousStop() @Test public void testTopNLexicographicDimExtractionWithSortingPreservedAndPreviousStop() { + String stopValue = keyType == ValueType.STRING ? "s" : "1"; TopNQuery query = new TopNQueryBuilder() .dataSource(QueryRunnerTestHelper.dataSource) .granularity(QueryRunnerTestHelper.allGran) .dimension( new ExtractionDimensionSpec( - QueryRunnerTestHelper.marketDimension, QueryRunnerTestHelper.marketDimension, + marketStr, marketStr, new DimExtractionFn() { @Override @@ -2318,7 +2386,7 @@ public ExtractionType getExtractionType() }, null ) ) - .metric(new LexicographicTopNMetricSpec("s")) + .metric(new LexicographicTopNMetricSpec(stopValue)) .threshold(4) .intervals(QueryRunnerTestHelper.firstToThird) .aggregators(QueryRunnerTestHelper.commonAggregators) @@ -2331,14 +2399,14 @@ public ExtractionType getExtractionType() new TopNResultValue( Arrays.>asList( ImmutableMap.of( - QueryRunnerTestHelper.marketDimension, "t", + marketStr, totalMarketVal.toString().substring(0,1), "rows", 4L, "index", 5351.814697265625D, "addRowsIndexConstant", 5356.814697265625D, "uniques", QueryRunnerTestHelper.UNIQUES_2 ), ImmutableMap.of( - QueryRunnerTestHelper.marketDimension, "u", + marketStr, upfrontVal.toString().substring(0,1), "rows", 4L, "index", 4875.669677734375D, "addRowsIndexConstant", 4880.669677734375D, @@ -2360,8 +2428,8 @@ public void testInvertedTopNLexicographicDimExtractionWithPreviousStop() .granularity(QueryRunnerTestHelper.allGran) .dimension( new ExtractionDimensionSpec( - QueryRunnerTestHelper.marketDimension, - QueryRunnerTestHelper.marketDimension, + marketStr, + marketStr, new RegexDimExtractionFn("(.)", false, null), null ) @@ -2373,27 +2441,54 @@ public void testInvertedTopNLexicographicDimExtractionWithPreviousStop() .postAggregators(Arrays.asList(QueryRunnerTestHelper.addRowsIndexConstant)) .build(); + List> expectedList; + if (keyType == ValueType.STRING) { + expectedList = Arrays.>asList( + ImmutableMap.of( + marketStr, "t", + "rows", 4L, + "index", 5351.814697265625D, + "addRowsIndexConstant", 5356.814697265625D, + "uniques", QueryRunnerTestHelper.UNIQUES_2 + ), + ImmutableMap.of( + marketStr, "s", + "rows", 18L, + "index", 2231.8768157958984D, + "addRowsIndexConstant", 2250.8768157958984D, + "uniques", QueryRunnerTestHelper.UNIQUES_9 + ) + ); + } else { + expectedList = Arrays.>asList( + ImmutableMap.of( + marketStr, "3", + "rows", 4L, + "index", 4875.669677734375D, + "addRowsIndexConstant", 4880.669677734375D, + "uniques", QueryRunnerTestHelper.UNIQUES_2 + ), + ImmutableMap.of( + marketStr, "2", + "rows", 4L, + "index", 5351.814697265625D, + "addRowsIndexConstant", 5356.814697265625D, + "uniques", QueryRunnerTestHelper.UNIQUES_2 + ), + ImmutableMap.of( + marketStr, "1", + "rows", 18L, + "index", 2231.8768157958984D, + "addRowsIndexConstant", 2250.8768157958984D, + "uniques", QueryRunnerTestHelper.UNIQUES_9 + ) + ); + } + List> expectedResults = Arrays.asList( new Result( new DateTime("2011-04-01T00:00:00.000Z"), - new TopNResultValue( - Arrays.>asList( - ImmutableMap.of( - QueryRunnerTestHelper.marketDimension, "t", - "rows", 4L, - "index", 5351.814697265625D, - "addRowsIndexConstant", 5356.814697265625D, - "uniques", QueryRunnerTestHelper.UNIQUES_2 - ), - ImmutableMap.of( - QueryRunnerTestHelper.marketDimension, "s", - "rows", 18L, - "index", 2231.8768157958984D, - "addRowsIndexConstant", 2250.8768157958984D, - "uniques", QueryRunnerTestHelper.UNIQUES_9 - ) - ) - ) + new TopNResultValue(expectedList) ) ); assertExpectedResults(expectedResults, query); @@ -2402,45 +2497,66 @@ public void testInvertedTopNLexicographicDimExtractionWithPreviousStop() @Test public void testInvertedTopNLexicographicDimExtractionWithPreviousStop2() { + String stopValue = keyType == ValueType.STRING ? "p" : "3"; TopNQuery query = new TopNQueryBuilder() .dataSource(QueryRunnerTestHelper.dataSource) .granularity(QueryRunnerTestHelper.allGran) .dimension( new ExtractionDimensionSpec( - QueryRunnerTestHelper.marketDimension, - QueryRunnerTestHelper.marketDimension, + marketStr, + marketStr, new RegexDimExtractionFn("..(.)", false, null), null ) ) - .metric(new InvertedTopNMetricSpec(new LexicographicTopNMetricSpec("p"))) + .metric(new InvertedTopNMetricSpec(new LexicographicTopNMetricSpec(stopValue))) .threshold(4) .intervals(QueryRunnerTestHelper.firstToThird) .aggregators(QueryRunnerTestHelper.commonAggregators) .postAggregators(Arrays.asList(QueryRunnerTestHelper.addRowsIndexConstant)) .build(); + List> expectedList; + if (keyType == ValueType.STRING) { + expectedList = Arrays.>asList( + ImmutableMap.of( + marketStr, "o", + "rows", 18L, + "index", 2231.8768157958984D, + "addRowsIndexConstant", 2250.8768157958984D, + "uniques", QueryRunnerTestHelper.UNIQUES_9 + ), + ImmutableMap.of( + marketStr, "f", + "rows", 4L, + "index", 4875.669677734375D, + "addRowsIndexConstant", 4880.669677734375D, + "uniques", QueryRunnerTestHelper.UNIQUES_2 + ) + ); + } else { + expectedList = Arrays.>asList( + ImmutableMap.of( + marketStr, "2", + "rows", 4L, + "index", 5351.814697265625D, + "addRowsIndexConstant", 5356.814697265625D, + "uniques", QueryRunnerTestHelper.UNIQUES_2 + ), + ImmutableMap.of( + marketStr, "1", + "rows", 18L, + "index", 2231.8768157958984D, + "addRowsIndexConstant", 2250.8768157958984D, + "uniques", QueryRunnerTestHelper.UNIQUES_9 + ) + ); + } + List> expectedResults = Arrays.asList( new Result( new DateTime("2011-04-01T00:00:00.000Z"), - new TopNResultValue( - Arrays.>asList( - ImmutableMap.of( - QueryRunnerTestHelper.marketDimension, "o", - "rows", 18L, - "index", 2231.8768157958984D, - "addRowsIndexConstant", 2250.8768157958984D, - "uniques", QueryRunnerTestHelper.UNIQUES_9 - ), - ImmutableMap.of( - QueryRunnerTestHelper.marketDimension, "f", - "rows", 4L, - "index", 4875.669677734375D, - "addRowsIndexConstant", 4880.669677734375D, - "uniques", QueryRunnerTestHelper.UNIQUES_2 - ) - ) - ) + new TopNResultValue(expectedList) ) ); assertExpectedResults(expectedResults, query); @@ -2460,7 +2576,7 @@ public byte[] getCacheKey() @Override public String apply(String dimValue) { - return dimValue.equals("total_market") ? null : dimValue; + return dimValue.equals(totalMarketVal.toString()) ? null : dimValue; } @Override @@ -2486,8 +2602,8 @@ public ExtractionType getExtractionType() .postAggregators(Arrays.asList(QueryRunnerTestHelper.addRowsIndexConstant)) .dimension( new ExtractionDimensionSpec( - QueryRunnerTestHelper.marketDimension, - QueryRunnerTestHelper.marketDimension, + marketStr, + marketStr, nullStringDimExtraction, null ) @@ -2501,7 +2617,7 @@ public ExtractionType getExtractionType() new TopNResultValue( Arrays.>asList( ImmutableMap.of( - QueryRunnerTestHelper.marketDimension, "spot", + marketStr, spotVal.toString(), "rows", 18L, "index", 2231.8768157958984D, "addRowsIndexConstant", 2250.8768157958984D, @@ -2509,14 +2625,14 @@ public ExtractionType getExtractionType() ), new LinkedHashMap() {{ - put(QueryRunnerTestHelper.marketDimension, null); + put(marketStr, null); put("rows", 4L); put("index", 5351.814697265625D); put("addRowsIndexConstant", 5356.814697265625D); put("uniques", QueryRunnerTestHelper.UNIQUES_2); }}, ImmutableMap.of( - QueryRunnerTestHelper.marketDimension, "upfront", + marketStr, upfrontVal.toString(), "rows", 4L, "index", 4875.669677734375D, "addRowsIndexConstant", 4880.669677734375D, @@ -2550,7 +2666,7 @@ public byte[] getCacheKey() @Override public String apply(String dimValue) { - return dimValue.equals("total_market") ? "" : dimValue; + return dimValue.equals(totalMarketVal.toString()) ? "" : dimValue; } @Override @@ -2576,8 +2692,8 @@ public ExtractionType getExtractionType() .postAggregators(Arrays.asList(QueryRunnerTestHelper.addRowsIndexConstant)) .dimension( new ExtractionDimensionSpec( - QueryRunnerTestHelper.marketDimension, - QueryRunnerTestHelper.marketDimension, + marketStr, + marketStr, emptyStringDimExtraction, null ) @@ -2591,7 +2707,7 @@ public ExtractionType getExtractionType() new TopNResultValue( Arrays.>asList( ImmutableMap.of( - QueryRunnerTestHelper.marketDimension, "spot", + marketStr, spotVal.toString(), "rows", 18L, "index", 2231.8768157958984D, "addRowsIndexConstant", 2250.8768157958984D, @@ -2599,14 +2715,14 @@ public ExtractionType getExtractionType() ), new LinkedHashMap() {{ - put(QueryRunnerTestHelper.marketDimension, ""); + put(marketStr, ""); put("rows", 4L); put("index", 5351.814697265625D); put("addRowsIndexConstant", 5356.814697265625D); put("uniques", QueryRunnerTestHelper.UNIQUES_2); }}, ImmutableMap.of( - QueryRunnerTestHelper.marketDimension, "upfront", + marketStr, upfrontVal.toString(), "rows", 4L, "index", 4875.669677734375D, "addRowsIndexConstant", 4880.669677734375D, @@ -2627,7 +2743,7 @@ public void testInvertedTopNQuery() new TopNQueryBuilder() .dataSource(QueryRunnerTestHelper.dataSource) .granularity(QueryRunnerTestHelper.allGran) - .dimension(QueryRunnerTestHelper.marketDimension) + .dimension(marketStr) .metric(new InvertedTopNMetricSpec(new NumericTopNMetricSpec(QueryRunnerTestHelper.indexMetric))) .threshold(3) .intervals(QueryRunnerTestHelper.firstToThird) @@ -2641,21 +2757,21 @@ public void testInvertedTopNQuery() new TopNResultValue( Arrays.>asList( ImmutableMap.of( - QueryRunnerTestHelper.marketDimension, "spot", + marketStr, spotVal, "rows", 18L, "index", 2231.8768157958984D, "addRowsIndexConstant", 2250.8768157958984D, "uniques", QueryRunnerTestHelper.UNIQUES_9 ), ImmutableMap.of( - QueryRunnerTestHelper.marketDimension, "upfront", + marketStr, upfrontVal, "rows", 4L, "index", 4875.669677734375D, "addRowsIndexConstant", 4880.669677734375D, "uniques", QueryRunnerTestHelper.UNIQUES_2 ), ImmutableMap.of( - QueryRunnerTestHelper.marketDimension, "total_market", + marketStr, totalMarketVal, "rows", 4L, "index", 5351.814697265625D, "addRowsIndexConstant", 5356.814697265625D, @@ -2675,14 +2791,14 @@ public void testTopNQueryByComplexMetric() new TopNQueryBuilder() .dataSource(QueryRunnerTestHelper.dataSource) .granularity(QueryRunnerTestHelper.allGran) - .dimension(QueryRunnerTestHelper.marketDimension) + .dimension(marketStr) .metric(new NumericTopNMetricSpec("numVals")) .threshold(10) .intervals(QueryRunnerTestHelper.firstToThird) .aggregators( Lists.newArrayList( new CardinalityAggregatorFactory( - "numVals", ImmutableList.of(QueryRunnerTestHelper.marketDimension), false + "numVals", ImmutableList.of(marketStr), false ) ) ) @@ -2694,15 +2810,15 @@ public void testTopNQueryByComplexMetric() new TopNResultValue( Arrays.>asList( ImmutableMap.of( - "market", "spot", + marketStr, spotVal, "numVals", 1.0002442201269182d ), ImmutableMap.of( - "market", "total_market", + marketStr, totalMarketVal, "numVals", 1.0002442201269182d ), ImmutableMap.of( - "market", "upfront", + marketStr, upfrontVal, "numVals", 1.0002442201269182d ) ) @@ -2718,7 +2834,7 @@ public void testTopNDependentPostAgg() TopNQuery query = new TopNQueryBuilder() .dataSource(QueryRunnerTestHelper.dataSource) .granularity(QueryRunnerTestHelper.allGran) - .dimension(QueryRunnerTestHelper.marketDimension) + .dimension(marketStr) .metric(QueryRunnerTestHelper.dependentPostAggMetric) .threshold(4) .intervals(QueryRunnerTestHelper.fullOnInterval) @@ -2748,7 +2864,7 @@ public void testTopNDependentPostAgg() new TopNResultValue( Arrays.>asList( ImmutableMap.builder() - .put(QueryRunnerTestHelper.marketDimension, "total_market") + .put(marketStr, totalMarketVal) .put("rows", 186L) .put("index", 215679.82879638672D) .put("addRowsIndexConstant", 215866.82879638672D) @@ -2762,7 +2878,7 @@ public void testTopNDependentPostAgg() ) .build(), ImmutableMap.builder() - .put(QueryRunnerTestHelper.marketDimension, "upfront") + .put(marketStr, upfrontVal) .put("rows", 186L) .put("index", 192046.1060180664D) .put("addRowsIndexConstant", 192233.1060180664D) @@ -2776,7 +2892,7 @@ public void testTopNDependentPostAgg() ) .build(), ImmutableMap.builder() - .put(QueryRunnerTestHelper.marketDimension, "spot") + .put(marketStr, spotVal) .put("rows", 837L) .put("index", 95606.57232284546D) .put("addRowsIndexConstant", 96444.57232284546D) @@ -2802,7 +2918,7 @@ public void testTopNBySegmentResults() TopNQuery query = new TopNQueryBuilder() .dataSource(QueryRunnerTestHelper.dataSource) .granularity(QueryRunnerTestHelper.allGran) - .dimension(QueryRunnerTestHelper.marketDimension) + .dimension(marketStr) .metric(QueryRunnerTestHelper.dependentPostAggMetric) .threshold(4) .intervals(QueryRunnerTestHelper.fullOnInterval) @@ -2828,7 +2944,7 @@ public void testTopNBySegmentResults() TopNResultValue topNResult = new TopNResultValue( Arrays.>asList( ImmutableMap.builder() - .put(QueryRunnerTestHelper.marketDimension, "total_market") + .put(marketStr, totalMarketVal) .put("rows", 186L) .put("index", 215679.82879638672D) .put("addRowsIndexConstant", 215866.82879638672D) @@ -2838,7 +2954,7 @@ public void testTopNBySegmentResults() .put("minIndex", 792.3260498046875D) .build(), ImmutableMap.builder() - .put(QueryRunnerTestHelper.marketDimension, "upfront") + .put(marketStr, upfrontVal) .put("rows", 186L) .put("index", 192046.1060180664D) .put("addRowsIndexConstant", 192233.1060180664D) @@ -2848,7 +2964,7 @@ public void testTopNBySegmentResults() .put("minIndex", 545.9906005859375D) .build(), ImmutableMap.builder() - .put(QueryRunnerTestHelper.marketDimension, "spot") + .put(marketStr, spotVal) .put("rows", 837L) .put("index", 95606.57232284546D) .put("addRowsIndexConstant", 96444.57232284546D) @@ -2877,7 +2993,7 @@ public void testTopNBySegmentResults() ); Sequence> results = runWithMerge(query); for (Result result : Sequences.toList(results, new ArrayList>())) { - Assert.assertEquals(result.getValue(), result.getValue()); // TODO: fix this test + Assert.assertEquals(result.getValue(), result.getValue()); } } @@ -2895,7 +3011,7 @@ public void testTopNWithTimeColumn() ) ) .granularity(QueryRunnerTestHelper.allGran) - .dimension(QueryRunnerTestHelper.marketDimension) + .dimension(marketStr) .metric("ntimestamps") .threshold(3) .build(); @@ -2906,7 +3022,7 @@ public void testTopNWithTimeColumn() new TopNResultValue( Arrays.>asList( ImmutableMap.of( - "market", "spot", + marketStr, spotVal, "rows", 18L, "ntimestamps", @@ -2915,7 +3031,7 @@ public void testTopNWithTimeColumn() 23429865600000L ), ImmutableMap.of( - "market", "total_market", + marketStr, totalMarketVal, "rows", 4L, "ntimestamps", @@ -2924,7 +3040,7 @@ public void testTopNWithTimeColumn() 5206636800000L ), ImmutableMap.of( - "market", "upfront", + marketStr, upfrontVal, "rows", 4L, "ntimestamps", @@ -3189,7 +3305,7 @@ public void testAlphaNumericTopNWithNullPreviousStop() TopNQuery query = new TopNQueryBuilder() .dataSource(QueryRunnerTestHelper.dataSource) .granularity(QueryGranularity.ALL) - .dimension(QueryRunnerTestHelper.marketDimension) + .dimension(marketStr) .metric(new AlphaNumericTopNMetricSpec(null)) .threshold(2) .intervals(QueryRunnerTestHelper.secondOnly) @@ -3201,11 +3317,11 @@ public void testAlphaNumericTopNWithNullPreviousStop() new TopNResultValue( Arrays.asList( ImmutableMap.of( - "market", "spot", + marketStr, spotVal, "rows", 9L ), ImmutableMap.of( - "market", "total_market", + marketStr, totalMarketVal, "rows", 2L ) ) @@ -3219,13 +3335,13 @@ public void testAlphaNumericTopNWithNullPreviousStop() public void testTopNWithExtractionFilter() { Map extractionMap = new HashMap<>(); - extractionMap.put("spot", "spot0"); + extractionMap.put(spotVal.toString(), "spot0"); MapLookupExtractor mapLookupExtractor = new MapLookupExtractor(extractionMap, false); LookupExtractionFn lookupExtractionFn = new LookupExtractionFn(mapLookupExtractor, false, null, true, false); TopNQuery query = new TopNQueryBuilder().dataSource(QueryRunnerTestHelper.dataSource) .granularity(QueryRunnerTestHelper.allGran) - .dimension(QueryRunnerTestHelper.marketDimension) + .dimension(marketStr) .metric("rows") .threshold(3) .intervals(QueryRunnerTestHelper.firstToThird) @@ -3233,7 +3349,7 @@ public void testTopNWithExtractionFilter() .postAggregators(Arrays.asList(QueryRunnerTestHelper.addRowsIndexConstant)) .filters( new ExtractionDimFilter( - QueryRunnerTestHelper.marketDimension, + marketStr, "spot0", lookupExtractionFn, null @@ -3247,7 +3363,7 @@ public void testTopNWithExtractionFilter() new TopNResultValue( Arrays.>asList( ImmutableMap.of( - QueryRunnerTestHelper.marketDimension, "spot", + marketStr, spotVal, "rows", 18L, "index", 2231.8768157958984D, "addRowsIndexConstant", 2250.8768157958984D, diff --git a/processing/src/test/java/io/druid/query/topn/TopNUnionQueryTest.java b/processing/src/test/java/io/druid/query/topn/TopNUnionQueryTest.java index a7b27c0f0f3f..ce5a37d0d109 100644 --- a/processing/src/test/java/io/druid/query/topn/TopNUnionQueryTest.java +++ b/processing/src/test/java/io/druid/query/topn/TopNUnionQueryTest.java @@ -23,6 +23,7 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.Iterables; import com.google.common.collect.Lists; +import com.google.common.collect.Maps; import io.druid.collections.StupidPool; import io.druid.query.QueryRunner; import io.druid.query.QueryRunnerTestHelper; @@ -33,6 +34,8 @@ import io.druid.query.aggregation.DoubleMinAggregatorFactory; import io.druid.query.aggregation.PostAggregator; import io.druid.segment.TestHelper; +import io.druid.segment.TestIndex; +import io.druid.segment.column.ValueType; import org.joda.time.DateTime; import org.junit.Test; import org.junit.runner.RunWith; @@ -49,17 +52,37 @@ public class TopNUnionQueryTest { private final QueryRunner runner; + private final ValueType keyType; + private final String marketStr; + private final Object spotVal; + private final Object totalMarketVal; + private final Object upfrontVal; + private final Map typeHints; public TopNUnionQueryTest( - QueryRunner runner + QueryRunner runner, + ValueType keyType ) { this.runner = runner; + this.keyType = keyType; + this.marketStr = (String) getDimMapping(QueryRunnerTestHelper.marketDimension); + this.spotVal = getDimMapping("spot"); + this.totalMarketVal = getDimMapping("total_market"); + this.upfrontVal = getDimMapping("upfront"); + typeHints = Maps.newHashMap(); + typeHints.put(marketStr, keyType); + } + + private Object getDimMapping(String value) { + return TestIndex.getDimMappingForType(keyType, value); } @Parameterized.Parameters public static Iterable constructorFeeder() throws IOException { + ValueType[] types = new ValueType[]{ValueType.STRING, ValueType.LONG, ValueType.FLOAT}; + return QueryRunnerTestHelper.cartesian( Iterables.concat( QueryRunnerTestHelper.makeUnionQueryRunners( @@ -93,7 +116,8 @@ public ByteBuffer get() ), QueryRunnerTestHelper.unionDataSource ) - ) + ), + Lists.newArrayList(types) ); } @@ -103,7 +127,7 @@ public void testTopNUnionQuery() TopNQuery query = new TopNQueryBuilder() .dataSource(QueryRunnerTestHelper.unionDataSource) .granularity(QueryRunnerTestHelper.allGran) - .dimension(QueryRunnerTestHelper.marketDimension) + .dimension(marketStr) .metric(QueryRunnerTestHelper.dependentPostAggMetric) .threshold(4) .intervals(QueryRunnerTestHelper.fullOnInterval) @@ -125,6 +149,7 @@ public void testTopNUnionQuery() QueryRunnerTestHelper.hyperUniqueFinalizingPostAgg ) ) + .context(ImmutableMap.of("typeHints", typeHints)) .build(); List> expectedResults = Arrays.asList( @@ -133,7 +158,7 @@ public void testTopNUnionQuery() new TopNResultValue( Arrays.>asList( ImmutableMap.builder() - .put(QueryRunnerTestHelper.marketDimension, "total_market") + .put(marketStr, totalMarketVal) .put("rows", 744L) .put("index", 862719.3151855469D) .put("addRowsIndexConstant", 863464.3151855469D) @@ -147,7 +172,7 @@ public void testTopNUnionQuery() ) .build(), ImmutableMap.builder() - .put(QueryRunnerTestHelper.marketDimension, "upfront") + .put(marketStr, upfrontVal) .put("rows", 744L) .put("index", 768184.4240722656D) .put("addRowsIndexConstant", 768929.4240722656D) @@ -161,7 +186,7 @@ public void testTopNUnionQuery() ) .build(), ImmutableMap.builder() - .put(QueryRunnerTestHelper.marketDimension, "spot") + .put(marketStr, spotVal) .put("rows", 3348L) .put("index", 382426.28929138184D) .put("addRowsIndexConstant", 385775.28929138184D) diff --git a/processing/src/test/java/io/druid/segment/IndexMergerTest.java b/processing/src/test/java/io/druid/segment/IndexMergerTest.java index d5f46b605de6..51013aecc562 100644 --- a/processing/src/test/java/io/druid/segment/IndexMergerTest.java +++ b/processing/src/test/java/io/druid/segment/IndexMergerTest.java @@ -31,7 +31,11 @@ import com.metamx.common.IAE; import com.metamx.common.ISE; import io.druid.data.input.MapBasedInputRow; +import io.druid.data.input.impl.DimensionSchema; import io.druid.data.input.impl.DimensionsSpec; +import io.druid.data.input.impl.FloatDimensionSchema; +import io.druid.data.input.impl.LongDimensionSchema; +import io.druid.data.input.impl.StringDimensionSchema; import io.druid.granularity.QueryGranularity; import io.druid.query.aggregation.AggregatorFactory; import io.druid.query.aggregation.CountAggregatorFactory; @@ -968,7 +972,7 @@ public void testDisjointDimMerge() throws Exception { IncrementalIndex toPersistA = getSingleDimIndex("dimA", Arrays.asList("1", "2")); IncrementalIndex toPersistB = getSingleDimIndex("dimB", Arrays.asList("1", "2", "3")); - IncrementalIndex toPersistB2 = getIndexWithDims(Arrays.asList("dimA", "dimB")); + IncrementalIndex toPersistB2 = getIndexWithDims(DimensionsSpec.getDefaultSchemas(Arrays.asList("dimA", "dimB"))); addDimValuesToIndex(toPersistB2, "dimB", Arrays.asList("1", "2", "3")); final File tmpDirA = temporaryFolder.newFolder(); @@ -1345,7 +1349,6 @@ public void testMergeWithSupersetOrdering() throws Exception final QueryableIndexIndexableAdapter adapter = new QueryableIndexIndexableAdapter(merged); final List boatList = ImmutableList.copyOf(adapter.getRows()); - final QueryableIndexIndexableAdapter adapter2 = new QueryableIndexIndexableAdapter(merged2); final List boatList2 = ImmutableList.copyOf(adapter2.getRows()); @@ -1555,6 +1558,110 @@ public void testMismatchedMetricsVarying() throws IOException Assert.assertEquals(ImmutableSet.of("A", "B", "C"), ImmutableSet.copyOf(adapter.getAvailableMetrics())); } + @Test + public void testMergeNumericDims() throws Exception + { + IncrementalIndex toPersist1 = getIndexWithNumericDims(); + IncrementalIndex toPersist2 = getIndexWithNumericDims(); + + final File tmpDir = temporaryFolder.newFolder(); + final File tmpDir2 = temporaryFolder.newFolder(); + final File tmpDirMerged = temporaryFolder.newFolder(); + + QueryableIndex index1 = closer.closeLater( + INDEX_IO.loadIndex( + INDEX_MERGER.persist( + toPersist1, + tmpDir, + indexSpec + ) + ) + ); + + QueryableIndex index2 = closer.closeLater( + INDEX_IO.loadIndex( + INDEX_MERGER.persist( + toPersist2, + tmpDir2, + indexSpec + ) + ) + ); + + final QueryableIndex merged = closer.closeLater( + INDEX_IO.loadIndex( + INDEX_MERGER.mergeQueryableIndex( + Arrays.asList(index1, index2), + new AggregatorFactory[]{new CountAggregatorFactory("count")}, + tmpDirMerged, + indexSpec + ) + ) + ); + + final IndexableAdapter adapter = new QueryableIndexIndexableAdapter(merged); + Iterable boats = adapter.getRows(); + List boatList = Lists.newArrayList(boats); + + Assert.assertEquals(ImmutableList.of("dimA", "dimB", "dimC"), ImmutableList.copyOf(adapter.getDimensionNames())); + Assert.assertEquals(4, boatList.size()); + Assert.assertArrayEquals(new Comparable[][]{{0L}, {0.0f}, {2}}, boatList.get(0).getDims()); + Assert.assertArrayEquals(new Object[]{2L}, boatList.get(0).getMetrics()); + Assert.assertArrayEquals(new Comparable[][]{{72L}, {60000.789f}, {3}}, boatList.get(1).getDims()); + Assert.assertArrayEquals(new Object[]{2L}, boatList.get(0).getMetrics()); + Assert.assertArrayEquals(new Comparable[][]{{100L}, {4000.567f}, {1}}, boatList.get(2).getDims()); + Assert.assertArrayEquals(new Object[]{2L}, boatList.get(1).getMetrics()); + Assert.assertArrayEquals(new Comparable[][]{{3001L}, {1.2345f}, {0}}, boatList.get(3).getDims()); + Assert.assertArrayEquals(new Object[]{2L}, boatList.get(2).getMetrics()); + + } + + + private IncrementalIndex getIndexWithNumericDims() throws Exception + { + IncrementalIndex index = getIndexWithDims( + Arrays.asList( + new LongDimensionSchema("dimA"), + new FloatDimensionSchema("dimB"), + new StringDimensionSchema("dimC") + ) + ); + + index.add( + new MapBasedInputRow( + 1, + Arrays.asList("dimA", "dimB", "dimC"), + ImmutableMap.of("dimA", 100L, "dimB", 4000.567, "dimC", "Hello") + ) + ); + + index.add( + new MapBasedInputRow( + 1, + Arrays.asList("dimA", "dimB", "dimC"), + ImmutableMap.of("dimA", 72L, "dimB", 60000.789, "dimC", "World") + ) + ); + + index.add( + new MapBasedInputRow( + 1, + Arrays.asList("dimA", "dimB", "dimC"), + ImmutableMap.of("dimA", 3001L, "dimB", 1.2345, "dimC", "Foobar") + ) + ); + + index.add( + new MapBasedInputRow( + 1, + Arrays.asList("dimA", "dimB", "dimC"), + ImmutableMap.of("dimC", "Nully Row") + ) + ); + + return index; + } + private IncrementalIndex getIndexD3() throws Exception { IncrementalIndex toPersist1 = new OnheapIncrementalIndex( @@ -1617,12 +1724,12 @@ private void addDimValuesToIndex(IncrementalIndex index, String dimName, List dims) + private IncrementalIndex getIndexWithDims(List dims) { IncrementalIndexSchema schema = new IncrementalIndexSchema( 0L, QueryGranularity.NONE, - new DimensionsSpec(DimensionsSpec.getDefaultSchemas(dims), null, null), + new DimensionsSpec(dims, null, null), new AggregatorFactory[]{new CountAggregatorFactory("count")} ); diff --git a/processing/src/test/java/io/druid/segment/IndexSpecTest.java b/processing/src/test/java/io/druid/segment/IndexSpecTest.java index 902039d062f7..2210fa027950 100644 --- a/processing/src/test/java/io/druid/segment/IndexSpecTest.java +++ b/processing/src/test/java/io/druid/segment/IndexSpecTest.java @@ -51,7 +51,7 @@ public void testSerdeUncompressed() throws Exception final IndexSpec spec = objectMapper.readValue(json, IndexSpec.class); Assert.assertEquals(IndexSpec.UNCOMPRESSED, spec.getDimensionCompression()); - Assert.assertEquals(null, spec.getDimensionCompressionStrategy()); + Assert.assertEquals(CompressedObjectStrategy.CompressionStrategy.UNCOMPRESSED, spec.getDimensionCompressionStrategy()); Assert.assertEquals(spec, objectMapper.readValue(objectMapper.writeValueAsBytes(spec), IndexSpec.class)); } diff --git a/processing/src/test/java/io/druid/segment/NullDimensionSelectorTest.java b/processing/src/test/java/io/druid/segment/NullDimensionSelectorTest.java index aab2dff99c64..fdc36c9f95d7 100644 --- a/processing/src/test/java/io/druid/segment/NullDimensionSelectorTest.java +++ b/processing/src/test/java/io/druid/segment/NullDimensionSelectorTest.java @@ -19,6 +19,7 @@ package io.druid.segment; +import io.druid.segment.column.ValueType; import io.druid.segment.data.IndexedInts; import org.junit.Assert; import org.junit.Test; @@ -27,7 +28,7 @@ public class NullDimensionSelectorTest { - private final NullDimensionSelector selector = new NullDimensionSelector(); + private final NullDimensionSelector selector = new NullDimensionSelector(ValueType.STRING); @Test public void testGetRow() throws Exception { diff --git a/processing/src/test/java/io/druid/segment/TestIndex.java b/processing/src/test/java/io/druid/segment/TestIndex.java index 14eaf8c2d7b8..0f10e268b361 100644 --- a/processing/src/test/java/io/druid/segment/TestIndex.java +++ b/processing/src/test/java/io/druid/segment/TestIndex.java @@ -21,13 +21,18 @@ import com.google.common.base.Charsets; import com.google.common.base.Throwables; +import com.google.common.collect.ImmutableMap; import com.google.common.hash.Hashing; import com.google.common.io.CharSource; import com.google.common.io.LineProcessor; import com.google.common.io.Resources; import com.metamx.common.logger.Logger; import io.druid.data.input.impl.DelimitedParseSpec; +import io.druid.data.input.impl.DimensionSchema; import io.druid.data.input.impl.DimensionsSpec; +import io.druid.data.input.impl.FloatDimensionSchema; +import io.druid.data.input.impl.LongDimensionSchema; +import io.druid.data.input.impl.StringDimensionSchema; import io.druid.data.input.impl.StringInputRowParser; import io.druid.data.input.impl.TimestampSpec; import io.druid.granularity.QueryGranularity; @@ -35,6 +40,7 @@ import io.druid.query.aggregation.DoubleSumAggregatorFactory; import io.druid.query.aggregation.hyperloglog.HyperUniquesAggregatorFactory; import io.druid.query.aggregation.hyperloglog.HyperUniquesSerde; +import io.druid.segment.column.ValueType; import io.druid.segment.incremental.IncrementalIndex; import io.druid.segment.incremental.IncrementalIndexSchema; import io.druid.segment.incremental.OnheapIncrementalIndex; @@ -46,6 +52,7 @@ import java.io.IOException; import java.net.URL; import java.util.Arrays; +import java.util.Map; import java.util.concurrent.atomic.AtomicLong; /** @@ -55,7 +62,11 @@ public class TestIndex public static final String[] COLUMNS = new String[]{ "ts", "market", + "market_long", + "market_float", "quality", + "quality_long", + "quality_float", "placement", "placementish", "index", @@ -63,14 +74,102 @@ public class TestIndex "null_column", "quality_uniques" }; + public static final String[] DIMENSIONS = new String[]{ "market", + "market_long", + "market_float", "quality", + "quality_long", + "quality_float", "placement", "placementish", "partial_null_column", - "null_column", - }; + "null_column" + }; + + public static final DimensionSchema[] DIMENSION_SCHEMAS = new DimensionSchema[]{ + new StringDimensionSchema("market"), + new LongDimensionSchema("market_long"), + new FloatDimensionSchema("market_float"), + new StringDimensionSchema("quality"), + new LongDimensionSchema("quality_long"), + new FloatDimensionSchema("quality_float"), + new StringDimensionSchema("placement"), + new StringDimensionSchema("placementish"), + new StringDimensionSchema("partial_null_column"), + new StringDimensionSchema("null_column") + }; + + private static final Map STRING_DIM_NAME_AND_VALUES_MAP = ImmutableMap.builder() + .put("quality", "quality") + .put("market", "market") + .put("automotive", "automotive") + .put("business", "business") + .put("entertainment", "entertainment") + .put("health", "health") + .put("mezzanine", "mezzanine") + .put("news", "news") + .put("premium", "premium") + .put("technology", "technology") + .put("travel", "travel") + .put("spot", "spot") + .put("total_market", "total_market") + .put("upfront", "upfront") + .put("billyblank", "billyblank") + .build(); + + private static final Map LONG_DIM_NAME_AND_VALUES_MAP = ImmutableMap.builder() + .put("quality", "quality_long") + .put("market", "market_long") + .put("automotive", 1111L) + .put("business", 2222L) + .put("entertainment", 3333L) + .put("health", 4444L) + .put("mezzanine", 5555L) + .put("news", 6666L) + .put("premium", 7777L) + .put("technology", 8888L) + .put("travel", 8899L) + .put("spot", 1111L) + .put("total_market", 2222L) + .put("upfront", 3333L) + .put("billyblank", 12345678L) + .build(); + + private static final Map FLOAT_DIM_NAME_AND_VALUES_MAP = ImmutableMap.builder() + .put("quality", "quality_float") + .put("market", "market_float") + .put("automotive", 111.111f) + .put("business", 222.222f) + .put("entertainment", 333.333f) + .put("health", 444.444f) + .put("mezzanine", 555.555f) + .put("news", 666.666f) + .put("premium", 777.777f) + .put("technology", 888.888f) + .put("travel", 889.999f) + .put("spot", 111.111f) + .put("total_market", 222.222f) + .put("upfront", 333.333f) + .put("billyblank", 1234.5678f) + .build(); + + public static Object getDimMappingForType(ValueType type, String lookupStr) + { + switch (type) { + case STRING: + return STRING_DIM_NAME_AND_VALUES_MAP.get(lookupStr); + case LONG: + return LONG_DIM_NAME_AND_VALUES_MAP.get(lookupStr); + case FLOAT: + return FLOAT_DIM_NAME_AND_VALUES_MAP.get(lookupStr); + default: + throw new UnsupportedOperationException("Invalid type: " + type); + } + } + + public static final String[] METRICS = new String[]{"index"}; private static final Logger log = new Logger(TestIndex.class); private static final Interval DATA_INTERVAL = new Interval("2011-01-12T00:00:00.000Z/2011-05-01T00:00:00.000Z"); @@ -101,7 +200,7 @@ public static IncrementalIndex getIncrementalTestIndex() } } - return realtimeIndex = makeRealtimeIndex("druid.sample.tsv"); + return realtimeIndex = makeRealtimeIndex("druid.sample.numericdims.tsv"); } public static QueryableIndex getMMappedTestIndex() @@ -126,8 +225,8 @@ public static QueryableIndex mergedRealtimeIndex() } try { - IncrementalIndex top = makeRealtimeIndex("druid.sample.tsv.top"); - IncrementalIndex bottom = makeRealtimeIndex("druid.sample.tsv.bottom"); + IncrementalIndex top = makeRealtimeIndex("druid.sample.numericdims.tsv.top"); + IncrementalIndex bottom = makeRealtimeIndex("druid.sample.numericdims.tsv.bottom"); File tmpFile = File.createTempFile("yay", "who"); tmpFile.delete(); @@ -176,7 +275,9 @@ public static IncrementalIndex makeRealtimeIndex(final String resourceFilename) public static IncrementalIndex makeRealtimeIndex(final CharSource source) { + final DimensionsSpec dimSpec = new DimensionsSpec(Arrays.asList(DIMENSION_SCHEMAS), null, null); final IncrementalIndexSchema schema = new IncrementalIndexSchema.Builder() + .withDimensionsSpec(dimSpec) .withMinTimestamp(new DateTime("2011-01-12T00:00:00.000Z").getMillis()) .withQueryGranularity(QueryGranularity.NONE) .withMetrics(METRIC_AGGS) diff --git a/processing/src/test/java/io/druid/segment/filter/ExtractionDimFilterTest.java b/processing/src/test/java/io/druid/segment/filter/ExtractionDimFilterTest.java index e1c6354dab23..fa746ac1213c 100644 --- a/processing/src/test/java/io/druid/segment/filter/ExtractionDimFilterTest.java +++ b/processing/src/test/java/io/druid/segment/filter/ExtractionDimFilterTest.java @@ -19,6 +19,7 @@ package io.druid.segment.filter; +import com.google.common.base.Predicate; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.metamx.collections.bitmap.BitmapFactory; @@ -27,11 +28,13 @@ import com.metamx.collections.bitmap.MutableBitmap; import com.metamx.collections.bitmap.RoaringBitmapFactory; import com.metamx.collections.spatial.ImmutableRTree; +import com.metamx.collections.spatial.search.Bound; import io.druid.query.extraction.DimExtractionFn; import io.druid.query.extraction.ExtractionFn; import io.druid.query.filter.BitmapIndexSelector; import io.druid.query.filter.DimFilters; import io.druid.query.filter.ExtractionDimFilter; +import io.druid.segment.column.ValueType; import io.druid.segment.data.ArrayIndexed; import io.druid.segment.data.Indexed; import org.junit.Assert; @@ -105,11 +108,28 @@ public ImmutableBitmap getBitmapIndex(String dimension, String value) } @Override - public ImmutableRTree getSpatialIndex(String dimension) + public ImmutableBitmap getBitmapIndex(String dimension, Bound bound) { return null; } + + @Override + public boolean hasBitmapIndexes(String dimension) { + return true; + } + + @Override + public ImmutableBitmap getBitmapIndexFromColumnScan(String dimension, Predicate predicate) { + return null; + } + + @Override + public ValueType getDimensionType(String dimension) + { + return ValueType.STRING; + } }; + private static final ExtractionFn DIM_EXTRACTION_FN = new DimExtractionFn() { @Override diff --git a/processing/src/test/java/io/druid/segment/filter/RowboatTest.java b/processing/src/test/java/io/druid/segment/filter/RowboatTest.java index 402b5f53b681..a77ad52934e8 100644 --- a/processing/src/test/java/io/druid/segment/filter/RowboatTest.java +++ b/processing/src/test/java/io/druid/segment/filter/RowboatTest.java @@ -19,7 +19,9 @@ package io.druid.segment.filter; +import io.druid.segment.DimensionHandler; import io.druid.segment.Rowboat; +import io.druid.segment.StringDimensionHandler; import org.junit.Assert; import org.junit.Test; @@ -28,63 +30,75 @@ */ public class RowboatTest { + private static DimensionHandler[] getDefaultHandlers(int size) { + DimensionHandler[] handlers = new DimensionHandler[size]; + for (int i = 0; i < size; i++) { + handlers[i] = new StringDimensionHandler(String.valueOf(i)); + } + return handlers; + } + @Test public void testRowboatCompare() { - Rowboat rb1 = new Rowboat(12345L, new int[][]{new int[]{1}, new int[]{2}}, new Object[]{new Integer(7)}, 5); - Rowboat rb2 = new Rowboat(12345L, new int[][]{new int[]{1}, new int[]{2}}, new Object[]{new Integer(7)}, 5); + DimensionHandler[] handlers = getDefaultHandlers(3); + Rowboat rb1 = new Rowboat(12345L, new Comparable[][]{new Comparable[]{1}, new Comparable[]{2}}, new Object[]{new Integer(7)}, 5, handlers); + Rowboat rb2 = new Rowboat(12345L, new Comparable[][]{new Comparable[]{1}, new Comparable[]{2}}, new Object[]{new Integer(7)}, 5, handlers); Assert.assertEquals(0, rb1.compareTo(rb2)); - Rowboat rb3 = new Rowboat(12345L, new int[][]{new int[]{3}, new int[]{2}}, new Object[]{new Integer(7)}, 5); + Rowboat rb3 = new Rowboat(12345L, new Comparable[][]{new Comparable[]{3}, new Comparable[]{2}}, new Object[]{new Integer(7)}, 5, handlers); Assert.assertNotEquals(0, rb1.compareTo(rb3)); } - @Test public void testBiggerCompare() { + DimensionHandler[] handlers = getDefaultHandlers(14); + Rowboat rb1 = new Rowboat( 0, - new int[][]{ - new int[]{0}, - new int[]{138}, - new int[]{44}, - new int[]{374}, - new int[]{0}, - new int[]{0}, - new int[]{552}, - new int[]{338}, - new int[]{910}, - new int[]{25570}, - new int[]{9}, - new int[]{0}, - new int[]{0}, - new int[]{0} + new Comparable[][]{ + new Comparable[]{0}, + new Comparable[]{138}, + new Comparable[]{44}, + new Comparable[]{374}, + new Comparable[]{0}, + new Comparable[]{0}, + new Comparable[]{552}, + new Comparable[]{338}, + new Comparable[]{910}, + new Comparable[]{25570}, + new Comparable[]{9}, + new Comparable[]{0}, + new Comparable[]{0}, + new Comparable[]{0} }, new Object[]{1.0, 47.0, "someMetric"}, - 0 + 0, + handlers ); Rowboat rb2 = new Rowboat( 0, - new int[][]{ - new int[]{0}, - new int[]{138}, - new int[]{44}, - new int[]{374}, - new int[]{0}, - new int[]{0}, - new int[]{553}, - new int[]{338}, - new int[]{910}, - new int[]{25580}, - new int[]{9}, - new int[]{0}, - new int[]{0}, - new int[]{0} + new Comparable[][]{ + new Comparable[]{0}, + new Comparable[]{138}, + new Comparable[]{44}, + new Comparable[]{374}, + new Comparable[]{0}, + new Comparable[]{0}, + new Comparable[]{553}, + new Comparable[]{338}, + new Comparable[]{910}, + new Comparable[]{25580}, + new Comparable[]{9}, + new Comparable[]{0}, + new Comparable[]{0}, + new Comparable[]{0} }, new Object[]{1.0, 47.0, "someMetric"}, - 0 + 0, + handlers ); Assert.assertNotEquals(0, rb1.compareTo(rb2)); @@ -93,18 +107,22 @@ public void testBiggerCompare() @Test public void testToString() { + DimensionHandler[] handlers = getDefaultHandlers(2); + Assert.assertEquals( "Rowboat{timestamp=1970-01-01T00:00:00.000Z, dims=[[1], [2]], metrics=[someMetric], comprisedRows={}}", - new Rowboat(0, new int[][]{new int[]{1}, new int[]{2}}, new Object[]{"someMetric"}, 5).toString() + new Rowboat(0, new Comparable[][]{new Comparable[]{1}, new Comparable[]{2}}, new Object[]{"someMetric"}, 5, handlers).toString() ); } @Test public void testLotsONullString() { + DimensionHandler[] handlers = getDefaultHandlers(0); + Assert.assertEquals( "Rowboat{timestamp=1970-01-01T00:00:00.000Z, dims=null, metrics=null, comprisedRows={}}", - new Rowboat(0, null, null, 5).toString() + new Rowboat(0, null, null, 5, handlers).toString() ); } } diff --git a/processing/src/test/java/io/druid/segment/filter/SpatialFilterTest.java b/processing/src/test/java/io/druid/segment/filter/SpatialFilterTest.java index 2f20f61caa7a..7650bd516787 100644 --- a/processing/src/test/java/io/druid/segment/filter/SpatialFilterTest.java +++ b/processing/src/test/java/io/druid/segment/filter/SpatialFilterTest.java @@ -500,6 +500,7 @@ private static QueryableIndex makeMergedQueryableIndex(IndexSpec indexSpec) ); return mergedRealtime; + } catch (IOException e) { throw Throwables.propagate(e); diff --git a/processing/src/test/java/io/druid/segment/incremental/OnheapIncrementalIndexBenchmark.java b/processing/src/test/java/io/druid/segment/incremental/OnheapIncrementalIndexBenchmark.java index 89a1195e3a8f..4a03265b5f11 100644 --- a/processing/src/test/java/io/druid/segment/incremental/OnheapIncrementalIndexBenchmark.java +++ b/processing/src/test/java/io/druid/segment/incremental/OnheapIncrementalIndexBenchmark.java @@ -157,7 +157,7 @@ protected Integer addToFacts( for (int i = 0; i < metrics.length; i++) { final AggregatorFactory agg = metrics[i]; aggs[i] = agg.factorize( - makeColumnSelectorFactory(agg, rowSupplier, deserializeComplexMetrics) + makeColumnSelectorFactory(agg, rowSupplier, deserializeComplexMetrics, null) ); } Integer rowIndex; diff --git a/processing/src/test/resources/druid.sample.numericdims.tsv b/processing/src/test/resources/druid.sample.numericdims.tsv new file mode 100644 index 000000000000..bce5844714dc --- /dev/null +++ b/processing/src/test/resources/druid.sample.numericdims.tsv @@ -0,0 +1,1209 @@ +2011-01-12T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 100.000000 +2011-01-12T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 100.000000 +2011-01-12T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 100.000000 +2011-01-12T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 100.000000 +2011-01-12T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 100.000000 +2011-01-12T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 100.000000 +2011-01-12T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 100.000000 +2011-01-12T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 100.000000 +2011-01-12T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 100.000000 +2011-01-12T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1000.000000 +2011-01-12T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 1000.000000 +2011-01-12T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 800.000000 value +2011-01-12T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 800.000000 value +2011-01-13T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 94.874713 +2011-01-13T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 103.629399 +2011-01-13T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 110.087299 +2011-01-13T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 114.947403 +2011-01-13T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 104.465767 +2011-01-13T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 102.851683 +2011-01-13T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 108.863011 +2011-01-13T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 111.356672 +2011-01-13T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 106.236928 +2011-01-13T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1040.945505 +2011-01-13T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 1689.012875 +2011-01-13T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 826.060182 value +2011-01-13T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 1564.617729 value +2011-01-14T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 86.450372 +2011-01-14T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 102.670409 +2011-01-14T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 109.573474 +2011-01-14T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 94.000432 +2011-01-14T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 97.903068 +2011-01-14T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 101.380760 +2011-01-14T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 104.611784 +2011-01-14T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 114.974216 +2011-01-14T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 112.259958 +2011-01-14T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1049.141912 +2011-01-14T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 1073.476545 +2011-01-14T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 1006.402111 value +2011-01-14T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 869.643722 value +2011-01-15T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 82.840417 +2011-01-15T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 99.781645 +2011-01-15T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 100.551072 +2011-01-15T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 87.954346 +2011-01-15T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 100.582654 +2011-01-15T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 99.383407 +2011-01-15T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 102.366798 +2011-01-15T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 111.680229 +2011-01-15T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 120.481420 +2011-01-15T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1007.365510 +2011-01-15T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 1545.708865 +2011-01-15T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 809.041763 value +2011-01-15T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 1458.402661 value +2011-01-16T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 71.315931 +2011-01-16T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 100.591602 +2011-01-16T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 99.007588 +2011-01-16T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 93.085943 +2011-01-16T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 99.863171 +2011-01-16T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 100.192013 +2011-01-16T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 103.348007 +2011-01-16T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 87.280816 +2011-01-16T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 116.779610 +2011-01-16T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1016.965229 +2011-01-16T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 1077.612663 +2011-01-16T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 852.437477 value +2011-01-16T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 879.988099 value +2011-01-17T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 105.442374 +2011-01-17T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 105.914315 +2011-01-17T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 99.189052 +2011-01-17T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 110.157325 +2011-01-17T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 103.692852 +2011-01-17T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 103.615039 +2011-01-17T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 106.696362 +2011-01-17T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 89.901887 +2011-01-17T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 134.415281 +2011-01-17T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1075.089574 +2011-01-17T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 953.995422 +2011-01-17T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 950.146770 value +2011-01-17T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 712.774595 value +2011-01-18T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 87.195139 +2011-01-18T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 107.244699 +2011-01-18T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 94.452739 +2011-01-18T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 103.018934 +2011-01-18T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 101.087367 +2011-01-18T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 104.724154 +2011-01-18T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 108.979936 +2011-01-18T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 88.764512 +2011-01-18T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 113.680094 +2011-01-18T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1022.783330 +2011-01-18T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 937.061939 +2011-01-18T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 846.267516 value +2011-01-18T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 682.885525 value +2011-01-19T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 85.681683 +2011-01-19T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 106.700550 +2011-01-19T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 92.314034 +2011-01-19T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 97.620315 +2011-01-19T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 105.087466 +2011-01-19T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 106.127830 +2011-01-19T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 104.568464 +2011-01-19T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 77.316731 +2011-01-19T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 109.772202 +2011-01-19T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1156.744712 +2011-01-19T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 849.877513 +2011-01-19T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 1109.874950 value +2011-01-19T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 594.381703 value +2011-01-20T01:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 93.396274 +2011-01-20T01:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 106.367723 +2011-01-20T01:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 90.439020 +2011-01-20T01:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 96.112901 +2011-01-20T01:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 105.669498 +2011-01-20T01:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 105.225158 +2011-01-20T01:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 101.305541 +2011-01-20T01:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 77.759854 +2011-01-20T01:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 140.179069 +2011-01-20T01:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1066.208012 +2011-01-20T01:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 904.340636 +2011-01-20T01:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 870.115926 value +2011-01-20T01:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 677.510973 value +2011-01-22T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 95.235266 +2011-01-22T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 105.282866 +2011-01-22T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 93.681096 +2011-01-22T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 103.527592 +2011-01-22T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 104.184494 +2011-01-22T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 103.399677 +2011-01-22T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 99.284525 +2011-01-22T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 107.627793 +2011-01-22T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 137.109783 +2011-01-22T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1240.525484 +2011-01-22T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 1343.232494 +2011-01-22T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 1410.278128 value +2011-01-22T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 1219.432170 value +2011-01-23T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 100.432710 +2011-01-23T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 107.348157 +2011-01-23T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 92.789692 +2011-01-23T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 96.826443 +2011-01-23T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 103.730730 +2011-01-23T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 106.418686 +2011-01-23T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 101.573522 +2011-01-23T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 110.467875 +2011-01-23T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 128.699746 +2011-01-23T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1088.943083 +2011-01-23T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 1349.254415 +2011-01-23T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 979.306038 value +2011-01-23T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 1224.501568 value +2011-01-24T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 96.671647 +2011-01-24T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 104.485760 +2011-01-24T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 88.748460 +2011-01-24T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 189.385952 +2011-01-24T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 97.906256 +2011-01-24T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 104.167373 +2011-01-24T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 101.581339 +2011-01-24T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 105.345921 +2011-01-24T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 131.695956 +2011-01-24T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1102.866656 +2011-01-24T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 939.244103 +2011-01-24T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 1215.589859 value +2011-01-24T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 716.609179 value +2011-01-25T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 90.111413 +2011-01-25T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 101.624789 +2011-01-25T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 85.974579 +2011-01-25T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 180.575246 +2011-01-25T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 96.594588 +2011-01-25T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 102.907866 +2011-01-25T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 102.512878 +2011-01-25T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 102.044542 +2011-01-25T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 122.077247 +2011-01-25T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1109.875413 +2011-01-25T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 997.994544 +2011-01-25T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 1301.023342 value +2011-01-25T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 786.363298 value +2011-01-26T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 84.906466 +2011-01-26T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 105.873942 +2011-01-26T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 84.710523 +2011-01-26T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 159.988606 +2011-01-26T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 96.046584 +2011-01-26T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 105.266058 +2011-01-26T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 101.088903 +2011-01-26T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 105.617702 +2011-01-26T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 122.160681 +2011-01-26T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1037.449471 +2011-01-26T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 1686.419659 +2011-01-26T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 989.931541 value +2011-01-26T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 1609.096706 value +2011-01-27T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 134.127106 +2011-01-27T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 104.882908 +2011-01-27T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 90.806201 +2011-01-27T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 170.735853 +2011-01-27T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 100.643435 +2011-01-27T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 104.609483 +2011-01-27T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 102.612747 +2011-01-27T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 116.979005 +2011-01-27T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 149.125271 +2011-01-27T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1074.006938 +2011-01-27T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 1486.201299 +2011-01-27T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 1023.295213 value +2011-01-27T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 1367.638074 value +2011-01-28T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 123.006128 +2011-01-28T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 111.641077 +2011-01-28T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 99.681629 +2011-01-28T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 107.788998 +2011-01-28T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 106.075672 +2011-01-28T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 108.106449 +2011-01-28T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 103.822842 +2011-01-28T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 93.869236 +2011-01-28T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 158.739359 +2011-01-28T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1300.302260 +2011-01-28T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 1021.334487 +2011-01-28T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 1627.598064 value +2011-01-28T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 810.889422 value +2011-01-29T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 127.450345 +2011-01-29T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 100.992147 +2011-01-29T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 103.345166 +2011-01-29T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 114.905745 +2011-01-29T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 106.663538 +2011-01-29T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 101.998823 +2011-01-29T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 104.311418 +2011-01-29T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 109.549035 +2011-01-29T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 138.865014 +2011-01-29T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1314.619452 +2011-01-29T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 792.326066 +2011-01-29T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 1685.500085 value +2011-01-29T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 545.990623 value +2011-01-30T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 124.943293 +2011-01-30T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 106.064111 +2011-01-30T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 108.415967 +2011-01-30T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 137.198397 +2011-01-30T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 97.023907 +2011-01-30T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 106.009926 +2011-01-30T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 104.298490 +2011-01-30T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 110.528451 +2011-01-30T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 137.932693 +2011-01-30T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1233.448863 +2011-01-30T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 805.930143 +2011-01-30T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 1870.061029 value +2011-01-30T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 555.476028 value +2011-01-31T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 133.740047 +2011-01-31T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 103.492964 +2011-01-31T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 104.548387 +2011-01-31T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 124.171944 +2011-01-31T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 97.017604 +2011-01-31T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 103.832040 +2011-01-31T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 103.021032 +2011-01-31T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 85.125795 +2011-01-31T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 155.744951 +2011-01-31T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1184.920651 +2011-01-31T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 1127.231000 +2011-01-31T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 1643.340851 value +2011-01-31T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 943.497198 value +2011-02-01T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 132.123776 +2011-02-01T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 103.890175 +2011-02-01T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 103.652865 +2011-02-01T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 113.896016 +2011-02-01T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 98.909356 +2011-02-01T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 104.383662 +2011-02-01T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 102.480377 +2011-02-01T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 83.931272 +2011-02-01T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 134.014606 +2011-02-01T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1203.465595 +2011-02-01T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 1100.904846 +2011-02-01T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 1667.497773 value +2011-02-01T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 913.561076 value +2011-02-02T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 113.492245 +2011-02-02T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 104.963233 +2011-02-02T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 112.042996 +2011-02-02T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 102.281859 +2011-02-02T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 97.711139 +2011-02-02T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 105.578807 +2011-02-02T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 102.152053 +2011-02-02T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 96.706279 +2011-02-02T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 123.170962 +2011-02-02T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1097.211164 +2011-02-02T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 1410.792943 +2011-02-02T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 1218.561908 value +2011-02-02T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 1273.707453 value +2011-02-03T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 85.770241 +2011-02-03T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 98.877952 +2011-02-03T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 106.425780 +2011-02-03T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 85.069784 +2011-02-03T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 100.559287 +2011-02-03T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 100.976362 +2011-02-03T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 102.349315 +2011-02-03T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 92.326431 +2011-02-03T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 134.140377 +2011-02-03T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1033.401241 +2011-02-03T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 1283.166055 +2011-02-03T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 888.705280 value +2011-02-03T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 1113.114125 value +2011-02-04T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 89.182906 +2011-02-04T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 106.888769 +2011-02-04T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 112.471918 +2011-02-04T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 99.837572 +2011-02-04T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 100.918373 +2011-02-04T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 106.050728 +2011-02-04T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 103.103922 +2011-02-04T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 93.973465 +2011-02-04T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 113.716758 +2011-02-04T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1025.633340 +2011-02-04T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 1331.860983 +2011-02-04T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 864.568891 value +2011-02-04T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 1308.582051 value +2011-02-05T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 93.001571 +2011-02-05T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 111.394244 +2011-02-05T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 117.030289 +2011-02-05T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 94.312960 +2011-02-05T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 105.660538 +2011-02-05T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 107.929804 +2011-02-05T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 100.646747 +2011-02-05T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 90.732978 +2011-02-05T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 114.723682 +2011-02-05T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1039.500513 +2011-02-05T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 1332.468373 +2011-02-05T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 785.078869 value +2011-02-05T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 1363.614929 value +2011-02-06T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 134.462521 +2011-02-06T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 110.897359 +2011-02-06T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 119.608310 +2011-02-06T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 93.585758 +2011-02-06T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 101.847544 +2011-02-06T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 110.053071 +2011-02-06T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 102.770913 +2011-02-06T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 93.620739 +2011-02-06T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 121.270562 +2011-02-06T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1011.205470 +2011-02-06T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 1029.995236 +2011-02-06T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 787.125330 value +2011-02-06T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 826.039207 value +2011-02-07T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 130.194219 +2011-02-07T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 98.815847 +2011-02-07T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 112.924874 +2011-02-07T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 97.480779 +2011-02-07T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 97.434318 +2011-02-07T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 100.706057 +2011-02-07T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 102.705243 +2011-02-07T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 83.902353 +2011-02-07T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 115.246714 +2011-02-07T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1047.212887 +2011-02-07T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 1057.079944 +2011-02-07T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 1107.243787 value +2011-02-07T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 872.625669 value +2011-02-08T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 126.243536 +2011-02-08T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 93.190129 +2011-02-08T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 106.969799 +2011-02-08T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 97.432302 +2011-02-08T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 96.790543 +2011-02-08T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 97.085047 +2011-02-08T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 103.308255 +2011-02-08T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 75.735586 +2011-02-08T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 111.040150 +2011-02-08T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1064.972638 +2011-02-08T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 1082.727640 +2011-02-08T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 1188.369265 value +2011-02-08T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 911.956790 value +2011-02-09T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 129.221792 +2011-02-09T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 115.548444 +2011-02-09T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 111.729360 +2011-02-09T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 97.071703 +2011-02-09T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 96.478571 +2011-02-09T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 113.554588 +2011-02-09T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 105.498315 +2011-02-09T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 83.742151 +2011-02-09T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 112.646238 +2011-02-09T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 971.050764 +2011-02-09T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 1320.638308 +2011-02-09T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 794.098825 value +2011-02-09T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 1299.093262 value +2011-02-10T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 115.461691 +2011-02-10T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 118.062165 +2011-02-10T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 117.629065 +2011-02-10T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 97.235999 +2011-02-10T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 95.698374 +2011-02-10T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 115.824976 +2011-02-10T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 105.708103 +2011-02-10T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 91.750911 +2011-02-10T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 105.557241 +2011-02-10T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1070.165582 +2011-02-10T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 1089.647884 +2011-02-10T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 1212.928303 value +2011-02-10T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 901.327272 value +2011-02-11T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 129.187009 +2011-02-11T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 114.637486 +2011-02-11T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 114.960877 +2011-02-11T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 96.617339 +2011-02-11T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 100.111873 +2011-02-11T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 112.571724 +2011-02-11T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 105.672256 +2011-02-11T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 87.904114 +2011-02-11T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 102.864842 +2011-02-11T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 980.386611 +2011-02-11T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 1179.695901 +2011-02-11T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 723.514254 value +2011-02-11T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 1061.973330 value +2011-02-12T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 130.104979 +2011-02-12T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 115.758445 +2011-02-12T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 115.225386 +2011-02-12T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 96.457082 +2011-02-12T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 101.515571 +2011-02-12T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 114.877503 +2011-02-12T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 108.637522 +2011-02-12T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 88.142774 +2011-02-12T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 102.850696 +2011-02-12T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 959.236186 +2011-02-12T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 1092.416967 +2011-02-12T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 602.979544 value +2011-02-12T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 879.406101 value +2011-02-13T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 119.490316 +2011-02-13T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 118.841176 +2011-02-13T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 119.907266 +2011-02-13T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 98.607490 +2011-02-13T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 100.905238 +2011-02-13T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 117.965974 +2011-02-13T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 112.514409 +2011-02-13T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 87.820236 +2011-02-13T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 106.033416 +2011-02-13T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 987.067381 +2011-02-13T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 1103.458199 +2011-02-13T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 724.262526 value +2011-02-13T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 862.931321 value +2011-02-14T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 119.323168 +2011-02-14T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 115.628202 +2011-02-14T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 123.098262 +2011-02-14T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 103.008650 +2011-02-14T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 101.645725 +2011-02-14T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 117.110451 +2011-02-14T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 121.060464 +2011-02-14T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 73.717033 +2011-02-14T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 107.663239 +2011-02-14T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1091.223197 +2011-02-14T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 1199.607472 +2011-02-14T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 1133.135123 value +2011-02-14T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 948.657939 value +2011-02-15T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 123.485071 +2011-02-15T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 115.527003 +2011-02-15T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 121.563912 +2011-02-15T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 103.519393 +2011-02-15T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 105.269599 +2011-02-15T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 117.138956 +2011-02-15T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 121.411398 +2011-02-15T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 79.700998 +2011-02-15T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 108.428302 +2011-02-15T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1044.384300 +2011-02-15T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 1183.240825 +2011-02-15T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 807.601674 value +2011-02-15T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 914.525048 value +2011-02-16T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 133.726576 +2011-02-16T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 116.432276 +2011-02-16T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 130.717934 +2011-02-16T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 105.762627 +2011-02-16T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 108.338531 +2011-02-16T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 117.334381 +2011-02-16T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 123.270869 +2011-02-16T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 98.918664 +2011-02-16T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 113.643571 +2011-02-16T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1289.097304 +2011-02-16T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 1360.032423 +2011-02-16T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 1553.348548 value +2011-02-16T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 1208.456692 value +2011-02-17T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 147.942017 +2011-02-17T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 125.032692 +2011-02-17T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 126.982673 +2011-02-17T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 101.092779 +2011-02-17T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 103.940963 +2011-02-17T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 121.872569 +2011-02-17T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 120.050545 +2011-02-17T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 91.969195 +2011-02-17T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 112.150745 +2011-02-17T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 993.591221 +2011-02-17T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 1021.071173 +2011-02-17T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 679.619354 value +2011-02-17T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 645.177645 value +2011-02-18T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 151.053296 +2011-02-18T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 127.611947 +2011-02-18T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 128.063524 +2011-02-18T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 100.849247 +2011-02-18T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 101.960196 +2011-02-18T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 124.513018 +2011-02-18T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 122.546253 +2011-02-18T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 92.174432 +2011-02-18T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 120.151389 +2011-02-18T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1105.383465 +2011-02-18T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 1601.829436 +2011-02-18T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 1120.088751 value +2011-02-18T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 1649.533329 value +2011-02-19T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 163.351690 +2011-02-19T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 123.447481 +2011-02-19T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 133.726878 +2011-02-19T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 108.822138 +2011-02-19T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 102.919452 +2011-02-19T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 119.371511 +2011-02-19T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 125.052129 +2011-02-19T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 91.547944 +2011-02-19T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 121.733400 +2011-02-19T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1200.527201 +2011-02-19T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 1600.723226 +2011-02-19T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 1433.398801 value +2011-02-19T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 1598.179271 value +2011-02-20T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 157.483005 +2011-02-20T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 129.409606 +2011-02-20T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 176.323916 +2011-02-20T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 109.790712 +2011-02-20T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 127.819268 +2011-02-20T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 122.082375 +2011-02-20T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 132.719065 +2011-02-20T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 105.985049 +2011-02-20T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 124.637709 +2011-02-20T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1317.458323 +2011-02-20T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 1304.326111 +2011-02-20T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 1192.563067 value +2011-02-20T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 1022.854576 value +2011-02-21T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 155.632898 +2011-02-21T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 132.231346 +2011-02-21T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 193.787574 +2011-02-21T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 111.386745 +2011-02-21T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 154.627912 +2011-02-21T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 126.995117 +2011-02-21T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 138.092468 +2011-02-21T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 119.850150 +2011-02-21T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 119.739112 +2011-02-21T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1544.108134 +2011-02-21T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 1488.737765 +2011-02-21T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 1228.502469 value +2011-02-21T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 1298.415763 value +2011-02-22T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 149.171056 +2011-02-22T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 124.524992 +2011-02-22T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 139.557139 +2011-02-22T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 108.370907 +2011-02-22T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 111.394542 +2011-02-22T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 122.510640 +2011-02-22T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 126.596847 +2011-02-22T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 86.333235 +2011-02-22T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 115.915849 +2011-02-22T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1224.827108 +2011-02-22T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 1421.648704 +2011-02-22T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 1345.964309 value +2011-02-22T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 1291.897942 value +2011-02-23T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 165.273009 +2011-02-23T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 127.199815 +2011-02-23T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 145.588115 +2011-02-23T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 107.896489 +2011-02-23T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 113.141185 +2011-02-23T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 122.404192 +2011-02-23T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 124.305608 +2011-02-23T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 91.191071 +2011-02-23T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 123.961542 +2011-02-23T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1251.906228 +2011-02-23T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 1414.619004 +2011-02-23T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 1306.495696 value +2011-02-23T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 1287.766687 value +2011-02-24T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 168.988478 +2011-02-24T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 123.553981 +2011-02-24T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 139.978575 +2011-02-24T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 104.951315 +2011-02-24T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 113.621184 +2011-02-24T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 118.862342 +2011-02-24T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 119.772575 +2011-02-24T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 91.962584 +2011-02-24T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 118.270052 +2011-02-24T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1350.175381 +2011-02-24T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 970.728273 +2011-02-24T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 1631.584352 value +2011-02-24T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 578.795979 value +2011-02-25T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 172.335540 +2011-02-25T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 107.539869 +2011-02-25T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 140.941317 +2011-02-25T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 99.698015 +2011-02-25T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 99.460461 +2011-02-25T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 106.827630 +2011-02-25T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 115.932803 +2011-02-25T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 82.350556 +2011-02-25T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 120.124862 +2011-02-25T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1057.427269 +2011-02-25T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 1073.967314 +2011-02-25T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 1017.573185 value +2011-02-25T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 782.013486 value +2011-02-26T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 225.243186 +2011-02-26T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 103.643952 +2011-02-26T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 138.924835 +2011-02-26T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 103.667031 +2011-02-26T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 98.314744 +2011-02-26T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 105.352891 +2011-02-26T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 118.896845 +2011-02-26T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 83.099365 +2011-02-26T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 131.310541 +2011-02-26T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 996.433708 +2011-02-26T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 1743.921750 +2011-02-26T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 829.916235 value +2011-02-26T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 1862.737933 value +2011-02-27T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 277.273533 +2011-02-27T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 96.864384 +2011-02-27T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 136.394846 +2011-02-27T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 114.634278 +2011-02-27T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 103.226967 +2011-02-27T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 99.158839 +2011-02-27T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 121.929932 +2011-02-27T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 78.727950 +2011-02-27T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 136.163414 +2011-02-27T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1044.562903 +2011-02-27T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 1474.591017 +2011-02-27T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 873.306547 value +2011-02-27T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 1427.016724 value +2011-02-28T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 122.258195 +2011-02-28T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 97.218943 +2011-02-28T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 141.261324 +2011-02-28T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 112.528286 +2011-02-28T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 102.185098 +2011-02-28T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 99.505465 +2011-02-28T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 121.786785 +2011-02-28T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 72.163651 +2011-02-28T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 114.284569 +2011-02-28T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1159.278766 +2011-02-28T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 1292.542896 +2011-02-28T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 1430.257348 value +2011-02-28T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 1101.918270 value +2011-03-01T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 153.059937 +2011-03-01T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 99.070796 +2011-03-01T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 143.424672 +2011-03-01T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 114.700932 +2011-03-01T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 105.453024 +2011-03-01T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 99.772347 +2011-03-01T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 123.251814 +2011-03-01T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 72.792970 +2011-03-01T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 116.975408 +2011-03-01T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1124.201419 +2011-03-01T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 1243.354010 +2011-03-01T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 1166.141121 value +2011-03-01T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 1004.940887 value +2011-03-02T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 174.890520 +2011-03-02T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 98.432014 +2011-03-02T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 147.117434 +2011-03-02T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 112.968782 +2011-03-02T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 109.239196 +2011-03-02T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 100.600391 +2011-03-02T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 120.212473 +2011-03-02T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 82.823988 +2011-03-02T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 116.460744 +2011-03-02T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1051.808940 +2011-03-02T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 1143.078414 +2011-03-02T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 740.183720 value +2011-03-02T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 865.777900 value +2011-03-03T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 127.994476 +2011-03-03T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 92.537499 +2011-03-03T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 140.215411 +2011-03-03T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 108.914095 +2011-03-03T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 108.784646 +2011-03-03T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 96.031371 +2011-03-03T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 115.393493 +2011-03-03T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 75.977564 +2011-03-03T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 114.188310 +2011-03-03T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1097.490771 +2011-03-03T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 1010.370296 +2011-03-03T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 901.307577 value +2011-03-03T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 691.958920 value +2011-03-04T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 119.851231 +2011-03-04T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 93.634505 +2011-03-04T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 132.832331 +2011-03-04T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 110.018472 +2011-03-04T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 107.285615 +2011-03-04T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 97.535226 +2011-03-04T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 113.883056 +2011-03-04T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 81.131208 +2011-03-04T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 109.607245 +2011-03-04T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1326.829155 +2011-03-04T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 1179.803776 +2011-03-04T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 1674.331703 value +2011-03-04T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 975.577927 value +2011-03-05T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 136.941770 +2011-03-05T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 97.942645 +2011-03-05T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 145.393016 +2011-03-05T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 108.394611 +2011-03-05T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 112.522435 +2011-03-05T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 102.486832 +2011-03-05T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 114.691277 +2011-03-05T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 81.105110 +2011-03-05T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 117.904527 +2011-03-05T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1281.601175 +2011-03-05T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 994.731237 +2011-03-05T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 1360.694785 value +2011-03-05T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 755.899363 value +2011-03-06T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 129.531062 +2011-03-06T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 99.508679 +2011-03-06T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 144.925734 +2011-03-06T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 113.069662 +2011-03-06T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 113.035167 +2011-03-06T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 102.536839 +2011-03-06T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 115.956859 +2011-03-06T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 81.612269 +2011-03-06T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 120.953163 +2011-03-06T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1081.650406 +2011-03-06T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 1103.239788 +2011-03-06T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 771.348460 value +2011-03-06T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 869.308360 value +2011-03-07T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 111.909348 +2011-03-07T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 105.575929 +2011-03-07T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 150.452695 +2011-03-07T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 118.024245 +2011-03-07T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 111.106693 +2011-03-07T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 107.220218 +2011-03-07T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 121.582721 +2011-03-07T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 68.699125 +2011-03-07T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 106.884238 +2011-03-07T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1177.858403 +2011-03-07T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 1152.547767 +2011-03-07T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 1095.637520 value +2011-03-07T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 906.373797 value +2011-03-08T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 109.764955 +2011-03-08T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 98.972716 +2011-03-08T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 143.214331 +2011-03-08T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 119.777621 +2011-03-08T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 107.465492 +2011-03-08T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 101.652185 +2011-03-08T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 122.692722 +2011-03-08T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 70.866726 +2011-03-08T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 111.704071 +2011-03-08T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1117.953961 +2011-03-08T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 1084.332554 +2011-03-08T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 988.893782 value +2011-03-08T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 835.762631 value +2011-03-09T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 139.260950 +2011-03-09T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 110.873407 +2011-03-09T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 138.466933 +2011-03-09T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 115.013313 +2011-03-09T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 105.613469 +2011-03-09T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 112.407868 +2011-03-09T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 121.220772 +2011-03-09T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 82.426362 +2011-03-09T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 107.998334 +2011-03-09T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1029.802500 +2011-03-09T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 1121.385333 +2011-03-09T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 776.702940 value +2011-03-09T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 875.683406 value +2011-03-10T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 148.809150 +2011-03-10T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 105.214709 +2011-03-10T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 134.212714 +2011-03-10T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 114.717338 +2011-03-10T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 119.613508 +2011-03-10T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 107.127962 +2011-03-10T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 118.864028 +2011-03-10T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 79.793836 +2011-03-10T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 107.706257 +2011-03-10T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1244.849915 +2011-03-10T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 1077.279402 +2011-03-10T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 1070.836247 value +2011-03-10T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 835.461226 value +2011-03-11T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 135.820968 +2011-03-11T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 106.898536 +2011-03-11T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 135.038992 +2011-03-11T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 112.856230 +2011-03-11T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 120.497687 +2011-03-11T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 108.135811 +2011-03-11T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 118.298350 +2011-03-11T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 67.731170 +2011-03-11T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 108.186877 +2011-03-11T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1098.543170 +2011-03-11T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 998.650727 +2011-03-11T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 715.516125 value +2011-03-11T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 755.646538 value +2011-03-12T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 155.728048 +2011-03-12T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 113.493460 +2011-03-12T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 132.687079 +2011-03-12T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 112.554597 +2011-03-12T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 114.681603 +2011-03-12T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 115.572940 +2011-03-12T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 118.574721 +2011-03-12T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 74.394926 +2011-03-12T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 109.384493 +2011-03-12T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1088.807596 +2011-03-12T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 1008.745525 +2011-03-12T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 771.100508 value +2011-03-12T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 764.508070 value +2011-03-13T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 149.637715 +2011-03-13T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 113.760384 +2011-03-13T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 120.113921 +2011-03-13T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 120.760130 +2011-03-13T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 103.227522 +2011-03-13T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 114.814070 +2011-03-13T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 120.620862 +2011-03-13T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 70.126017 +2011-03-13T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 108.579283 +2011-03-13T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 998.753955 +2011-03-13T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 1129.723252 +2011-03-13T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 736.409261 value +2011-03-13T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 884.837267 value +2011-03-14T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 153.191744 +2011-03-14T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 109.461442 +2011-03-14T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 123.248581 +2011-03-14T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 120.487244 +2011-03-14T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 104.716583 +2011-03-14T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 111.688901 +2011-03-14T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 122.275869 +2011-03-14T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 59.021022 +2011-03-14T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 104.724023 +2011-03-14T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1075.243024 +2011-03-14T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 1141.588400 +2011-03-14T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 918.722840 value +2011-03-14T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 893.985017 value +2011-03-15T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 145.963558 +2011-03-15T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 109.382273 +2011-03-15T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 121.386341 +2011-03-15T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 119.250945 +2011-03-15T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 103.583295 +2011-03-15T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 112.354294 +2011-03-15T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 122.038585 +2011-03-15T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 59.266595 +2011-03-15T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 103.134338 +2011-03-15T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1037.381049 +2011-03-15T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 1099.197263 +2011-03-15T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 832.874861 value +2011-03-15T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 850.995007 value +2011-03-16T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 147.471464 +2011-03-16T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 110.565004 +2011-03-16T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 110.070846 +2011-03-16T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 115.750963 +2011-03-16T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 99.137980 +2011-03-16T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 112.577264 +2011-03-16T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 120.455865 +2011-03-16T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 69.329723 +2011-03-16T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 107.903885 +2011-03-16T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 981.577244 +2011-03-16T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 1092.942008 +2011-03-16T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 767.973326 value +2011-03-16T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 848.339888 value +2011-03-17T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 148.905410 +2011-03-17T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 113.501786 +2011-03-17T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 109.666402 +2011-03-17T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 114.540037 +2011-03-17T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 105.996125 +2011-03-17T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 116.816958 +2011-03-17T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 122.740143 +2011-03-17T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 69.258523 +2011-03-17T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 110.797348 +2011-03-17T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1072.239320 +2011-03-17T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 1154.415689 +2011-03-17T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 878.683776 value +2011-03-17T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 906.101957 value +2011-03-18T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 180.343171 +2011-03-18T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 110.037579 +2011-03-18T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 130.260926 +2011-03-18T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 113.490115 +2011-03-18T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 111.540639 +2011-03-18T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 113.238556 +2011-03-18T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 119.629977 +2011-03-18T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 68.573162 +2011-03-18T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 114.564808 +2011-03-18T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1311.178603 +2011-03-18T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 1176.605164 +2011-03-18T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 1403.830217 value +2011-03-18T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 936.429632 value +2011-03-19T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 177.514270 +2011-03-19T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 109.788875 +2011-03-19T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 134.147573 +2011-03-19T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 117.197085 +2011-03-19T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 112.999693 +2011-03-19T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 112.236468 +2011-03-19T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 120.638001 +2011-03-19T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 72.369471 +2011-03-19T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 115.384807 +2011-03-19T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1107.220174 +2011-03-19T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 1102.698977 +2011-03-19T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 846.288386 value +2011-03-19T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 856.490089 value +2011-03-20T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 178.454262 +2011-03-20T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 123.507497 +2011-03-20T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 157.749330 +2011-03-20T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 117.851058 +2011-03-20T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 128.274705 +2011-03-20T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 125.496367 +2011-03-20T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 129.519442 +2011-03-20T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 85.013155 +2011-03-20T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 128.705337 +2011-03-20T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1285.090048 +2011-03-20T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 1217.547439 +2011-03-20T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 1032.257527 value +2011-03-20T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 954.754185 value +2011-03-21T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 182.035296 +2011-03-21T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 124.411632 +2011-03-21T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 157.153730 +2011-03-21T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 122.462424 +2011-03-21T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 128.149976 +2011-03-21T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 125.243882 +2011-03-21T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 131.807919 +2011-03-21T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 75.936640 +2011-03-21T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 123.653645 +2011-03-21T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1283.957016 +2011-03-21T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 1178.830164 +2011-03-21T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 1031.990042 value +2011-03-21T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 907.021565 value +2011-03-22T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 177.460613 +2011-03-22T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 121.270611 +2011-03-22T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 151.407583 +2011-03-22T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 124.400780 +2011-03-22T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 126.415884 +2011-03-22T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 124.970533 +2011-03-22T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 133.124963 +2011-03-22T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 79.948248 +2011-03-22T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 122.357549 +2011-03-22T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1301.778098 +2011-03-22T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 1119.247202 +2011-03-22T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 1110.788895 value +2011-03-22T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 843.952139 value +2011-03-23T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 154.019632 +2011-03-23T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 126.764513 +2011-03-23T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 158.592715 +2011-03-23T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 118.972310 +2011-03-23T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 126.672392 +2011-03-23T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 129.864384 +2011-03-23T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 131.786598 +2011-03-23T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 102.603152 +2011-03-23T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 123.754240 +2011-03-23T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1403.338838 +2011-03-23T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 1156.601892 +2011-03-23T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 1362.650586 value +2011-03-23T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 884.801502 value +2011-03-24T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 135.569784 +2011-03-24T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 123.895027 +2011-03-24T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 156.650862 +2011-03-24T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 119.777998 +2011-03-24T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 115.945757 +2011-03-24T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 125.044877 +2011-03-24T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 129.023978 +2011-03-24T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 88.521042 +2011-03-24T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 124.062061 +2011-03-24T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1429.580257 +2011-03-24T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 1137.842315 +2011-03-24T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 1684.268799 value +2011-03-24T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 873.652030 value +2011-03-25T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 140.577121 +2011-03-25T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 125.766952 +2011-03-25T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 157.432108 +2011-03-25T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 119.400783 +2011-03-25T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 114.706960 +2011-03-25T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 126.057960 +2011-03-25T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 128.943094 +2011-03-25T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 89.408906 +2011-03-25T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 136.136598 +2011-03-25T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1124.935193 +2011-03-25T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 1256.499779 +2011-03-25T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 855.717712 value +2011-03-25T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 996.564152 value +2011-03-26T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 148.957194 +2011-03-26T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 130.824022 +2011-03-26T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 162.815450 +2011-03-26T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 118.463523 +2011-03-26T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 117.449116 +2011-03-26T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 142.972964 +2011-03-26T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 139.214665 +2011-03-26T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 101.686196 +2011-03-26T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 138.663182 +2011-03-26T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1217.877395 +2011-03-26T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 1247.890809 +2011-03-26T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 1061.678577 value +2011-03-26T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 962.235801 value +2011-03-27T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 144.056669 +2011-03-27T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 135.183271 +2011-03-27T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 163.161361 +2011-03-27T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 130.599006 +2011-03-27T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 114.952545 +2011-03-27T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 139.294248 +2011-03-27T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 142.430177 +2011-03-27T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 108.489598 +2011-03-27T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 141.820068 +2011-03-27T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1185.709973 +2011-03-27T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 1345.781728 +2011-03-27T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 1019.898509 value +2011-03-27T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 1056.419292 value +2011-03-28T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 156.155294 +2011-03-28T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 124.336139 +2011-03-28T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 163.100154 +2011-03-28T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 131.191818 +2011-03-28T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 115.709767 +2011-03-28T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 127.403089 +2011-03-28T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 140.941296 +2011-03-28T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 103.578536 +2011-03-28T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 130.880788 +2011-03-28T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1250.166788 +2011-03-28T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 1390.754050 +2011-03-28T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 1198.723103 value +2011-03-28T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 1108.136072 value +2011-03-29T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 134.084672 +2011-03-29T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 117.626804 +2011-03-29T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 158.073319 +2011-03-29T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 128.074393 +2011-03-29T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 109.678515 +2011-03-29T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 122.620188 +2011-03-29T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 144.446039 +2011-03-29T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 91.604463 +2011-03-29T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 132.477651 +2011-03-29T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1224.116225 +2011-03-29T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 1361.080245 +2011-03-29T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 1289.009485 value +2011-03-29T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 1069.431801 value +2011-03-30T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 135.942820 +2011-03-30T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 120.283054 +2011-03-30T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 167.960620 +2011-03-30T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 121.215839 +2011-03-30T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 116.587746 +2011-03-30T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 122.612114 +2011-03-30T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 138.827311 +2011-03-30T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 93.331887 +2011-03-30T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 128.645571 +2011-03-30T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1190.933753 +2011-03-30T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 1310.797070 +2011-03-30T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 1001.134025 value +2011-03-30T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 1030.499562 value +2011-03-31T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 151.752485 +2011-03-31T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 124.414321 +2011-03-31T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 175.778647 +2011-03-31T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 120.607382 +2011-03-31T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 117.060598 +2011-03-31T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 125.243245 +2011-03-31T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 150.247713 +2011-03-31T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 93.390841 +2011-03-31T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 125.839686 +2011-03-31T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1466.209327 +2011-03-31T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 1366.447617 +2011-03-31T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 1734.274909 value +2011-03-31T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 1063.201156 value +2011-04-01T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 135.885094 +2011-04-01T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 118.570340 +2011-04-01T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 158.747224 +2011-04-01T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 120.134704 +2011-04-01T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 109.705815 +2011-04-01T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 121.583581 +2011-04-01T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 144.507368 +2011-04-01T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 78.622547 +2011-04-01T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 119.922742 +2011-04-01T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1314.839715 +2011-04-01T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 1522.043733 +2011-04-01T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 1447.341160 value +2011-04-01T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 1234.247546 value +2011-04-02T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 147.425935 +2011-04-02T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 112.987027 +2011-04-02T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 166.016049 +2011-04-02T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 113.446008 +2011-04-02T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 110.931934 +2011-04-02T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 114.290141 +2011-04-02T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 135.301506 +2011-04-02T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 97.387433 +2011-04-02T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 126.411364 +2011-04-02T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1193.556278 +2011-04-02T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 1321.375057 +2011-04-02T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 1144.342401 value +2011-04-02T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 1049.738585 value +2011-04-03T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 122.971856 +2011-04-03T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 105.735462 +2011-04-03T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 153.927965 +2011-04-03T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 103.532351 +2011-04-03T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 107.047773 +2011-04-03T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 107.919674 +2011-04-03T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 122.141707 +2011-04-03T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 80.861743 +2011-04-03T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 117.247070 +2011-04-03T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1055.783661 +2011-04-03T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 1021.638673 +2011-04-03T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 811.991286 value +2011-04-03T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 768.423077 value +2011-04-04T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 110.919829 +2011-04-04T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 107.613577 +2011-04-04T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 146.729242 +2011-04-04T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 105.375351 +2011-04-04T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 110.573670 +2011-04-04T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 114.382255 +2011-04-04T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 125.285894 +2011-04-04T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 72.668430 +2011-04-04T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 117.703023 +2011-04-04T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1197.008423 +2011-04-04T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 1131.531986 +2011-04-04T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 1151.069173 value +2011-04-04T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 877.079396 value +2011-04-05T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 113.318712 +2011-04-05T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 105.615563 +2011-04-05T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 141.713507 +2011-04-05T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 106.207931 +2011-04-05T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 109.890586 +2011-04-05T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 110.012987 +2011-04-05T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 124.478408 +2011-04-05T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 86.683603 +2011-04-05T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 117.051694 +2011-04-05T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1192.144303 +2011-04-05T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 1154.289559 +2011-04-05T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 1146.423036 value +2011-04-05T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 902.615706 value +2011-04-06T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 115.334018 +2011-04-06T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 109.700256 +2011-04-06T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 147.553562 +2011-04-06T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 100.775597 +2011-04-06T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 108.659345 +2011-04-06T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 113.408308 +2011-04-06T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 121.079585 +2011-04-06T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 92.336403 +2011-04-06T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 112.921482 +2011-04-06T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1272.677122 +2011-04-06T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 1141.514652 +2011-04-06T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 1355.843374 value +2011-04-06T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 897.393445 value +2011-04-07T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 117.508062 +2011-04-07T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 115.418054 +2011-04-07T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 141.565031 +2011-04-07T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 95.562446 +2011-04-07T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 102.933171 +2011-04-07T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 122.696621 +2011-04-07T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 119.763135 +2011-04-07T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 84.357701 +2011-04-07T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 117.432760 +2011-04-07T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1190.896088 +2011-04-07T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 1009.363132 +2011-04-07T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 1260.143027 value +2011-04-07T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 762.862488 value +2011-04-08T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 120.973860 +2011-04-08T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 116.248860 +2011-04-08T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 146.830618 +2011-04-08T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 96.226609 +2011-04-08T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 95.447888 +2011-04-08T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 120.709912 +2011-04-08T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 120.309688 +2011-04-08T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 94.631354 +2011-04-08T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 122.743898 +2011-04-08T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1006.913816 +2011-04-08T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 1032.599837 +2011-04-08T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 935.168026 value +2011-04-08T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 782.107861 value +2011-04-09T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 116.080323 +2011-04-09T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 116.060759 +2011-04-09T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 158.682525 +2011-04-09T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 95.509796 +2011-04-09T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 110.007248 +2011-04-09T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 121.905685 +2011-04-09T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 126.369367 +2011-04-09T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 92.905070 +2011-04-09T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 123.791320 +2011-04-09T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1137.385764 +2011-04-09T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 1030.075553 +2011-04-09T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 996.205369 value +2011-04-09T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 767.692135 value +2011-04-10T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 113.221448 +2011-04-10T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 95.570457 +2011-04-10T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 131.766616 +2011-04-10T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 99.950855 +2011-04-10T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 91.470524 +2011-04-10T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 99.393076 +2011-04-10T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 123.207579 +2011-04-10T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 84.898691 +2011-04-10T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 114.353962 +2011-04-10T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1005.253077 +2011-04-10T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 1030.094757 +2011-04-10T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 1031.741509 value +2011-04-10T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 775.965555 value +2011-04-11T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 130.165796 +2011-04-11T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 107.765101 +2011-04-11T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 142.751726 +2011-04-11T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 104.847285 +2011-04-11T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 95.272956 +2011-04-11T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 106.229286 +2011-04-11T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 126.823859 +2011-04-11T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 89.250155 +2011-04-11T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 122.049678 +2011-04-11T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1112.794811 +2011-04-11T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 1113.357530 +2011-04-11T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 1374.968412 value +2011-04-11T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 853.163039 value +2011-04-12T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 122.386348 +2011-04-12T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 106.380995 +2011-04-12T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 141.932300 +2011-04-12T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 103.142372 +2011-04-12T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 97.340631 +2011-04-12T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 105.381244 +2011-04-12T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 125.189098 +2011-04-12T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 90.533391 +2011-04-12T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 122.128172 +2011-04-12T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1153.974725 +2011-04-12T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 1069.640880 +2011-04-12T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 1456.611830 value +2011-04-12T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 811.925240 value +2011-04-13T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 122.688340 +2011-04-13T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 105.739623 +2011-04-13T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 136.983407 +2011-04-13T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 100.860813 +2011-04-13T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 94.839191 +2011-04-13T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 105.261296 +2011-04-13T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 119.836611 +2011-04-13T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 91.972558 +2011-04-13T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 120.145572 +2011-04-13T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1016.137449 +2011-04-13T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 994.902292 +2011-04-13T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 989.032799 value +2011-04-13T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 744.744657 value +2011-04-14T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 111.179339 +2011-04-14T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 101.984377 +2011-04-14T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 133.606430 +2011-04-14T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 99.738319 +2011-04-14T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 91.270553 +2011-04-14T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 101.251756 +2011-04-14T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 118.285128 +2011-04-14T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 84.951300 +2011-04-14T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 119.768525 +2011-04-14T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1032.154263 +2011-04-14T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 999.586450 +2011-04-14T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 1166.401205 value +2011-04-14T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 753.104985 value +2011-04-15T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 106.793700 +2011-04-15T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 94.469747 +2011-04-15T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 135.109191 +2011-04-15T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 99.596909 +2011-04-15T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 92.782760 +2011-04-15T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 97.859766 +2011-04-15T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 120.508160 +2011-04-15T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 89.646236 +2011-04-15T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 120.290348 +2011-04-15T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 994.752744 +2011-04-15T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 1029.056992 +2011-04-15T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 962.731172 value +2011-04-15T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 780.271977 value diff --git a/processing/src/test/resources/druid.sample.numericdims.tsv.bottom b/processing/src/test/resources/druid.sample.numericdims.tsv.bottom new file mode 100644 index 000000000000..5454e3d45165 --- /dev/null +++ b/processing/src/test/resources/druid.sample.numericdims.tsv.bottom @@ -0,0 +1,732 @@ +2011-01-12T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 100.000000 +2011-01-12T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 100.000000 +2011-01-12T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 100.000000 +2011-01-12T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 100.000000 +2011-01-12T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1000.000000 +2011-01-12T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 800.000000 value +2011-01-13T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 94.874713 +2011-01-13T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 110.087299 +2011-01-13T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 104.465767 +2011-01-13T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 108.863011 +2011-01-13T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 106.236928 +2011-01-13T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 1689.012875 +2011-01-13T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 1564.617729 value +2011-01-14T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 102.670409 +2011-01-14T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 94.000432 +2011-01-14T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 101.380760 +2011-01-14T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 114.974216 +2011-01-14T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1049.141912 +2011-01-14T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 1006.402111 value +2011-01-15T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 82.840417 +2011-01-15T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 100.551072 +2011-01-15T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 100.582654 +2011-01-15T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 102.366798 +2011-01-15T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 120.481420 +2011-01-15T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 1545.708865 +2011-01-15T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 1458.402661 value +2011-01-16T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 100.591602 +2011-01-16T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 93.085943 +2011-01-16T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 100.192013 +2011-01-16T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 87.280816 +2011-01-16T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1016.965229 +2011-01-16T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 852.437477 value +2011-01-17T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 105.442374 +2011-01-17T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 99.189052 +2011-01-17T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 103.692852 +2011-01-17T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 106.696362 +2011-01-17T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 134.415281 +2011-01-17T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 953.995422 +2011-01-17T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 712.774595 value +2011-01-18T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 107.244699 +2011-01-18T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 103.018934 +2011-01-18T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 104.724154 +2011-01-18T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 88.764512 +2011-01-18T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1022.783330 +2011-01-18T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 846.267516 value +2011-01-19T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 85.681683 +2011-01-19T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 92.314034 +2011-01-19T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 105.087466 +2011-01-19T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 104.568464 +2011-01-19T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 109.772202 +2011-01-19T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 849.877513 +2011-01-19T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 594.381703 value +2011-01-20T01:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 106.367723 +2011-01-20T01:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 96.112901 +2011-01-20T01:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 105.225158 +2011-01-20T01:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 77.759854 +2011-01-20T01:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1066.208012 +2011-01-20T01:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 870.115926 value +2011-01-22T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 105.282866 +2011-01-22T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 103.527592 +2011-01-22T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 103.399677 +2011-01-22T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 107.627793 +2011-01-22T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1240.525484 +2011-01-22T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 1410.278128 value +2011-01-23T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 100.432710 +2011-01-23T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 92.789692 +2011-01-23T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 103.730730 +2011-01-23T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 101.573522 +2011-01-23T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 128.699746 +2011-01-23T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 1349.254415 +2011-01-23T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 1224.501568 value +2011-01-24T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 104.485760 +2011-01-24T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 189.385952 +2011-01-24T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 104.167373 +2011-01-24T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 105.345921 +2011-01-24T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1102.866656 +2011-01-24T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 1215.589859 value +2011-01-25T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 90.111413 +2011-01-25T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 85.974579 +2011-01-25T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 96.594588 +2011-01-25T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 102.512878 +2011-01-25T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 122.077247 +2011-01-25T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 997.994544 +2011-01-25T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 786.363298 value +2011-01-26T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 105.873942 +2011-01-26T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 159.988606 +2011-01-26T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 105.266058 +2011-01-26T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 105.617702 +2011-01-26T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1037.449471 +2011-01-26T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 989.931541 value +2011-01-27T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 134.127106 +2011-01-27T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 90.806201 +2011-01-27T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 100.643435 +2011-01-27T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 102.612747 +2011-01-27T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 149.125271 +2011-01-27T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 1486.201299 +2011-01-27T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 1367.638074 value +2011-01-28T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 111.641077 +2011-01-28T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 107.788998 +2011-01-28T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 108.106449 +2011-01-28T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 93.869236 +2011-01-28T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1300.302260 +2011-01-28T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 1627.598064 value +2011-01-29T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 127.450345 +2011-01-29T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 103.345166 +2011-01-29T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 106.663538 +2011-01-29T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 104.311418 +2011-01-29T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 138.865014 +2011-01-29T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 792.326066 +2011-01-29T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 545.990623 value +2011-01-30T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 106.064111 +2011-01-30T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 137.198397 +2011-01-30T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 106.009926 +2011-01-30T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 110.528451 +2011-01-30T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1233.448863 +2011-01-30T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 1870.061029 value +2011-01-31T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 133.740047 +2011-01-31T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 104.548387 +2011-01-31T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 97.017604 +2011-01-31T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 103.021032 +2011-01-31T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 155.744951 +2011-01-31T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 1127.231000 +2011-01-31T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 943.497198 value +2011-02-01T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 103.890175 +2011-02-01T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 113.896016 +2011-02-01T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 104.383662 +2011-02-01T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 83.931272 +2011-02-01T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1203.465595 +2011-02-01T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 1667.497773 value +2011-02-02T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 113.492245 +2011-02-02T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 112.042996 +2011-02-02T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 97.711139 +2011-02-02T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 102.152053 +2011-02-02T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 123.170962 +2011-02-02T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 1410.792943 +2011-02-02T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 1273.707453 value +2011-02-03T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 98.877952 +2011-02-03T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 85.069784 +2011-02-03T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 100.976362 +2011-02-03T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 92.326431 +2011-02-03T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1033.401241 +2011-02-03T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 888.705280 value +2011-02-04T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 89.182906 +2011-02-04T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 112.471918 +2011-02-04T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 100.918373 +2011-02-04T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 103.103922 +2011-02-04T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 113.716758 +2011-02-04T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 1331.860983 +2011-02-04T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 1308.582051 value +2011-02-05T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 111.394244 +2011-02-05T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 94.312960 +2011-02-05T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 107.929804 +2011-02-05T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 90.732978 +2011-02-05T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1039.500513 +2011-02-05T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 785.078869 value +2011-02-06T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 134.462521 +2011-02-06T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 119.608310 +2011-02-06T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 101.847544 +2011-02-06T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 102.770913 +2011-02-06T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 121.270562 +2011-02-06T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 1029.995236 +2011-02-06T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 826.039207 value +2011-02-07T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 98.815847 +2011-02-07T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 97.480779 +2011-02-07T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 100.706057 +2011-02-07T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 83.902353 +2011-02-07T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1047.212887 +2011-02-07T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 1107.243787 value +2011-02-08T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 126.243536 +2011-02-08T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 106.969799 +2011-02-08T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 96.790543 +2011-02-08T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 103.308255 +2011-02-08T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 111.040150 +2011-02-08T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 1082.727640 +2011-02-08T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 911.956790 value +2011-02-09T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 115.548444 +2011-02-09T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 97.071703 +2011-02-09T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 113.554588 +2011-02-09T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 83.742151 +2011-02-09T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 971.050764 +2011-02-09T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 794.098825 value +2011-02-10T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 115.461691 +2011-02-10T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 117.629065 +2011-02-10T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 95.698374 +2011-02-10T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 105.708103 +2011-02-10T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 105.557241 +2011-02-10T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 1089.647884 +2011-02-10T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 901.327272 value +2011-02-11T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 114.637486 +2011-02-11T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 96.617339 +2011-02-11T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 112.571724 +2011-02-11T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 87.904114 +2011-02-11T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 980.386611 +2011-02-11T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 723.514254 value +2011-02-12T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 130.104979 +2011-02-12T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 115.225386 +2011-02-12T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 101.515571 +2011-02-12T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 108.637522 +2011-02-12T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 102.850696 +2011-02-12T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 1092.416967 +2011-02-12T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 879.406101 value +2011-02-13T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 118.841176 +2011-02-13T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 98.607490 +2011-02-13T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 117.965974 +2011-02-13T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 87.820236 +2011-02-13T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 987.067381 +2011-02-13T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 724.262526 value +2011-02-14T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 119.323168 +2011-02-14T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 123.098262 +2011-02-14T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 101.645725 +2011-02-14T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 121.060464 +2011-02-14T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 107.663239 +2011-02-14T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 1199.607472 +2011-02-14T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 948.657939 value +2011-02-15T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 115.527003 +2011-02-15T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 103.519393 +2011-02-15T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 117.138956 +2011-02-15T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 79.700998 +2011-02-15T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1044.384300 +2011-02-15T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 807.601674 value +2011-02-16T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 133.726576 +2011-02-16T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 130.717934 +2011-02-16T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 108.338531 +2011-02-16T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 123.270869 +2011-02-16T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 113.643571 +2011-02-16T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 1360.032423 +2011-02-16T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 1208.456692 value +2011-02-17T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 125.032692 +2011-02-17T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 101.092779 +2011-02-17T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 121.872569 +2011-02-17T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 91.969195 +2011-02-17T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 993.591221 +2011-02-17T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 679.619354 value +2011-02-18T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 151.053296 +2011-02-18T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 128.063524 +2011-02-18T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 101.960196 +2011-02-18T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 122.546253 +2011-02-18T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 120.151389 +2011-02-18T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 1601.829436 +2011-02-18T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 1649.533329 value +2011-02-19T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 123.447481 +2011-02-19T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 108.822138 +2011-02-19T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 119.371511 +2011-02-19T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 91.547944 +2011-02-19T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1200.527201 +2011-02-19T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 1433.398801 value +2011-02-20T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 157.483005 +2011-02-20T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 176.323916 +2011-02-20T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 127.819268 +2011-02-20T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 132.719065 +2011-02-20T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 124.637709 +2011-02-20T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 1304.326111 +2011-02-20T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 1022.854576 value +2011-02-21T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 132.231346 +2011-02-21T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 111.386745 +2011-02-21T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 126.995117 +2011-02-21T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 119.850150 +2011-02-21T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1544.108134 +2011-02-21T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 1228.502469 value +2011-02-22T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 149.171056 +2011-02-22T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 139.557139 +2011-02-22T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 111.394542 +2011-02-22T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 126.596847 +2011-02-22T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 115.915849 +2011-02-22T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 1421.648704 +2011-02-22T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 1291.897942 value +2011-02-23T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 127.199815 +2011-02-23T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 107.896489 +2011-02-23T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 122.404192 +2011-02-23T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 91.191071 +2011-02-23T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1251.906228 +2011-02-23T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 1306.495696 value +2011-02-24T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 168.988478 +2011-02-24T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 139.978575 +2011-02-24T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 113.621184 +2011-02-24T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 119.772575 +2011-02-24T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 118.270052 +2011-02-24T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 970.728273 +2011-02-24T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 578.795979 value +2011-02-25T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 107.539869 +2011-02-25T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 99.698015 +2011-02-25T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 106.827630 +2011-02-25T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 82.350556 +2011-02-25T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1057.427269 +2011-02-25T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 1017.573185 value +2011-02-26T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 225.243186 +2011-02-26T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 138.924835 +2011-02-26T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 98.314744 +2011-02-26T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 118.896845 +2011-02-26T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 131.310541 +2011-02-26T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 1743.921750 +2011-02-26T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 1862.737933 value +2011-02-27T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 96.864384 +2011-02-27T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 114.634278 +2011-02-27T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 99.158839 +2011-02-27T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 78.727950 +2011-02-27T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1044.562903 +2011-02-27T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 873.306547 value +2011-02-28T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 122.258195 +2011-02-28T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 141.261324 +2011-02-28T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 102.185098 +2011-02-28T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 121.786785 +2011-02-28T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 114.284569 +2011-02-28T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 1292.542896 +2011-02-28T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 1101.918270 value +2011-03-01T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 99.070796 +2011-03-01T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 114.700932 +2011-03-01T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 99.772347 +2011-03-01T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 72.792970 +2011-03-01T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1124.201419 +2011-03-01T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 1166.141121 value +2011-03-02T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 174.890520 +2011-03-02T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 147.117434 +2011-03-02T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 109.239196 +2011-03-02T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 120.212473 +2011-03-02T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 116.460744 +2011-03-02T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 1143.078414 +2011-03-02T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 865.777900 value +2011-03-03T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 92.537499 +2011-03-03T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 108.914095 +2011-03-03T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 96.031371 +2011-03-03T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 75.977564 +2011-03-03T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1097.490771 +2011-03-03T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 901.307577 value +2011-03-04T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 119.851231 +2011-03-04T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 132.832331 +2011-03-04T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 107.285615 +2011-03-04T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 113.883056 +2011-03-04T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 109.607245 +2011-03-04T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 1179.803776 +2011-03-04T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 975.577927 value +2011-03-05T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 97.942645 +2011-03-05T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 108.394611 +2011-03-05T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 102.486832 +2011-03-05T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 81.105110 +2011-03-05T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1281.601175 +2011-03-05T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 1360.694785 value +2011-03-06T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 129.531062 +2011-03-06T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 144.925734 +2011-03-06T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 113.035167 +2011-03-06T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 115.956859 +2011-03-06T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 120.953163 +2011-03-06T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 1103.239788 +2011-03-06T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 869.308360 value +2011-03-07T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 105.575929 +2011-03-07T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 118.024245 +2011-03-07T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 107.220218 +2011-03-07T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 68.699125 +2011-03-07T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1177.858403 +2011-03-07T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 1095.637520 value +2011-03-08T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 109.764955 +2011-03-08T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 143.214331 +2011-03-08T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 107.465492 +2011-03-08T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 122.692722 +2011-03-08T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 111.704071 +2011-03-08T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 1084.332554 +2011-03-08T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 835.762631 value +2011-03-09T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 110.873407 +2011-03-09T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 115.013313 +2011-03-09T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 112.407868 +2011-03-09T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 82.426362 +2011-03-09T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1029.802500 +2011-03-09T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 776.702940 value +2011-03-10T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 148.809150 +2011-03-10T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 134.212714 +2011-03-10T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 119.613508 +2011-03-10T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 118.864028 +2011-03-10T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 107.706257 +2011-03-10T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 1077.279402 +2011-03-10T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 835.461226 value +2011-03-11T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 106.898536 +2011-03-11T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 112.856230 +2011-03-11T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 108.135811 +2011-03-11T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 67.731170 +2011-03-11T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1098.543170 +2011-03-11T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 715.516125 value +2011-03-12T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 155.728048 +2011-03-12T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 132.687079 +2011-03-12T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 114.681603 +2011-03-12T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 118.574721 +2011-03-12T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 109.384493 +2011-03-12T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 1008.745525 +2011-03-12T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 764.508070 value +2011-03-13T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 113.760384 +2011-03-13T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 120.760130 +2011-03-13T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 114.814070 +2011-03-13T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 70.126017 +2011-03-13T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 998.753955 +2011-03-13T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 736.409261 value +2011-03-14T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 153.191744 +2011-03-14T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 123.248581 +2011-03-14T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 104.716583 +2011-03-14T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 122.275869 +2011-03-14T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 104.724023 +2011-03-14T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 1141.588400 +2011-03-14T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 893.985017 value +2011-03-15T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 109.382273 +2011-03-15T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 119.250945 +2011-03-15T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 112.354294 +2011-03-15T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 59.266595 +2011-03-15T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1037.381049 +2011-03-15T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 832.874861 value +2011-03-16T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 147.471464 +2011-03-16T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 110.070846 +2011-03-16T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 99.137980 +2011-03-16T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 120.455865 +2011-03-16T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 107.903885 +2011-03-16T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 1092.942008 +2011-03-16T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 848.339888 value +2011-03-17T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 113.501786 +2011-03-17T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 114.540037 +2011-03-17T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 116.816958 +2011-03-17T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 69.258523 +2011-03-17T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1072.239320 +2011-03-17T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 878.683776 value +2011-03-18T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 180.343171 +2011-03-18T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 130.260926 +2011-03-18T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 111.540639 +2011-03-18T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 119.629977 +2011-03-18T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 114.564808 +2011-03-18T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 1176.605164 +2011-03-18T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 936.429632 value +2011-03-19T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 109.788875 +2011-03-19T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 117.197085 +2011-03-19T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 112.236468 +2011-03-19T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 72.369471 +2011-03-19T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1107.220174 +2011-03-19T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 846.288386 value +2011-03-20T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 178.454262 +2011-03-20T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 157.749330 +2011-03-20T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 128.274705 +2011-03-20T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 129.519442 +2011-03-20T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 128.705337 +2011-03-20T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 1217.547439 +2011-03-20T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 954.754185 value +2011-03-21T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 124.411632 +2011-03-21T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 122.462424 +2011-03-21T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 125.243882 +2011-03-21T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 75.936640 +2011-03-21T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1283.957016 +2011-03-21T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 1031.990042 value +2011-03-22T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 177.460613 +2011-03-22T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 151.407583 +2011-03-22T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 126.415884 +2011-03-22T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 133.124963 +2011-03-22T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 122.357549 +2011-03-22T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 1119.247202 +2011-03-22T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 843.952139 value +2011-03-23T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 126.764513 +2011-03-23T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 118.972310 +2011-03-23T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 129.864384 +2011-03-23T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 102.603152 +2011-03-23T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1403.338838 +2011-03-23T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 1362.650586 value +2011-03-24T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 135.569784 +2011-03-24T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 156.650862 +2011-03-24T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 115.945757 +2011-03-24T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 129.023978 +2011-03-24T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 124.062061 +2011-03-24T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 1137.842315 +2011-03-24T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 873.652030 value +2011-03-25T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 125.766952 +2011-03-25T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 119.400783 +2011-03-25T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 126.057960 +2011-03-25T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 89.408906 +2011-03-25T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1124.935193 +2011-03-25T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 855.717712 value +2011-03-26T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 148.957194 +2011-03-26T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 162.815450 +2011-03-26T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 117.449116 +2011-03-26T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 139.214665 +2011-03-26T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 138.663182 +2011-03-26T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 1247.890809 +2011-03-26T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 962.235801 value +2011-03-27T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 135.183271 +2011-03-27T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 130.599006 +2011-03-27T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 114.952545 +2011-03-27T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 139.294248 +2011-03-27T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 142.430177 +2011-03-27T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 108.489598 +2011-03-27T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 141.820068 +2011-03-27T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1185.709973 +2011-03-27T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 1345.781728 +2011-03-27T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 1019.898509 value +2011-03-27T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 1056.419292 value +2011-03-28T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 156.155294 +2011-03-28T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 124.336139 +2011-03-28T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 163.100154 +2011-03-28T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 131.191818 +2011-03-28T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 115.709767 +2011-03-28T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 127.403089 +2011-03-28T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 140.941296 +2011-03-28T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 103.578536 +2011-03-28T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 130.880788 +2011-03-28T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1250.166788 +2011-03-28T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 1390.754050 +2011-03-28T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 1198.723103 value +2011-03-28T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 1108.136072 value +2011-03-29T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 134.084672 +2011-03-29T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 117.626804 +2011-03-29T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 158.073319 +2011-03-29T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 128.074393 +2011-03-29T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 109.678515 +2011-03-29T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 122.620188 +2011-03-29T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 144.446039 +2011-03-29T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 91.604463 +2011-03-29T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 132.477651 +2011-03-29T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1224.116225 +2011-03-29T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 1361.080245 +2011-03-29T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 1289.009485 value +2011-03-29T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 1069.431801 value +2011-03-30T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 135.942820 +2011-03-30T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 120.283054 +2011-03-30T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 167.960620 +2011-03-30T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 121.215839 +2011-03-30T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 116.587746 +2011-03-30T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 122.612114 +2011-03-30T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 138.827311 +2011-03-30T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 93.331887 +2011-03-30T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 128.645571 +2011-03-30T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1190.933753 +2011-03-30T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 1310.797070 +2011-03-30T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 1001.134025 value +2011-03-30T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 1030.499562 value +2011-03-31T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 151.752485 +2011-03-31T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 124.414321 +2011-03-31T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 175.778647 +2011-03-31T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 120.607382 +2011-03-31T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 117.060598 +2011-03-31T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 125.243245 +2011-03-31T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 150.247713 +2011-03-31T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 93.390841 +2011-03-31T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 125.839686 +2011-03-31T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1466.209327 +2011-03-31T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 1366.447617 +2011-03-31T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 1734.274909 value +2011-03-31T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 1063.201156 value +2011-04-01T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 135.885094 +2011-04-01T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 118.570340 +2011-04-01T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 158.747224 +2011-04-01T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 120.134704 +2011-04-01T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 109.705815 +2011-04-01T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 121.583581 +2011-04-01T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 144.507368 +2011-04-01T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 78.622547 +2011-04-01T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 119.922742 +2011-04-01T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1314.839715 +2011-04-01T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 1522.043733 +2011-04-01T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 1447.341160 value +2011-04-01T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 1234.247546 value +2011-04-02T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 147.425935 +2011-04-02T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 112.987027 +2011-04-02T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 166.016049 +2011-04-02T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 113.446008 +2011-04-02T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 110.931934 +2011-04-02T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 114.290141 +2011-04-02T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 135.301506 +2011-04-02T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 97.387433 +2011-04-02T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 126.411364 +2011-04-02T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1193.556278 +2011-04-02T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 1321.375057 +2011-04-02T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 1144.342401 value +2011-04-02T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 1049.738585 value +2011-04-03T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 122.971856 +2011-04-03T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 105.735462 +2011-04-03T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 153.927965 +2011-04-03T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 103.532351 +2011-04-03T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 107.047773 +2011-04-03T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 107.919674 +2011-04-03T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 122.141707 +2011-04-03T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 80.861743 +2011-04-03T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 117.247070 +2011-04-03T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1055.783661 +2011-04-03T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 1021.638673 +2011-04-03T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 811.991286 value +2011-04-03T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 768.423077 value +2011-04-04T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 110.919829 +2011-04-04T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 107.613577 +2011-04-04T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 146.729242 +2011-04-04T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 105.375351 +2011-04-04T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 110.573670 +2011-04-04T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 114.382255 +2011-04-04T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 125.285894 +2011-04-04T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 72.668430 +2011-04-04T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 117.703023 +2011-04-04T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1197.008423 +2011-04-04T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 1131.531986 +2011-04-04T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 1151.069173 value +2011-04-04T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 877.079396 value +2011-04-05T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 113.318712 +2011-04-05T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 105.615563 +2011-04-05T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 141.713507 +2011-04-05T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 106.207931 +2011-04-05T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 109.890586 +2011-04-05T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 110.012987 +2011-04-05T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 124.478408 +2011-04-05T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 86.683603 +2011-04-05T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 117.051694 +2011-04-05T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1192.144303 +2011-04-05T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 1154.289559 +2011-04-05T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 1146.423036 value +2011-04-05T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 902.615706 value +2011-04-06T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 115.334018 +2011-04-06T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 109.700256 +2011-04-06T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 147.553562 +2011-04-06T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 100.775597 +2011-04-06T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 108.659345 +2011-04-06T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 113.408308 +2011-04-06T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 121.079585 +2011-04-06T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 92.336403 +2011-04-06T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 112.921482 +2011-04-06T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1272.677122 +2011-04-06T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 1141.514652 +2011-04-06T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 1355.843374 value +2011-04-06T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 897.393445 value +2011-04-07T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 117.508062 +2011-04-07T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 115.418054 +2011-04-07T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 141.565031 +2011-04-07T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 95.562446 +2011-04-07T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 102.933171 +2011-04-07T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 122.696621 +2011-04-07T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 119.763135 +2011-04-07T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 84.357701 +2011-04-07T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 117.432760 +2011-04-07T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1190.896088 +2011-04-07T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 1009.363132 +2011-04-07T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 1260.143027 value +2011-04-07T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 762.862488 value +2011-04-08T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 120.973860 +2011-04-08T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 116.248860 +2011-04-08T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 146.830618 +2011-04-08T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 96.226609 +2011-04-08T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 95.447888 +2011-04-08T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 120.709912 +2011-04-08T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 120.309688 +2011-04-08T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 94.631354 +2011-04-08T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 122.743898 +2011-04-08T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1006.913816 +2011-04-08T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 1032.599837 +2011-04-08T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 935.168026 value +2011-04-08T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 782.107861 value +2011-04-09T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 116.080323 +2011-04-09T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 116.060759 +2011-04-09T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 158.682525 +2011-04-09T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 95.509796 +2011-04-09T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 110.007248 +2011-04-09T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 121.905685 +2011-04-09T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 126.369367 +2011-04-09T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 92.905070 +2011-04-09T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 123.791320 +2011-04-09T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1137.385764 +2011-04-09T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 1030.075553 +2011-04-09T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 996.205369 value +2011-04-09T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 767.692135 value +2011-04-10T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 113.221448 +2011-04-10T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 95.570457 +2011-04-10T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 131.766616 +2011-04-10T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 99.950855 +2011-04-10T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 91.470524 +2011-04-10T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 99.393076 +2011-04-10T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 123.207579 +2011-04-10T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 84.898691 +2011-04-10T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 114.353962 +2011-04-10T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1005.253077 +2011-04-10T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 1030.094757 +2011-04-10T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 1031.741509 value +2011-04-10T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 775.965555 value +2011-04-11T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 130.165796 +2011-04-11T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 107.765101 +2011-04-11T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 142.751726 +2011-04-11T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 104.847285 +2011-04-11T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 95.272956 +2011-04-11T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 106.229286 +2011-04-11T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 126.823859 +2011-04-11T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 89.250155 +2011-04-11T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 122.049678 +2011-04-11T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1112.794811 +2011-04-11T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 1113.357530 +2011-04-11T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 1374.968412 value +2011-04-11T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 853.163039 value +2011-04-12T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 122.386348 +2011-04-12T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 106.380995 +2011-04-12T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 141.932300 +2011-04-12T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 103.142372 +2011-04-12T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 97.340631 +2011-04-12T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 105.381244 +2011-04-12T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 125.189098 +2011-04-12T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 90.533391 +2011-04-12T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 122.128172 +2011-04-12T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1153.974725 +2011-04-12T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 1069.640880 +2011-04-12T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 1456.611830 value +2011-04-12T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 811.925240 value +2011-04-13T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 122.688340 +2011-04-13T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 105.739623 +2011-04-13T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 136.983407 +2011-04-13T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 100.860813 +2011-04-13T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 94.839191 +2011-04-13T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 105.261296 +2011-04-13T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 119.836611 +2011-04-13T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 91.972558 +2011-04-13T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 120.145572 +2011-04-13T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1016.137449 +2011-04-13T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 994.902292 +2011-04-13T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 989.032799 value +2011-04-13T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 744.744657 value +2011-04-14T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 111.179339 +2011-04-14T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 101.984377 +2011-04-14T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 133.606430 +2011-04-14T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 99.738319 +2011-04-14T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 91.270553 +2011-04-14T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 101.251756 +2011-04-14T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 118.285128 +2011-04-14T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 84.951300 +2011-04-14T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 119.768525 +2011-04-14T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1032.154263 +2011-04-14T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 999.586450 +2011-04-14T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 1166.401205 value +2011-04-14T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 753.104985 value +2011-04-15T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 106.793700 +2011-04-15T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 94.469747 +2011-04-15T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 135.109191 +2011-04-15T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 99.596909 +2011-04-15T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 92.782760 +2011-04-15T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 97.859766 +2011-04-15T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 120.508160 +2011-04-15T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 89.646236 +2011-04-15T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 120.290348 +2011-04-15T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 994.752744 +2011-04-15T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 1029.056992 +2011-04-15T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 962.731172 value +2011-04-15T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 780.271977 value diff --git a/processing/src/test/resources/druid.sample.numericdims.tsv.top b/processing/src/test/resources/druid.sample.numericdims.tsv.top new file mode 100644 index 000000000000..63496d1eb905 --- /dev/null +++ b/processing/src/test/resources/druid.sample.numericdims.tsv.top @@ -0,0 +1,477 @@ +2011-01-12T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 100.000000 +2011-01-12T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 100.000000 +2011-01-12T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 100.000000 +2011-01-12T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 100.000000 +2011-01-12T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 100.000000 +2011-01-12T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 1000.000000 +2011-01-12T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 800.000000 value +2011-01-13T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 103.629399 +2011-01-13T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 114.947403 +2011-01-13T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 102.851683 +2011-01-13T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 111.356672 +2011-01-13T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1040.945505 +2011-01-13T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 826.060182 value +2011-01-14T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 86.450372 +2011-01-14T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 109.573474 +2011-01-14T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 97.903068 +2011-01-14T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 104.611784 +2011-01-14T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 112.259958 +2011-01-14T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 1073.476545 +2011-01-14T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 869.643722 value +2011-01-15T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 99.781645 +2011-01-15T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 87.954346 +2011-01-15T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 99.383407 +2011-01-15T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 111.680229 +2011-01-15T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1007.365510 +2011-01-15T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 809.041763 value +2011-01-16T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 71.315931 +2011-01-16T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 99.007588 +2011-01-16T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 99.863171 +2011-01-16T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 103.348007 +2011-01-16T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 116.779610 +2011-01-16T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 1077.612663 +2011-01-16T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 879.988099 value +2011-01-17T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 105.914315 +2011-01-17T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 110.157325 +2011-01-17T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 103.615039 +2011-01-17T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 89.901887 +2011-01-17T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1075.089574 +2011-01-17T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 950.146770 value +2011-01-18T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 87.195139 +2011-01-18T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 94.452739 +2011-01-18T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 101.087367 +2011-01-18T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 108.979936 +2011-01-18T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 113.680094 +2011-01-18T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 937.061939 +2011-01-18T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 682.885525 value +2011-01-19T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 106.700550 +2011-01-19T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 97.620315 +2011-01-19T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 106.127830 +2011-01-19T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 77.316731 +2011-01-19T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1156.744712 +2011-01-19T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 1109.874950 value +2011-01-20T01:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 93.396274 +2011-01-20T01:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 90.439020 +2011-01-20T01:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 105.669498 +2011-01-20T01:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 101.305541 +2011-01-20T01:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 140.179069 +2011-01-20T01:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 904.340636 +2011-01-20T01:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 677.510973 value +2011-01-22T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 95.235266 +2011-01-22T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 93.681096 +2011-01-22T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 104.184494 +2011-01-22T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 99.284525 +2011-01-22T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 137.109783 +2011-01-22T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 1343.232494 +2011-01-22T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 1219.432170 value +2011-01-23T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 107.348157 +2011-01-23T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 96.826443 +2011-01-23T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 106.418686 +2011-01-23T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 110.467875 +2011-01-23T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1088.943083 +2011-01-23T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 979.306038 value +2011-01-24T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 96.671647 +2011-01-24T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 88.748460 +2011-01-24T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 97.906256 +2011-01-24T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 101.581339 +2011-01-24T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 131.695956 +2011-01-24T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 939.244103 +2011-01-24T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 716.609179 value +2011-01-25T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 101.624789 +2011-01-25T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 180.575246 +2011-01-25T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 102.907866 +2011-01-25T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 102.044542 +2011-01-25T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1109.875413 +2011-01-25T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 1301.023342 value +2011-01-26T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 84.906466 +2011-01-26T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 84.710523 +2011-01-26T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 96.046584 +2011-01-26T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 101.088903 +2011-01-26T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 122.160681 +2011-01-26T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 1686.419659 +2011-01-26T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 1609.096706 value +2011-01-27T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 104.882908 +2011-01-27T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 170.735853 +2011-01-27T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 104.609483 +2011-01-27T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 116.979005 +2011-01-27T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1074.006938 +2011-01-27T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 1023.295213 value +2011-01-28T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 123.006128 +2011-01-28T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 99.681629 +2011-01-28T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 106.075672 +2011-01-28T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 103.822842 +2011-01-28T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 158.739359 +2011-01-28T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 1021.334487 +2011-01-28T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 810.889422 value +2011-01-29T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 100.992147 +2011-01-29T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 114.905745 +2011-01-29T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 101.998823 +2011-01-29T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 109.549035 +2011-01-29T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1314.619452 +2011-01-29T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 1685.500085 value +2011-01-30T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 124.943293 +2011-01-30T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 108.415967 +2011-01-30T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 97.023907 +2011-01-30T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 104.298490 +2011-01-30T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 137.932693 +2011-01-30T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 805.930143 +2011-01-30T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 555.476028 value +2011-01-31T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 103.492964 +2011-01-31T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 124.171944 +2011-01-31T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 103.832040 +2011-01-31T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 85.125795 +2011-01-31T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1184.920651 +2011-01-31T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 1643.340851 value +2011-02-01T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 132.123776 +2011-02-01T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 103.652865 +2011-02-01T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 98.909356 +2011-02-01T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 102.480377 +2011-02-01T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 134.014606 +2011-02-01T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 1100.904846 +2011-02-01T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 913.561076 value +2011-02-02T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 104.963233 +2011-02-02T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 102.281859 +2011-02-02T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 105.578807 +2011-02-02T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 96.706279 +2011-02-02T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1097.211164 +2011-02-02T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 1218.561908 value +2011-02-03T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 85.770241 +2011-02-03T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 106.425780 +2011-02-03T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 100.559287 +2011-02-03T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 102.349315 +2011-02-03T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 134.140377 +2011-02-03T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 1283.166055 +2011-02-03T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 1113.114125 value +2011-02-04T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 106.888769 +2011-02-04T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 99.837572 +2011-02-04T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 106.050728 +2011-02-04T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 93.973465 +2011-02-04T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1025.633340 +2011-02-04T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 864.568891 value +2011-02-05T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 93.001571 +2011-02-05T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 117.030289 +2011-02-05T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 105.660538 +2011-02-05T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 100.646747 +2011-02-05T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 114.723682 +2011-02-05T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 1332.468373 +2011-02-05T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 1363.614929 value +2011-02-06T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 110.897359 +2011-02-06T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 93.585758 +2011-02-06T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 110.053071 +2011-02-06T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 93.620739 +2011-02-06T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1011.205470 +2011-02-06T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 787.125330 value +2011-02-07T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 130.194219 +2011-02-07T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 112.924874 +2011-02-07T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 97.434318 +2011-02-07T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 102.705243 +2011-02-07T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 115.246714 +2011-02-07T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 1057.079944 +2011-02-07T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 872.625669 value +2011-02-08T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 93.190129 +2011-02-08T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 97.432302 +2011-02-08T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 97.085047 +2011-02-08T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 75.735586 +2011-02-08T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1064.972638 +2011-02-08T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 1188.369265 value +2011-02-09T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 129.221792 +2011-02-09T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 111.729360 +2011-02-09T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 96.478571 +2011-02-09T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 105.498315 +2011-02-09T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 112.646238 +2011-02-09T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 1320.638308 +2011-02-09T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 1299.093262 value +2011-02-10T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 118.062165 +2011-02-10T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 97.235999 +2011-02-10T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 115.824976 +2011-02-10T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 91.750911 +2011-02-10T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1070.165582 +2011-02-10T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 1212.928303 value +2011-02-11T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 129.187009 +2011-02-11T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 114.960877 +2011-02-11T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 100.111873 +2011-02-11T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 105.672256 +2011-02-11T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 102.864842 +2011-02-11T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 1179.695901 +2011-02-11T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 1061.973330 value +2011-02-12T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 115.758445 +2011-02-12T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 96.457082 +2011-02-12T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 114.877503 +2011-02-12T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 88.142774 +2011-02-12T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 959.236186 +2011-02-12T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 602.979544 value +2011-02-13T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 119.490316 +2011-02-13T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 119.907266 +2011-02-13T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 100.905238 +2011-02-13T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 112.514409 +2011-02-13T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 106.033416 +2011-02-13T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 1103.458199 +2011-02-13T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 862.931321 value +2011-02-14T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 115.628202 +2011-02-14T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 103.008650 +2011-02-14T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 117.110451 +2011-02-14T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 73.717033 +2011-02-14T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1091.223197 +2011-02-14T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 1133.135123 value +2011-02-15T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 123.485071 +2011-02-15T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 121.563912 +2011-02-15T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 105.269599 +2011-02-15T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 121.411398 +2011-02-15T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 108.428302 +2011-02-15T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 1183.240825 +2011-02-15T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 914.525048 value +2011-02-16T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 116.432276 +2011-02-16T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 105.762627 +2011-02-16T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 117.334381 +2011-02-16T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 98.918664 +2011-02-16T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1289.097304 +2011-02-16T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 1553.348548 value +2011-02-17T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 147.942017 +2011-02-17T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 126.982673 +2011-02-17T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 103.940963 +2011-02-17T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 120.050545 +2011-02-17T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 112.150745 +2011-02-17T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 1021.071173 +2011-02-17T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 645.177645 value +2011-02-18T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 127.611947 +2011-02-18T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 100.849247 +2011-02-18T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 124.513018 +2011-02-18T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 92.174432 +2011-02-18T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1105.383465 +2011-02-18T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 1120.088751 value +2011-02-19T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 163.351690 +2011-02-19T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 133.726878 +2011-02-19T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 102.919452 +2011-02-19T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 125.052129 +2011-02-19T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 121.733400 +2011-02-19T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 1600.723226 +2011-02-19T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 1598.179271 value +2011-02-20T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 129.409606 +2011-02-20T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 109.790712 +2011-02-20T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 122.082375 +2011-02-20T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 105.985049 +2011-02-20T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1317.458323 +2011-02-20T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 1192.563067 value +2011-02-21T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 155.632898 +2011-02-21T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 193.787574 +2011-02-21T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 154.627912 +2011-02-21T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 138.092468 +2011-02-21T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 119.739112 +2011-02-21T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 1488.737765 +2011-02-21T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 1298.415763 value +2011-02-22T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 124.524992 +2011-02-22T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 108.370907 +2011-02-22T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 122.510640 +2011-02-22T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 86.333235 +2011-02-22T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1224.827108 +2011-02-22T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 1345.964309 value +2011-02-23T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 165.273009 +2011-02-23T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 145.588115 +2011-02-23T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 113.141185 +2011-02-23T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 124.305608 +2011-02-23T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 123.961542 +2011-02-23T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 1414.619004 +2011-02-23T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 1287.766687 value +2011-02-24T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 123.553981 +2011-02-24T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 104.951315 +2011-02-24T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 118.862342 +2011-02-24T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 91.962584 +2011-02-24T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1350.175381 +2011-02-24T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 1631.584352 value +2011-02-25T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 172.335540 +2011-02-25T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 140.941317 +2011-02-25T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 99.460461 +2011-02-25T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 115.932803 +2011-02-25T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 120.124862 +2011-02-25T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 1073.967314 +2011-02-25T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 782.013486 value +2011-02-26T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 103.643952 +2011-02-26T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 103.667031 +2011-02-26T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 105.352891 +2011-02-26T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 83.099365 +2011-02-26T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 996.433708 +2011-02-26T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 829.916235 value +2011-02-27T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 277.273533 +2011-02-27T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 136.394846 +2011-02-27T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 103.226967 +2011-02-27T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 121.929932 +2011-02-27T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 136.163414 +2011-02-27T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 1474.591017 +2011-02-27T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 1427.016724 value +2011-02-28T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 97.218943 +2011-02-28T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 112.528286 +2011-02-28T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 99.505465 +2011-02-28T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 72.163651 +2011-02-28T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1159.278766 +2011-02-28T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 1430.257348 value +2011-03-01T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 153.059937 +2011-03-01T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 143.424672 +2011-03-01T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 105.453024 +2011-03-01T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 123.251814 +2011-03-01T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 116.975408 +2011-03-01T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 1243.354010 +2011-03-01T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 1004.940887 value +2011-03-02T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 98.432014 +2011-03-02T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 112.968782 +2011-03-02T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 100.600391 +2011-03-02T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 82.823988 +2011-03-02T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1051.808940 +2011-03-02T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 740.183720 value +2011-03-03T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 127.994476 +2011-03-03T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 140.215411 +2011-03-03T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 108.784646 +2011-03-03T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 115.393493 +2011-03-03T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 114.188310 +2011-03-03T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 1010.370296 +2011-03-03T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 691.958920 value +2011-03-04T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 93.634505 +2011-03-04T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 110.018472 +2011-03-04T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 97.535226 +2011-03-04T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 81.131208 +2011-03-04T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1326.829155 +2011-03-04T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 1674.331703 value +2011-03-05T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 136.941770 +2011-03-05T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 145.393016 +2011-03-05T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 112.522435 +2011-03-05T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 114.691277 +2011-03-05T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 117.904527 +2011-03-05T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 994.731237 +2011-03-05T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 755.899363 value +2011-03-06T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 99.508679 +2011-03-06T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 113.069662 +2011-03-06T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 102.536839 +2011-03-06T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 81.612269 +2011-03-06T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1081.650406 +2011-03-06T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 771.348460 value +2011-03-07T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 111.909348 +2011-03-07T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 150.452695 +2011-03-07T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 111.106693 +2011-03-07T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 121.582721 +2011-03-07T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 106.884238 +2011-03-07T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 1152.547767 +2011-03-07T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 906.373797 value +2011-03-08T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 98.972716 +2011-03-08T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 119.777621 +2011-03-08T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 101.652185 +2011-03-08T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 70.866726 +2011-03-08T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1117.953961 +2011-03-08T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 988.893782 value +2011-03-09T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 139.260950 +2011-03-09T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 138.466933 +2011-03-09T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 105.613469 +2011-03-09T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 121.220772 +2011-03-09T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 107.998334 +2011-03-09T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 1121.385333 +2011-03-09T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 875.683406 value +2011-03-10T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 105.214709 +2011-03-10T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 114.717338 +2011-03-10T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 107.127962 +2011-03-10T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 79.793836 +2011-03-10T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1244.849915 +2011-03-10T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 1070.836247 value +2011-03-11T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 135.820968 +2011-03-11T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 135.038992 +2011-03-11T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 120.497687 +2011-03-11T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 118.298350 +2011-03-11T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 108.186877 +2011-03-11T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 998.650727 +2011-03-11T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 755.646538 value +2011-03-12T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 113.493460 +2011-03-12T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 112.554597 +2011-03-12T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 115.572940 +2011-03-12T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 74.394926 +2011-03-12T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1088.807596 +2011-03-12T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 771.100508 value +2011-03-13T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 149.637715 +2011-03-13T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 120.113921 +2011-03-13T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 103.227522 +2011-03-13T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 120.620862 +2011-03-13T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 108.579283 +2011-03-13T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 1129.723252 +2011-03-13T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 884.837267 value +2011-03-14T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 109.461442 +2011-03-14T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 120.487244 +2011-03-14T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 111.688901 +2011-03-14T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 59.021022 +2011-03-14T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1075.243024 +2011-03-14T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 918.722840 value +2011-03-15T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 145.963558 +2011-03-15T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 121.386341 +2011-03-15T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 103.583295 +2011-03-15T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 122.038585 +2011-03-15T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 103.134338 +2011-03-15T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 1099.197263 +2011-03-15T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 850.995007 value +2011-03-16T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 110.565004 +2011-03-16T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 115.750963 +2011-03-16T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 112.577264 +2011-03-16T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 69.329723 +2011-03-16T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 981.577244 +2011-03-16T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 767.973326 value +2011-03-17T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 148.905410 +2011-03-17T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 109.666402 +2011-03-17T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 105.996125 +2011-03-17T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 122.740143 +2011-03-17T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 110.797348 +2011-03-17T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 1154.415689 +2011-03-17T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 906.101957 value +2011-03-18T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 110.037579 +2011-03-18T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 113.490115 +2011-03-18T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 113.238556 +2011-03-18T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 68.573162 +2011-03-18T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1311.178603 +2011-03-18T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 1403.830217 value +2011-03-19T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 177.514270 +2011-03-19T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 134.147573 +2011-03-19T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 112.999693 +2011-03-19T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 120.638001 +2011-03-19T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 115.384807 +2011-03-19T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 1102.698977 +2011-03-19T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 856.490089 value +2011-03-20T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 123.507497 +2011-03-20T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 117.851058 +2011-03-20T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 125.496367 +2011-03-20T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 85.013155 +2011-03-20T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1285.090048 +2011-03-20T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 1032.257527 value +2011-03-21T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 182.035296 +2011-03-21T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 157.153730 +2011-03-21T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 128.149976 +2011-03-21T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 131.807919 +2011-03-21T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 123.653645 +2011-03-21T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 1178.830164 +2011-03-21T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 907.021565 value +2011-03-22T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 121.270611 +2011-03-22T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 124.400780 +2011-03-22T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 124.970533 +2011-03-22T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 79.948248 +2011-03-22T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1301.778098 +2011-03-22T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 1110.788895 value +2011-03-23T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 154.019632 +2011-03-23T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 158.592715 +2011-03-23T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 126.672392 +2011-03-23T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 131.786598 +2011-03-23T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 123.754240 +2011-03-23T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 1156.601892 +2011-03-23T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 884.801502 value +2011-03-24T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 123.895027 +2011-03-24T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 119.777998 +2011-03-24T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 125.044877 +2011-03-24T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 88.521042 +2011-03-24T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1429.580257 +2011-03-24T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 1684.268799 value +2011-03-25T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 140.577121 +2011-03-25T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 157.432108 +2011-03-25T00:00:00.000Z spot 1111 111.111 mezzanine 5555 555.555 preferred mpreferred 114.706960 +2011-03-25T00:00:00.000Z spot 1111 111.111 premium 7777 777.777 preferred ppreferred 128.943094 +2011-03-25T00:00:00.000Z spot 1111 111.111 travel 8899 889.999 preferred tpreferred 136.136598 +2011-03-25T00:00:00.000Z total_market 2222 222.222 premium 7777 777.777 preferred ppreferred 1256.499779 +2011-03-25T00:00:00.000Z upfront 3333 333.333 premium 7777 777.777 preferred ppreferred 996.564152 value +2011-03-26T00:00:00.000Z spot 1111 111.111 business 2222 222.222 preferred bpreferred 130.824022 +2011-03-26T00:00:00.000Z spot 1111 111.111 health 4444 444.444 preferred hpreferred 118.463523 +2011-03-26T00:00:00.000Z spot 1111 111.111 news 6666 666.666 preferred npreferred 142.972964 +2011-03-26T00:00:00.000Z spot 1111 111.111 technology 8888 888.888 preferred tpreferred 101.686196 +2011-03-26T00:00:00.000Z total_market 2222 222.222 mezzanine 5555 555.555 preferred mpreferred 1217.877395 +2011-03-26T00:00:00.000Z upfront 3333 333.333 mezzanine 5555 555.555 preferred mpreferred 1061.678577 value +2011-03-27T00:00:00.000Z spot 1111 111.111 automotive 1111 111.111 preferred apreferred 144.056669 +2011-03-27T00:00:00.000Z spot 1111 111.111 entertainment 3333 333.333 preferred epreferred 163.161361 diff --git a/server/src/main/java/io/druid/segment/realtime/firehose/IngestSegmentFirehose.java b/server/src/main/java/io/druid/segment/realtime/firehose/IngestSegmentFirehose.java index be34ddc92b2f..aada39e8c4a0 100644 --- a/server/src/main/java/io/druid/segment/realtime/firehose/IngestSegmentFirehose.java +++ b/server/src/main/java/io/druid/segment/realtime/firehose/IngestSegmentFirehose.java @@ -23,6 +23,7 @@ import com.google.common.collect.Iterables; import com.google.common.collect.Lists; import com.google.common.collect.Maps; +import com.metamx.common.IAE; import com.metamx.common.guava.Sequence; import com.metamx.common.guava.Sequences; import com.metamx.common.guava.Yielder; @@ -39,7 +40,9 @@ import io.druid.segment.LongColumnSelector; import io.druid.segment.ObjectColumnSelector; import io.druid.segment.column.Column; +import io.druid.segment.data.IndexedFloats; import io.druid.segment.data.IndexedInts; +import io.druid.segment.data.IndexedLongs; import io.druid.segment.filter.Filters; import io.druid.utils.Runnables; import org.joda.time.DateTime; @@ -128,18 +131,39 @@ public InputRow next() for (Map.Entry dimSelector : dimSelectors.entrySet()) { final String dim = dimSelector.getKey(); final DimensionSelector selector = dimSelector.getValue(); - final IndexedInts vals = selector.getRow(); - - if (vals.size() == 1) { - final String dimVal = selector.lookupName(vals.get(0)); - theEvent.put(dim, dimVal); - } else { - List dimVals = Lists.newArrayList(); - for (int i = 0; i < vals.size(); ++i) { - dimVals.add(selector.lookupName(vals.get(i))); - } - theEvent.put(dim, dimVals); + + Object dimVals; + switch (selector.getDimCapabilities().getType()) { + case STRING: + final IndexedInts vals = selector.getRow(); + + if (vals.size() == 1) { + dimVals = selector.lookupName(vals.get(0)); + } else { + List strVals = Lists.newArrayList(); + for (int i = 0; i < vals.size(); ++i) { + strVals.add(selector.lookupName(vals.get(i))); + } + dimVals = strVals; + } + break; + case LONG: + final IndexedLongs longVals = selector.getLongRow(); + dimVals = longVals.get(0); + break; + case FLOAT: + final IndexedFloats floatVals = selector.getFloatRow(); + dimVals = floatVals.get(0); + break; + case COMPLEX: + final Comparable comparableVal = selector.getComparableRow(); + dimVals = comparableVal; + break; + default: + throw new IAE("Invalid type: " + selector.getDimCapabilities().getType()); } + + theEvent.put(dim, dimVals); } for (Map.Entry metSelector : metSelectors.entrySet()) { diff --git a/server/src/main/java/io/druid/segment/realtime/plumber/Sink.java b/server/src/main/java/io/druid/segment/realtime/plumber/Sink.java index 91326192fdd9..8db23d8d0f20 100644 --- a/server/src/main/java/io/druid/segment/realtime/plumber/Sink.java +++ b/server/src/main/java/io/druid/segment/realtime/plumber/Sink.java @@ -24,12 +24,14 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.Iterators; import com.google.common.collect.Lists; +import com.google.common.collect.Maps; import com.google.common.collect.Sets; import com.metamx.common.IAE; import com.metamx.common.ISE; import io.druid.data.input.InputRow; import io.druid.query.aggregation.AggregatorFactory; import io.druid.segment.QueryableIndex; +import io.druid.segment.column.ColumnCapabilitiesImpl; import io.druid.segment.incremental.IncrementalIndex; import io.druid.segment.incremental.IncrementalIndexSchema; import io.druid.segment.incremental.IndexSizeExceededException; @@ -45,6 +47,7 @@ import java.util.Iterator; import java.util.LinkedHashSet; import java.util.List; +import java.util.Map; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.atomic.AtomicInteger; @@ -248,16 +251,20 @@ private FireHydrant makeNewCurrIndex(long minTimestamp, DataSchema schema) FireHydrant lastHydrant = hydrants.get(numHydrants - 1); newCount = lastHydrant.getCount() + 1; if (!indexSchema.getDimensionsSpec().hasCustomDimensions()) { + Map oldCapabilities; if (lastHydrant.hasSwapped()) { + oldCapabilities = Maps.newHashMap(); QueryableIndex oldIndex = lastHydrant.getSegment().asQueryableIndex(); for (String dim : oldIndex.getAvailableDimensions()) { dimOrder.add(dim); + oldCapabilities.put(dim, (ColumnCapabilitiesImpl) oldIndex.getColumn(dim).getCapabilities()); } } else { IncrementalIndex oldIndex = lastHydrant.getIndex(); dimOrder.addAll(oldIndex.getDimensionOrder()); + oldCapabilities = oldIndex.getColumnCapabilities(); } - newIndex.loadDimensionIterable(dimOrder); + newIndex.loadDimensionIterable(dimOrder, oldCapabilities); } } currHydrant = new FireHydrant(newIndex, newCount, getSegment().getIdentifier()); diff --git a/server/src/test/java/io/druid/segment/realtime/RealtimeManagerTest.java b/server/src/test/java/io/druid/segment/realtime/RealtimeManagerTest.java index b8edd87bc3df..bbbc321b14b9 100644 --- a/server/src/test/java/io/druid/segment/realtime/RealtimeManagerTest.java +++ b/server/src/test/java/io/druid/segment/realtime/RealtimeManagerTest.java @@ -605,13 +605,13 @@ public void testQueryWithMultipleSegmentSpec() throws IOException, InterruptedEx interval_26_28, QueryRunnerTestHelper.makeQueryRunner( factory, - "druid.sample.tsv.top" + "druid.sample.numericdims.tsv.top" ) , interval_28_29, QueryRunnerTestHelper.makeQueryRunner( factory, - "druid.sample.tsv.bottom" + "druid.sample.numericdims.tsv.bottom" ) ); plumber.setRunners(runnerMap); diff --git a/server/src/test/java/io/druid/segment/realtime/firehose/IngestSegmentFirehoseTest.java b/server/src/test/java/io/druid/segment/realtime/firehose/IngestSegmentFirehoseTest.java index 48fc264afe5b..809a43fb31e7 100644 --- a/server/src/test/java/io/druid/segment/realtime/firehose/IngestSegmentFirehoseTest.java +++ b/server/src/test/java/io/druid/segment/realtime/firehose/IngestSegmentFirehoseTest.java @@ -23,6 +23,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; import io.druid.data.input.impl.CSVParseSpec; +import io.druid.data.input.impl.DimensionSchema; import io.druid.data.input.impl.DimensionsSpec; import io.druid.data.input.impl.StringInputRowParser; import io.druid.data.input.impl.TimestampSpec;