diff --git a/benchmarks/src/main/java/org/apache/druid/benchmark/query/SelectBenchmark.java b/benchmarks/src/main/java/org/apache/druid/benchmark/query/SelectBenchmark.java deleted file mode 100644 index 439e98461827..000000000000 --- a/benchmarks/src/main/java/org/apache/druid/benchmark/query/SelectBenchmark.java +++ /dev/null @@ -1,385 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.druid.benchmark.query; - -import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.common.base.Supplier; -import com.google.common.base.Suppliers; -import com.google.common.io.Files; -import org.apache.commons.io.FileUtils; -import org.apache.druid.benchmark.datagen.BenchmarkDataGenerator; -import org.apache.druid.benchmark.datagen.BenchmarkSchemaInfo; -import org.apache.druid.benchmark.datagen.BenchmarkSchemas; -import org.apache.druid.data.input.InputRow; -import org.apache.druid.data.input.Row; -import org.apache.druid.jackson.DefaultObjectMapper; -import org.apache.druid.java.util.common.concurrent.Execs; -import org.apache.druid.java.util.common.granularity.Granularities; -import org.apache.druid.java.util.common.guava.Sequence; -import org.apache.druid.java.util.common.logger.Logger; -import org.apache.druid.query.Druids; -import org.apache.druid.query.FinalizeResultsQueryRunner; -import org.apache.druid.query.Query; -import org.apache.druid.query.QueryPlus; -import org.apache.druid.query.QueryRunner; -import org.apache.druid.query.QueryRunnerFactory; -import org.apache.druid.query.QueryToolChest; -import org.apache.druid.query.Result; -import org.apache.druid.query.TableDataSource; -import org.apache.druid.query.aggregation.hyperloglog.HyperUniquesSerde; -import org.apache.druid.query.context.ResponseContext; -import org.apache.druid.query.dimension.DefaultDimensionSpec; -import org.apache.druid.query.select.EventHolder; -import org.apache.druid.query.select.PagingSpec; -import org.apache.druid.query.select.SelectQuery; -import org.apache.druid.query.select.SelectQueryConfig; -import org.apache.druid.query.select.SelectQueryEngine; -import org.apache.druid.query.select.SelectQueryQueryToolChest; -import org.apache.druid.query.select.SelectQueryRunnerFactory; -import org.apache.druid.query.select.SelectResultValue; -import org.apache.druid.query.spec.MultipleIntervalSegmentSpec; -import org.apache.druid.query.spec.QuerySegmentSpec; -import org.apache.druid.segment.IncrementalIndexSegment; -import org.apache.druid.segment.IndexIO; -import org.apache.druid.segment.IndexMergerV9; -import org.apache.druid.segment.IndexSpec; -import org.apache.druid.segment.QueryableIndex; -import org.apache.druid.segment.QueryableIndexSegment; -import org.apache.druid.segment.column.ColumnConfig; -import org.apache.druid.segment.incremental.IncrementalIndex; -import org.apache.druid.segment.serde.ComplexMetrics; -import org.apache.druid.segment.writeout.OffHeapMemorySegmentWriteOutMediumFactory; -import org.apache.druid.timeline.SegmentId; -import org.openjdk.jmh.annotations.Benchmark; -import org.openjdk.jmh.annotations.BenchmarkMode; -import org.openjdk.jmh.annotations.Fork; -import org.openjdk.jmh.annotations.Measurement; -import org.openjdk.jmh.annotations.Mode; -import org.openjdk.jmh.annotations.OutputTimeUnit; -import org.openjdk.jmh.annotations.Param; -import org.openjdk.jmh.annotations.Scope; -import org.openjdk.jmh.annotations.Setup; -import org.openjdk.jmh.annotations.State; -import org.openjdk.jmh.annotations.TearDown; -import org.openjdk.jmh.annotations.Warmup; -import org.openjdk.jmh.infra.Blackhole; - -import java.io.File; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.TimeUnit; - -@State(Scope.Benchmark) -@Fork(value = 1) -@Warmup(iterations = 10) -@Measurement(iterations = 25) -public class SelectBenchmark -{ - @Param({"1"}) - private int numSegments; - - @Param({"25000"}) - private int rowsPerSegment; - - @Param({"basic.A"}) - private String schemaAndQuery; - - @Param({"1000"}) - private int pagingThreshold; - - private static final Logger log = new Logger(SelectBenchmark.class); - private static final int RNG_SEED = 9999; - private static final IndexMergerV9 INDEX_MERGER_V9; - private static final IndexIO INDEX_IO; - public static final ObjectMapper JSON_MAPPER; - - private List incIndexes; - private List qIndexes; - - private QueryRunnerFactory factory; - - private BenchmarkSchemaInfo schemaInfo; - private Druids.SelectQueryBuilder queryBuilder; - private SelectQuery query; - private File tmpDir; - - private ExecutorService executorService; - - static { - JSON_MAPPER = new DefaultObjectMapper(); - INDEX_IO = new IndexIO( - JSON_MAPPER, - new ColumnConfig() - { - @Override - public int columnCacheSizeBytes() - { - return 0; - } - } - ); - INDEX_MERGER_V9 = new IndexMergerV9(JSON_MAPPER, INDEX_IO, OffHeapMemorySegmentWriteOutMediumFactory.instance()); - } - - private static final Map> SCHEMA_QUERY_MAP = new LinkedHashMap<>(); - - private void setupQueries() - { - // queries for the basic schema - Map basicQueries = new LinkedHashMap<>(); - BenchmarkSchemaInfo basicSchema = BenchmarkSchemas.SCHEMA_MAP.get("basic"); - - { // basic.A - QuerySegmentSpec intervalSpec = new MultipleIntervalSegmentSpec(Collections.singletonList(basicSchema.getDataInterval())); - - Druids.SelectQueryBuilder queryBuilderA = - Druids.newSelectQueryBuilder() - .dataSource(new TableDataSource("blah")) - .dimensionSpecs(DefaultDimensionSpec.toSpec(Collections.emptyList())) - .metrics(Collections.emptyList()) - .intervals(intervalSpec) - .granularity(Granularities.ALL) - .descending(false); - - basicQueries.put("A", queryBuilderA); - } - - SCHEMA_QUERY_MAP.put("basic", basicQueries); - } - - @Setup - public void setup() throws IOException - { - log.info("SETUP CALLED AT " + System.currentTimeMillis()); - - ComplexMetrics.registerSerde("hyperUnique", new HyperUniquesSerde()); - - executorService = Execs.multiThreaded(numSegments, "SelectThreadPool"); - - setupQueries(); - - String[] schemaQuery = schemaAndQuery.split("\\."); - String schemaName = schemaQuery[0]; - String queryName = schemaQuery[1]; - - schemaInfo = BenchmarkSchemas.SCHEMA_MAP.get(schemaName); - queryBuilder = SCHEMA_QUERY_MAP.get(schemaName).get(queryName); - queryBuilder.pagingSpec(PagingSpec.newSpec(pagingThreshold)); - query = queryBuilder.build(); - - incIndexes = new ArrayList<>(); - for (int i = 0; i < numSegments; i++) { - BenchmarkDataGenerator gen = new BenchmarkDataGenerator( - schemaInfo.getColumnSchemas(), - RNG_SEED + i, - schemaInfo.getDataInterval(), - rowsPerSegment - ); - - IncrementalIndex incIndex = makeIncIndex(); - - for (int j = 0; j < rowsPerSegment; j++) { - InputRow row = gen.nextRow(); - if (j % 10000 == 0) { - log.info(j + " rows generated."); - } - incIndex.add(row); - } - incIndexes.add(incIndex); - } - - tmpDir = Files.createTempDir(); - log.info("Using temp dir: " + tmpDir.getAbsolutePath()); - - qIndexes = new ArrayList<>(); - for (int i = 0; i < numSegments; i++) { - File indexFile = INDEX_MERGER_V9.persist( - incIndexes.get(i), - tmpDir, - new IndexSpec(), - null - ); - QueryableIndex qIndex = INDEX_IO.loadIndex(indexFile); - qIndexes.add(qIndex); - } - - final Supplier selectConfigSupplier = Suppliers.ofInstance(new SelectQueryConfig(true)); - - factory = new SelectQueryRunnerFactory( - new SelectQueryQueryToolChest( - JSON_MAPPER, - QueryBenchmarkUtil.noopIntervalChunkingQueryRunnerDecorator() - ), - new SelectQueryEngine(), - QueryBenchmarkUtil.NOOP_QUERYWATCHER - ); - } - - @TearDown - public void tearDown() throws IOException - { - FileUtils.deleteDirectory(tmpDir); - } - - private IncrementalIndex makeIncIndex() - { - return new IncrementalIndex.Builder() - .setSimpleTestingIndexSchema(schemaInfo.getAggsArray()) - .setReportParseExceptions(false) - .setMaxRowCount(rowsPerSegment) - .buildOnheap(); - } - - private static List runQuery(QueryRunnerFactory factory, QueryRunner runner, Query query) - { - - QueryToolChest toolChest = factory.getToolchest(); - QueryRunner theRunner = new FinalizeResultsQueryRunner<>( - toolChest.mergeResults(toolChest.preMergeQueryDecoration(runner)), - toolChest - ); - - Sequence queryResult = theRunner.run(QueryPlus.wrap(query), ResponseContext.createEmpty()); - return queryResult.toList(); - } - - /** - * Don't run this benchmark with a query that doesn't use {@link Granularities#ALL}, - * this pagination function probably doesn't work correctly in that case. - */ - private SelectQuery incrementQueryPagination(SelectQuery query, SelectResultValue prevResult) - { - Map pagingIdentifiers = prevResult.getPagingIdentifiers(); - Map newPagingIdentifers = new HashMap<>(); - - for (String segmentId : pagingIdentifiers.keySet()) { - int newOffset = pagingIdentifiers.get(segmentId) + 1; - newPagingIdentifers.put(segmentId, newOffset); - } - - return query.withPagingSpec(new PagingSpec(newPagingIdentifers, pagingThreshold)); - } - - @Benchmark - @BenchmarkMode(Mode.AverageTime) - @OutputTimeUnit(TimeUnit.MICROSECONDS) - public void queryIncrementalIndex(Blackhole blackhole) - { - SelectQuery queryCopy = query.withPagingSpec(PagingSpec.newSpec(pagingThreshold)); - - SegmentId segmentId = SegmentId.dummy("incIndex"); - QueryRunner runner = QueryBenchmarkUtil.makeQueryRunner( - factory, - segmentId, - new IncrementalIndexSegment(incIndexes.get(0), segmentId) - ); - - boolean done = false; - while (!done) { - List> results = SelectBenchmark.runQuery(factory, runner, queryCopy); - SelectResultValue result = results.get(0).getValue(); - if (result.getEvents().size() == 0) { - done = true; - } else { - for (EventHolder eh : result.getEvents()) { - blackhole.consume(eh); - } - queryCopy = incrementQueryPagination(queryCopy, result); - } - } - } - - @Benchmark - @BenchmarkMode(Mode.AverageTime) - @OutputTimeUnit(TimeUnit.MICROSECONDS) - public void queryQueryableIndex(Blackhole blackhole) - { - SelectQuery queryCopy = query.withPagingSpec(PagingSpec.newSpec(pagingThreshold)); - - SegmentId segmentId = SegmentId.dummy("qIndex"); - QueryRunner> runner = QueryBenchmarkUtil.makeQueryRunner( - factory, - segmentId, - new QueryableIndexSegment(qIndexes.get(0), segmentId) - ); - - boolean done = false; - while (!done) { - List> results = SelectBenchmark.runQuery(factory, runner, queryCopy); - SelectResultValue result = results.get(0).getValue(); - if (result.getEvents().size() == 0) { - done = true; - } else { - blackhole.consume(result); - queryCopy = incrementQueryPagination(queryCopy, result); - } - } - } - - - @Benchmark - @BenchmarkMode(Mode.AverageTime) - @OutputTimeUnit(TimeUnit.MICROSECONDS) - public void queryMultiQueryableIndex(Blackhole blackhole) - { - SelectQuery queryCopy = query.withPagingSpec(PagingSpec.newSpec(pagingThreshold)); - - List>> singleSegmentRunners = new ArrayList<>(); - QueryToolChest toolChest = factory.getToolchest(); - for (int i = 0; i < numSegments; i++) { - SegmentId segmentId = SegmentId.dummy("qIndex" + i); - QueryRunner> runner = QueryBenchmarkUtil.makeQueryRunner( - factory, - segmentId, - new QueryableIndexSegment(qIndexes.get(i), segmentId) - ); - singleSegmentRunners.add(toolChest.preMergeQueryDecoration(runner)); - } - - QueryRunner theRunner = toolChest.postMergeQueryDecoration( - new FinalizeResultsQueryRunner<>( - toolChest.mergeResults(factory.mergeRunners(executorService, singleSegmentRunners)), - toolChest - ) - ); - - - boolean done = false; - while (!done) { - Sequence> queryResult = theRunner.run(QueryPlus.wrap(queryCopy), ResponseContext.createEmpty()); - List> results = queryResult.toList(); - - SelectResultValue result = results.get(0).getValue(); - - if (result.getEvents().size() == 0) { - done = true; - } else { - blackhole.consume(result); - queryCopy = incrementQueryPagination(queryCopy, result); - } - } - } -} diff --git a/core/src/main/java/org/apache/druid/data/input/impl/TimestampSpec.java b/core/src/main/java/org/apache/druid/data/input/impl/TimestampSpec.java index 29334f9306c5..3f848d4d4aec 100644 --- a/core/src/main/java/org/apache/druid/data/input/impl/TimestampSpec.java +++ b/core/src/main/java/org/apache/druid/data/input/impl/TimestampSpec.java @@ -41,7 +41,7 @@ private static class ParseCtx DateTime lastDateTime = null; } - private static final String DEFAULT_COLUMN = "timestamp"; + public static final String DEFAULT_COLUMN = "timestamp"; private static final String DEFAULT_FORMAT = "auto"; private static final DateTime DEFAULT_MISSING_VALUE = null; diff --git a/docs/configuration/index.md b/docs/configuration/index.md index f59305853039..7abb0acff6cc 100644 --- a/docs/configuration/index.md +++ b/docs/configuration/index.md @@ -1177,7 +1177,7 @@ You can optionally configure caching to be enabled on the peons by setting cachi |--------|---------------|-----------|-------| |`druid.realtime.cache.useCache`|true, false|Enable the cache on the realtime.|false| |`druid.realtime.cache.populateCache`|true, false|Populate the cache on the realtime.|false| -|`druid.realtime.cache.unCacheable`|All druid query types|All query types to not cache.|`["groupBy", "select"]`| +|`druid.realtime.cache.unCacheable`|All druid query types|All query types to not cache.|`[]`| |`druid.realtime.cache.maxEntrySize`|Maximum cache entry size in bytes.|1_000_000| See [cache configuration](#cache-configuration) for how to configure cache settings. @@ -1311,7 +1311,7 @@ You can optionally configure caching to be enabled on the Indexer by setting cac |--------|---------------|-----------|-------| |`druid.realtime.cache.useCache`|true, false|Enable the cache on the realtime.|false| |`druid.realtime.cache.populateCache`|true, false|Populate the cache on the realtime.|false| -|`druid.realtime.cache.unCacheable`|All druid query types|All query types to not cache.|`["groupBy", "select"]`| +|`druid.realtime.cache.unCacheable`|All druid query types|All query types to not cache.|`[]`| |`druid.realtime.cache.maxEntrySize`|Maximum cache entry size in bytes.|1_000_000| See [cache configuration](#cache-configuration) for how to configure cache settings. @@ -1407,7 +1407,7 @@ You can optionally only configure caching to be enabled on the Historical by set |--------|---------------|-----------|-------| |`druid.historical.cache.useCache`|true, false|Enable the cache on the Historical.|false| |`druid.historical.cache.populateCache`|true, false|Populate the cache on the Historical.|false| -|`druid.historical.cache.unCacheable`|All druid query types|All query types to not cache.|["groupBy", "select"]| +|`druid.historical.cache.unCacheable`|All druid query types|All query types to not cache.|`[]`| |`druid.historical.cache.maxEntrySize`|Maximum cache entry size in bytes.|1_000_000| See [cache configuration](#cache-configuration) for how to configure cache settings. @@ -1544,7 +1544,7 @@ You can optionally only configure caching to be enabled on the Broker by setting |`druid.broker.cache.useResultLevelCache`|true, false|Enable result level caching on the Broker.|false| |`druid.broker.cache.populateResultLevelCache`|true, false|Populate the result level cache on the Broker.|false| |`druid.broker.cache.resultLevelCacheLimit`|positive integer|Maximum size of query response that can be cached.|`Integer.MAX_VALUE`| -|`druid.broker.cache.unCacheable`|All druid query types|All query types to not cache.|`["groupBy", "select"]`| +|`druid.broker.cache.unCacheable`|All druid query types|All query types to not cache.|`[]`| |`druid.broker.cache.cacheBulkMergeLimit`|positive integer or 0|Queries with more segments than this number will not attempt to fetch from cache at the broker level, leaving potential caching fetches (and cache result merging) to the Historicals|`Integer.MAX_VALUE`| |`druid.broker.cache.maxEntrySize`|Maximum cache entry size in bytes.|1_000_000| diff --git a/docs/querying/scan-query.md b/docs/querying/scan-query.md index 9e793bc4365b..934f5eb6ae58 100644 --- a/docs/querying/scan-query.md +++ b/docs/querying/scan-query.md @@ -24,10 +24,7 @@ sidebar_label: "Scan" --> -The Scan query returns raw Apache Druid (incubating) rows in streaming mode. The biggest difference between the Select query and the Scan -query is that the Scan query does not retain all the returned rows in memory before they are returned to the client. -The Select query _will_ retain the rows in memory, causing memory pressure if too many rows are returned. -The Scan query can return all the rows without issuing another pagination query. +The Scan query returns raw Apache Druid (incubating) rows in streaming mode. In addition to straightforward usage where a Scan query is issued to the Broker, the Scan query can also be issued directly to Historical processes or streaming ingestion tasks. This can be useful if you want to retrieve large diff --git a/docs/querying/select-query.md b/docs/querying/select-query.md index 83cd0fc20f23..13da7cee9db6 100644 --- a/docs/querying/select-query.md +++ b/docs/querying/select-query.md @@ -23,236 +23,7 @@ sidebar_label: "Select" ~ under the License. --> + +Older versions of Apache Druid (incubating) included a Select query type. Since Druid 0.17.0, it has been removed and replaced by the [Scan query](../querying/scan-query.md), which offers improved memory usage and performance. This solves issues that users had with Select queries causing Druid to run out of memory or slow down. -> We encourage you to use the [Scan query](../querying/scan-query.md) type rather than Select whenever possible. -> In situations involving larger numbers of segments, the Select query can have very high memory and performance overhead. -> The Scan query does not have this issue. -> The major difference between the two is that the Scan query does not support pagination. -> However, the Scan query type is able to return a virtually unlimited number of results even without pagination, making it unnecessary in many cases. - -Select queries return raw Druid rows and support pagination. - -```json - { - "queryType": "select", - "dataSource": "wikipedia", - "descending": "false", - "dimensions":[], - "metrics":[], - "granularity": "all", - "intervals": [ - "2013-01-01/2013-01-02" - ], - "pagingSpec":{"pagingIdentifiers": {}, "threshold":5} - } -``` - -There are several main parts to a select query: - -|property|description|required?| -|--------|-----------|---------| -|queryType|This String should always be "select"; this is the first thing Druid looks at to figure out how to interpret the query|yes| -|dataSource|A String or Object defining the data source to query, very similar to a table in a relational database. See [DataSource](../querying/datasource.md) for more information.|yes| -|intervals|A JSON Object representing ISO-8601 Intervals. This defines the time ranges to run the query over.|yes| -|descending|Whether to make descending ordered result. Default is `false`(ascending). When this is `true`, page identifier and offsets will be negative value.|no| -|filter|See [Filters](../querying/filters.md)|no| -|dimensions|A JSON list of dimensions to select; or see [DimensionSpec](../querying/dimensionspecs.md) for ways to extract dimensions. If left empty, all dimensions are returned.|no| -|metrics|A String array of metrics to select. If left empty, all metrics are returned.|no| -|granularity|Defines the granularity of the query. See [Granularities](../querying/granularities.md). Default is `Granularity.ALL`.|no| -|pagingSpec|A JSON object indicating offsets into different scanned segments. Query results will return a `pagingIdentifiers` value that can be reused in the next query for pagination.|yes| -|context|An additional JSON Object which can be used to specify certain flags.|no| - -The format of the result is: - -```json - [{ - "timestamp" : "2013-01-01T00:00:00.000Z", - "result" : { - "pagingIdentifiers" : { - "wikipedia_2012-12-29T00:00:00.000Z_2013-01-10T08:00:00.000Z_2013-01-10T08:13:47.830Z_v9" : 4 - }, - "events" : [ { - "segmentId" : "wikipedia_editstream_2012-12-29T00:00:00.000Z_2013-01-10T08:00:00.000Z_2013-01-10T08:13:47.830Z_v9", - "offset" : 0, - "event" : { - "timestamp" : "2013-01-01T00:00:00.000Z", - "robot" : "1", - "namespace" : "article", - "anonymous" : "0", - "unpatrolled" : "0", - "page" : "11._korpus_(NOVJ)", - "language" : "sl", - "newpage" : "0", - "user" : "EmausBot", - "count" : 1.0, - "added" : 39.0, - "delta" : 39.0, - "variation" : 39.0, - "deleted" : 0.0 - } - }, { - "segmentId" : "wikipedia_2012-12-29T00:00:00.000Z_2013-01-10T08:00:00.000Z_2013-01-10T08:13:47.830Z_v9", - "offset" : 1, - "event" : { - "timestamp" : "2013-01-01T00:00:00.000Z", - "robot" : "0", - "namespace" : "article", - "anonymous" : "0", - "unpatrolled" : "0", - "page" : "112_U.S._580", - "language" : "en", - "newpage" : "1", - "user" : "MZMcBride", - "count" : 1.0, - "added" : 70.0, - "delta" : 70.0, - "variation" : 70.0, - "deleted" : 0.0 - } - }, { - "segmentId" : "wikipedia_2012-12-29T00:00:00.000Z_2013-01-10T08:00:00.000Z_2013-01-10T08:13:47.830Z_v9", - "offset" : 2, - "event" : { - "timestamp" : "2013-01-01T00:00:00.000Z", - "robot" : "0", - "namespace" : "article", - "anonymous" : "0", - "unpatrolled" : "0", - "page" : "113_U.S._243", - "language" : "en", - "newpage" : "1", - "user" : "MZMcBride", - "count" : 1.0, - "added" : 77.0, - "delta" : 77.0, - "variation" : 77.0, - "deleted" : 0.0 - } - }, { - "segmentId" : "wikipedia_2012-12-29T00:00:00.000Z_2013-01-10T08:00:00.000Z_2013-01-10T08:13:47.830Z_v9", - "offset" : 3, - "event" : { - "timestamp" : "2013-01-01T00:00:00.000Z", - "robot" : "0", - "namespace" : "article", - "anonymous" : "0", - "unpatrolled" : "0", - "page" : "113_U.S._73", - "language" : "en", - "newpage" : "1", - "user" : "MZMcBride", - "count" : 1.0, - "added" : 70.0, - "delta" : 70.0, - "variation" : 70.0, - "deleted" : 0.0 - } - }, { - "segmentId" : "wikipedia_2012-12-29T00:00:00.000Z_2013-01-10T08:00:00.000Z_2013-01-10T08:13:47.830Z_v9", - "offset" : 4, - "event" : { - "timestamp" : "2013-01-01T00:00:00.000Z", - "robot" : "0", - "namespace" : "article", - "anonymous" : "0", - "unpatrolled" : "0", - "page" : "113_U.S._756", - "language" : "en", - "newpage" : "1", - "user" : "MZMcBride", - "count" : 1.0, - "added" : 68.0, - "delta" : 68.0, - "variation" : 68.0, - "deleted" : 0.0 - } - } ] - } -} ] -``` - -The `threshold` determines how many hits are returned, with each hit indexed by an offset. When `descending` is true, the offset will be negative value. - -The results above include: - -```json - "pagingIdentifiers" : { - "wikipedia_2012-12-29T00:00:00.000Z_2013-01-10T08:00:00.000Z_2013-01-10T08:13:47.830Z_v9" : 4 - }, -``` - -### Result pagination - -The PagingSpec allows the user to specify that the results of a select query should be returned as a paginated set. - -The `threshold` option controls how many rows are returned in each block of paginated results. - -To initiate a paginated query, the user should specify a PagingSpec with a `threshold` set and a blank `pagingIdentifiers` field, e.g.: - -```json -"pagingSpec":{"pagingIdentifiers": {}, "threshold":5} -``` - -When the query returns, the results will contain a `pagingIndentifers` field indicating the current pagination point in the result set (an identifier and an offset). - -To retrieve the next part of the result set, the user should issue the same select query, but replace the `pagingIdentifiers` field of the query with the `pagingIdentifiers` from the previous result. - -When an empty result set is received, all rows have been returned. - -#### fromNext backwards compatibility - -In older versions of Druid, when using paginated select queries, it was necessary for the user to manually increment the paging offset by 1 in each `pagingIdentifiers` before submitting the next query to retrieve the next set of results. This offset increment happens automatically in the current version of Druid by default, the user does not need to modify the `pagingIdentifiers` offset to retrieve the next result set. - -Setting the `fromNext` field of the PagingSpec to false instructs Druid to operate in the older mode where the user must manually increment the offset (or decrement for descending queries). - -For example, suppose the user issues the following initial paginated query, with `fromNext` false: - -```json - { - "queryType": "select", - "dataSource": "wikipedia", - "descending": "false", - "dimensions":[], - "metrics":[], - "granularity": "all", - "intervals": [ - "2013-01-01/2013-01-02" - ], - "pagingSpec":{"fromNext": "false", "pagingIdentifiers": {}, "threshold":5} - } -``` - - -The paginated query with `fromNext` set to false returns a result set with the following `pagingIdentifiers`: - -```json - "pagingIdentifiers" : { - "wikipedia_2012-12-29T00:00:00.000Z_2013-01-10T08:00:00.000Z_2013-01-10T08:13:47.830Z_v9" : 4 - }, -``` - -To retrieve the next result set, the next query must be sent with the paging offset (4) incremented by 1. - -```json - { - "queryType": "select", - "dataSource": "wikipedia", - "dimensions":[], - "metrics":[], - "granularity": "all", - "intervals": [ - "2013-01-01/2013-01-02" - ], - "pagingSpec":{"fromNext": "false", "pagingIdentifiers": {"wikipedia_2012-12-29T00:00:00.000Z_2013-01-10T08:00:00.000Z_2013-01-10T08:13:47.830Z_v9" : 5}, "threshold":5} - } -``` - -Note that specifying the `fromNext` option in the `pagingSpec` overrides the default value set by `druid.query.select.enableFromNextDefault` in the server configuration. See [Server configuration](#server-configuration) for more details. - -### Server configuration - -The following runtime properties apply to select queries: - -|Property|Description|Default| -|--------|-----------|-------| -|`druid.query.select.enableFromNextDefault`|If the `fromNext` property in a query's `pagingSpec` is left unspecified, the system will use the value of this property as the default value for `fromNext`. This option is true by default: the option of setting `fromNext` to false by default is intended to support backwards compatibility for deployments where some users may still expect behavior from older versions of Druid.|true| +The Scan query has a different syntax, but supports many of the features of the Select query, including time ordering and limiting. Scan does not include the Select query's pagination feature; however, in many cases pagination is unnecessary with Scan due to its ability to return a virtually unlimited number of results in one call. diff --git a/extensions-contrib/time-min-max/src/test/java/org/apache/druid/query/aggregation/TimestampAggregationSelectTest.java b/extensions-contrib/time-min-max/src/test/java/org/apache/druid/query/aggregation/TimestampAggregationSelectTest.java deleted file mode 100644 index 4c2951e5f673..000000000000 --- a/extensions-contrib/time-min-max/src/test/java/org/apache/druid/query/aggregation/TimestampAggregationSelectTest.java +++ /dev/null @@ -1,157 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.druid.query.aggregation; - -import com.google.common.base.Function; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.Iterables; -import com.google.common.io.Resources; -import org.apache.druid.java.util.common.DateTimes; -import org.apache.druid.java.util.common.granularity.Granularities; -import org.apache.druid.java.util.common.guava.Sequence; -import org.apache.druid.query.Result; -import org.apache.druid.query.select.SelectResultValue; -import org.apache.druid.segment.ColumnSelectorFactory; -import org.easymock.EasyMock; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; - -import javax.annotation.Nullable; -import java.io.File; -import java.nio.charset.StandardCharsets; -import java.sql.Timestamp; -import java.util.List; -import java.util.zip.ZipFile; - -@RunWith(Parameterized.class) -public class TimestampAggregationSelectTest -{ - private AggregationTestHelper helper; - - @Rule - public final TemporaryFolder temporaryFolder = new TemporaryFolder(); - - private ColumnSelectorFactory selectorFactory; - private TestObjectColumnSelector selector; - - private Timestamp[] values = new Timestamp[10]; - - @Parameterized.Parameters(name = "{index}: Test for {0}") - public static Iterable constructorFeeder() - { - return Iterables.transform( - ImmutableList.of( - ImmutableList.of("timeMin", "tmin", DateTimes.of("2011-01-12T01:00:00.000Z").getMillis()), - ImmutableList.of("timeMax", "tmax", DateTimes.of("2011-01-31T01:00:00.000Z").getMillis()) - ), - new Function, Object[]>() - { - @Nullable - @Override - public Object[] apply(List input) - { - return input.toArray(); - } - } - ); - } - - private String aggType; - private String aggField; - private Long expected; - - public TimestampAggregationSelectTest(String aggType, String aggField, Long expected) - { - this.aggType = aggType; - this.aggField = aggField; - this.expected = expected; - } - - @Before - public void setup() - { - helper = AggregationTestHelper.createSelectQueryAggregationTestHelper( - new TimestampMinMaxModule().getJacksonModules(), - temporaryFolder - ); - - selector = new TestObjectColumnSelector<>(values); - selectorFactory = EasyMock.createMock(ColumnSelectorFactory.class); - EasyMock.expect(selectorFactory.makeColumnValueSelector("test")).andReturn(selector); - EasyMock.replay(selectorFactory); - - } - - @Test - public void testSimpleDataIngestionAndSelectTest() throws Exception - { - String recordParser = "{\n" + - " \"type\": \"string\",\n" + - " \"parseSpec\": {\n" + - " \"format\": \"tsv\",\n" + - " \"timestampSpec\": {\n" + - " \"column\": \"timestamp\",\n" + - " \"format\": \"auto\"\n" + - " },\n" + - " \"dimensionsSpec\": {\n" + - " \"dimensions\": [\n" + - " \"product\"\n" + - " ],\n" + - " \"dimensionExclusions\": [],\n" + - " \"spatialDimensions\": []\n" + - " },\n" + - " \"columns\": [\n" + - " \"timestamp\",\n" + - " \"cat\",\n" + - " \"product\",\n" + - " \"prefer\",\n" + - " \"prefer2\",\n" + - " \"pty_country\"\n" + - " ]\n" + - " }\n" + - "}"; - String aggregator = "[\n" + - " {\n" + - " \"type\": \"" + aggType + "\",\n" + - " \"name\": \"" + aggField + "\",\n" + - " \"fieldName\": \"timestamp\"\n" + - " }\n" + - "]"; - ZipFile zip = new ZipFile(new File(this.getClass().getClassLoader().getResource("druid.sample.tsv.zip").toURI())); - Sequence seq = helper.createIndexAndRunQueryOnSegment( - zip.getInputStream(zip.getEntry("druid.sample.tsv")), - recordParser, - aggregator, - 0, - Granularities.MONTH, - 100, - Resources.toString(Resources.getResource("select.json"), StandardCharsets.UTF_8) - ); - - Result result = (Result) Iterables.getOnlyElement(seq.toList()); - Assert.assertEquals(36, result.getValue().getEvents().size()); - Assert.assertEquals(expected, result.getValue().getEvents().get(0).getEvent().get(aggField)); - } -} diff --git a/extensions-contrib/time-min-max/src/test/resources/select.json b/extensions-contrib/time-min-max/src/test/resources/select.json deleted file mode 100644 index 95547819aae0..000000000000 --- a/extensions-contrib/time-min-max/src/test/resources/select.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "queryType": "select", - "dataSource": "test_datasource", - "dimensions":[], - "metrics":[], - "granularity": "ALL", - "intervals": [ - "2011-01-01T00:00:00.000Z/2011-05-01T00:00:00.000Z" - ], - "pagingSpec":{"pagingIdentifiers": {}, "threshold":100} -} diff --git a/extensions-contrib/virtual-columns/src/test/java/org/apache/druid/segment/MapVirtualColumnSelectTest.java b/extensions-contrib/virtual-columns/src/test/java/org/apache/druid/segment/MapVirtualColumnSelectTest.java deleted file mode 100644 index 1f6155a254bc..000000000000 --- a/extensions-contrib/virtual-columns/src/test/java/org/apache/druid/segment/MapVirtualColumnSelectTest.java +++ /dev/null @@ -1,204 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.druid.segment; - -import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.common.base.Supplier; -import com.google.common.base.Suppliers; -import com.google.common.io.CharSource; -import org.apache.druid.data.input.impl.DelimitedParseSpec; -import org.apache.druid.data.input.impl.DimensionsSpec; -import org.apache.druid.data.input.impl.StringInputRowParser; -import org.apache.druid.data.input.impl.TimestampSpec; -import org.apache.druid.jackson.DefaultObjectMapper; -import org.apache.druid.java.util.common.DateTimes; -import org.apache.druid.query.Druids; -import org.apache.druid.query.QueryPlus; -import org.apache.druid.query.QueryRunner; -import org.apache.druid.query.QueryRunnerTestHelper; -import org.apache.druid.query.Result; -import org.apache.druid.query.select.EventHolder; -import org.apache.druid.query.select.PagingSpec; -import org.apache.druid.query.select.SelectQuery; -import org.apache.druid.query.select.SelectQueryConfig; -import org.apache.druid.query.select.SelectQueryEngine; -import org.apache.druid.query.select.SelectQueryQueryToolChest; -import org.apache.druid.query.select.SelectQueryRunnerFactory; -import org.apache.druid.query.select.SelectResultValue; -import org.apache.druid.segment.incremental.IncrementalIndex; -import org.apache.druid.segment.incremental.IncrementalIndexSchema; -import org.apache.druid.timeline.SegmentId; -import org.junit.Assert; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; - -import java.io.IOException; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; -import java.util.Map; - -/** - */ -@RunWith(Parameterized.class) -public class MapVirtualColumnSelectTest -{ - @Parameterized.Parameters - public static Iterable constructorFeeder() throws IOException - { - final Supplier selectConfigSupplier = Suppliers.ofInstance(new SelectQueryConfig(true)); - - SelectQueryRunnerFactory factory = new SelectQueryRunnerFactory( - new SelectQueryQueryToolChest( - new DefaultObjectMapper(), - QueryRunnerTestHelper.noopIntervalChunkingQueryRunnerDecorator() - ), - new SelectQueryEngine(), - QueryRunnerTestHelper.NOOP_QUERYWATCHER - ); - - final IncrementalIndexSchema schema = new IncrementalIndexSchema.Builder() - .withMinTimestamp(DateTimes.of("2011-01-12T00:00:00.000Z").getMillis()) - .build(); - final IncrementalIndex index = new IncrementalIndex.Builder() - .setIndexSchema(schema) - .setMaxRowCount(10000) - .buildOnheap(); - - final StringInputRowParser parser = new StringInputRowParser( - new DelimitedParseSpec( - new TimestampSpec("ts", "iso", null), - new DimensionsSpec(DimensionsSpec.getDefaultSchemas(Arrays.asList("dim", "keys", "values")), null, null), - "\t", - ",", - Arrays.asList("ts", "dim", "keys", "values"), - false, - 0 - ), - "utf8" - ); - - CharSource input = CharSource.wrap( - "2011-01-12T00:00:00.000Z\ta\tkey1,key2,key3\tvalue1,value2,value3\n" + - "2011-01-12T00:00:00.000Z\tb\tkey4,key5,key6\tvalue4\n" + - "2011-01-12T00:00:00.000Z\tc\tkey1,key5\tvalue1,value5,value9\n" - ); - - IncrementalIndex index1 = TestIndex.loadIncrementalIndex(() -> index, input, parser); - QueryableIndex index2 = TestIndex.persistRealtimeAndLoadMMapped(index1); - - return QueryRunnerTestHelper.transformToConstructionFeeder( - Arrays.asList( - QueryRunnerTestHelper.makeQueryRunner( - factory, - SegmentId.dummy("index1"), - new IncrementalIndexSegment(index1, SegmentId.dummy("index1")), - "incremental" - ), - QueryRunnerTestHelper.makeQueryRunner( - factory, - SegmentId.dummy("index2"), - new QueryableIndexSegment(index2, SegmentId.dummy("index2")), - "queryable" - ) - ) - ); - } - - private final QueryRunner runner; - - public MapVirtualColumnSelectTest(QueryRunner runner) - { - this.runner = runner; - } - - private Druids.SelectQueryBuilder testBuilder() - { - return Druids.newSelectQueryBuilder() - .dataSource(QueryRunnerTestHelper.DATA_SOURCE) - .granularity(QueryRunnerTestHelper.ALL_GRAN) - .intervals(QueryRunnerTestHelper.FULL_ON_INTERVAL_SPEC) - .pagingSpec(new PagingSpec(null, 3)); - } - - @Test - public void testSerde() throws IOException - { - final ObjectMapper mapper = new DefaultObjectMapper(); - new DruidVirtualColumnsModule().getJacksonModules().forEach(mapper::registerModule); - - final MapVirtualColumn column = new MapVirtualColumn("keys", "values", "params"); - final String json = mapper.writeValueAsString(column); - final VirtualColumn fromJson = mapper.readValue(json, VirtualColumn.class); - Assert.assertEquals(column, fromJson); - } - - @Test - public void testBasic() - { - Druids.SelectQueryBuilder builder = testBuilder(); - - List expectedResults = Arrays.asList( - MapVirtualColumnTestBase.mapOf( - "dim", "a", - "params.key1", "value1", - "params.key3", "value3", - "params.key5", null, - "params", MapVirtualColumnTestBase.mapOf("key1", "value1", "key2", "value2", "key3", "value3") - ), - MapVirtualColumnTestBase.mapOf( - "dim", "b", - "params.key1", null, - "params.key3", null, - "params.key5", null, - "params", MapVirtualColumnTestBase.mapOf("key4", "value4") - ), - MapVirtualColumnTestBase.mapOf( - "dim", "c", - "params.key1", "value1", - "params.key3", null, - "params.key5", "value5", - "params", MapVirtualColumnTestBase.mapOf("key1", "value1", "key5", "value5") - ) - ); - List virtualColumns = Collections.singletonList(new MapVirtualColumn("keys", "values", "params")); - SelectQuery selectQuery = builder.dimensions(Collections.singletonList("dim")) - .metrics(Arrays.asList("params.key1", "params.key3", "params.key5", "params")) - .virtualColumns(virtualColumns) - .build(); - checkSelectQuery(selectQuery, expectedResults); - } - - private void checkSelectQuery(SelectQuery searchQuery, List expected) - { - List> results = runner.run(QueryPlus.wrap(searchQuery)).toList(); - Assert.assertEquals(1, results.size()); - - List events = results.get(0).getValue().getEvents(); - - Assert.assertEquals(expected.size(), events.size()); - for (int i = 0; i < events.size(); i++) { - Map event = events.get(i).getEvent(); - event.remove(EventHolder.TIMESTAMP_KEY); - Assert.assertEquals(expected.get(i), event); - } - } -} diff --git a/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/theta/SketchAggregationWithSimpleDataTest.java b/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/theta/SketchAggregationWithSimpleDataTest.java index 8581eb1776e9..b5cb8f24abc8 100644 --- a/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/theta/SketchAggregationWithSimpleDataTest.java +++ b/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/theta/SketchAggregationWithSimpleDataTest.java @@ -34,7 +34,6 @@ import org.apache.druid.query.groupby.GroupByQueryConfig; import org.apache.druid.query.groupby.GroupByQueryRunnerTest; import org.apache.druid.query.groupby.ResultRow; -import org.apache.druid.query.select.SelectResultValue; import org.apache.druid.query.timeseries.TimeseriesResultValue; import org.apache.druid.query.topn.DimensionAndMetricValueExtractor; import org.apache.druid.query.topn.TopNResultValue; @@ -268,30 +267,6 @@ public void testSimpleDataIngestAndTopNQuery() throws Exception Assert.assertEquals("product_3", value.getDimensionValue("product")); } - @Test - public void testSimpleDataIngestAndSelectQuery() throws Exception - { - SketchModule.registerSerde(); - SketchModule sm = new SketchModule(); - AggregationTestHelper selectQueryAggregationTestHelper = AggregationTestHelper.createSelectQueryAggregationTestHelper( - sm.getJacksonModules(), - tempFolder - ); - - Sequence seq = selectQueryAggregationTestHelper.runQueryOnSegments( - ImmutableList.of(s1, s2), - readFileFromClasspathAsString("select_query.json") - ); - - Result result = (Result) Iterables.getOnlyElement(seq.toList()); - Assert.assertEquals(DateTimes.of("2014-10-20T00:00:00.000Z"), result.getTimestamp()); - Assert.assertEquals(100, result.getValue().getEvents().size()); - Assert.assertEquals( - "AgMDAAAazJMCAAAAAACAPzz9j7pWTMdROWGf15uY1nI=", - result.getValue().getEvents().get(0).getEvent().get("pty_country") - ); - } - @Test public void testTopNQueryWithSketchConstant() throws Exception { diff --git a/extensions-core/datasketches/src/test/resources/select_query.json b/extensions-core/datasketches/src/test/resources/select_query.json deleted file mode 100644 index f56116b54a8b..000000000000 --- a/extensions-core/datasketches/src/test/resources/select_query.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "queryType": "select", - "dataSource": "test_datasource", - "dimensions":[], - "metrics":[], - "granularity": "ALL", - "intervals": [ - "2014-10-19T00:00:00.000Z/2014-10-22T00:00:00.000Z" - ], - "pagingSpec":{"pagingIdentifiers": {}, "threshold":100} -} diff --git a/integration-tests/src/test/resources/indexer/select.query b/integration-tests/src/test/resources/indexer/select.query deleted file mode 100644 index 463ffef08e19..000000000000 --- a/integration-tests/src/test/resources/indexer/select.query +++ /dev/null @@ -1,19 +0,0 @@ -{ - "queryType": "select", - "intervals": ["2013-08-31/2013-09-01"], - "dataSource": "wikipedia_index_test", - "granularity": "all", - "filter": { - "type": "selector", - "dimension": "language", - "value": "en" - }, - "pagingSpec": { - "threshold": 10 - }, - "context": { - "useCache": "false", - "populateCache": "false", - "timeout": 60000 - } - } diff --git a/integration-tests/src/test/resources/indexer/select_reindex.query b/integration-tests/src/test/resources/indexer/select_reindex.query deleted file mode 100644 index e6a162377dc9..000000000000 --- a/integration-tests/src/test/resources/indexer/select_reindex.query +++ /dev/null @@ -1,19 +0,0 @@ -{ - "queryType": "select", - "intervals": ["2013-08-31/2013-09-01"], - "dataSource": "wikipedia_reindex_test", - "granularity": "all", - "filter": { - "type": "selector", - "dimension": "language", - "value": "en" - }, - "pagingSpec": { - "threshold": 10 - }, - "context": { - "useCache": "false", - "populateCache": "false", - "timeout": 60000 - } - } diff --git a/integration-tests/src/test/resources/indexer/union_select_query.json b/integration-tests/src/test/resources/indexer/union_select_query.json deleted file mode 100644 index e2af57cf2373..000000000000 --- a/integration-tests/src/test/resources/indexer/union_select_query.json +++ /dev/null @@ -1,121 +0,0 @@ -{ - "query": { - "queryType": "select", - "intervals": ["2013-08-31/2013-09-01"], - "dataSource": { - "type": "union", - "dataSources": [ - "wikipedia_index_test" - ] - }, - "granularity": "all", - "filter": { - "type": "selector", - "dimension": "language", - "value": "en" - }, - "pagingSpec": { - "threshold": 10 - }, - "context": { - "useCache": "false", - "populateCache": "false", - "timeout": 60000 - } - }, - "expectedResults": [ - { - "timestamp": "2013-08-31T01:02:33.000Z", - "result": { - "pagingIdentifiers": { - "wikipedia_index_test0_2013-08-31T00:00:00.000Z_2013-09-01T00:00:00.000Z_2014-05-01T15:27:43.993Z": 0, - "wikipedia_index_test1_2013-08-31T00:00:00.000Z_2013-09-01T00:00:00.000Z_2014-05-01T15:27:44.108Z": 0, - "wikipedia_index_test2_2013-08-31T00:00:00.000Z_2013-09-01T00:00:00.000Z_2014-05-01T15:27:44.236Z": 0, - "wikipedia_index_test3_2013-08-31T00:00:00.000Z_2013-09-01T00:00:00.000Z_2014-05-01T15:27:44.374Z": 0 - }, - "events": [ - { - "segmentId": "wikipedia_index_test0_2013-08-31T00:00:00.000Z_2013-09-01T00:00:00.000Z_2014-05-01T15:27:43.993Z", - "offset": 0, - "event": { - "timestamp": "2013-08-31T01:02:33.000Z", - "page": "Gypsy Danger", - "added": 57.0, - "deleted": 200.0 - } - }, - { - "segmentId": "wikipedia_index_test1_2013-08-31T00:00:00.000Z_2013-09-01T00:00:00.000Z_2014-05-01T15:27:44.108Z", - "offset": 0, - "event": { - "timestamp": "2013-08-31T01:02:33.000Z", - "page": "Gypsy Danger", - "added": 57.0, - "deleted": 200.0 - } - }, - { - "segmentId": "wikipedia_index_test2_2013-08-31T00:00:00.000Z_2013-09-01T00:00:00.000Z_2014-05-01T15:27:44.236Z", - "offset": 0, - "event": { - "timestamp": "2013-08-31T01:02:33.000Z", - "page": "Gypsy Danger", - "added": 57.0, - "deleted": 200.0 - } - }, - { - "segmentId": "wikipedia_index_test3_2013-08-31T00:00:00.000Z_2013-09-01T00:00:00.000Z_2014-05-01T15:27:44.374Z", - "offset": 0, - "event": { - "timestamp": "2013-08-31T01:02:33.000Z", - "page": "Gypsy Danger", - "added": 57.0, - "deleted": 200.0 - } - }, - { - "segmentId": "wikipedia_index_test0_2013-08-31T0org.apache.druid.java.util.common.ISE: one or more twitter queries failed0:00:00.000Z_2013-09-01T00:00:00.000Z_2014-05-01T15:27:43.993Z", - "offset": 0, - "event": { - "timestamp": "2013-08-31T03:32:45.000Z", - "page": "Striker Eureka", - "added": 459.0, - "deleted": 129.0 - } - }, - { - "segmentId": "wikipedia_index_test1_2013-08-31T00:00:00.000Z_2013-09-01T00:00:00.000Z_2014-05-01T15:27:44.108Z", - "offset": 0, - "event": { - "timestamp": "2013-08-31T03:32:45.000Z", - "page": "Striker Eureka", - "added": 459.0, - "deleted": 129.0 - } - }, - { - "segmentId": "wikipedia_index_test2_2013-08-31T00:00:00.000Z_2013-09-01T00:00:00.000Z_2014-05-01T15:27:44.236Z", - "offset": 0, - "event": { - "timestamp": "2013-08-31T03:32:45.000Z", - "page": "Striker Eureka", - "added": 459.0, - "deleted": 129.0 - } - }, - { - "segmentId": "wikipedia_index_test3_2013-08-31T00:00:00.000Z_2013-09-01T00:00:00.000Z_2014-05-01T15:27:44.374Z", - "offset": 0, - "event": { - "timestamp": "2013-08-31T03:32:45.000Z", - "page": "Striker Eureka", - "added": 459.0, - "deleted": 129.0 - } - } - ] - } - } - ] -} diff --git a/processing/src/main/java/org/apache/druid/query/Druids.java b/processing/src/main/java/org/apache/druid/query/Druids.java index 42836b242415..1dcd0767af0e 100644 --- a/processing/src/main/java/org/apache/druid/query/Druids.java +++ b/processing/src/main/java/org/apache/druid/query/Druids.java @@ -43,8 +43,6 @@ import org.apache.druid.query.search.SearchQuery; import org.apache.druid.query.search.SearchQuerySpec; import org.apache.druid.query.search.SearchSortSpec; -import org.apache.druid.query.select.PagingSpec; -import org.apache.druid.query.select.SelectQuery; import org.apache.druid.query.spec.LegacySegmentSpec; import org.apache.druid.query.spec.QuerySegmentSpec; import org.apache.druid.query.timeboundary.TimeBoundaryQuery; @@ -730,173 +728,6 @@ public static SegmentMetadataQueryBuilder newSegmentMetadataQueryBuilder() return new SegmentMetadataQueryBuilder(); } - /** - * A Builder for SelectQuery. - *

- * Required: dataSource(), intervals() must be called before build() - *

- * Usage example: - * 


-   *   SelectQuery query = new SelectQueryBuilder()
-   *                                  .dataSource("Example")
-   *                                  .interval("2010/2013")
-   *                                  .build();
-   *
- * - * @see SelectQuery - */ - public static class SelectQueryBuilder - { - private DataSource dataSource; - private QuerySegmentSpec querySegmentSpec; - private boolean descending; - private Map context; - private DimFilter dimFilter; - private Granularity granularity; - private List dimensions; - private List metrics; - private VirtualColumns virtualColumns; - private PagingSpec pagingSpec; - - public SelectQueryBuilder() - { - dataSource = null; - querySegmentSpec = null; - descending = false; - context = null; - dimFilter = null; - granularity = Granularities.ALL; - dimensions = new ArrayList<>(); - metrics = new ArrayList<>(); - virtualColumns = null; - pagingSpec = null; - } - - public SelectQuery build() - { - return new SelectQuery( - dataSource, - querySegmentSpec, - descending, - dimFilter, - granularity, - dimensions, - metrics, - virtualColumns, - pagingSpec, - context - ); - } - - public static SelectQueryBuilder copy(SelectQuery query) - { - return new SelectQueryBuilder() - .dataSource(query.getDataSource()) - .intervals(query.getQuerySegmentSpec()) - .descending(query.isDescending()) - .filters(query.getFilter()) - .granularity(query.getGranularity()) - .dimensionSpecs(query.getDimensions()) - .metrics(query.getMetrics()) - .virtualColumns(query.getVirtualColumns()) - .pagingSpec(query.getPagingSpec()) - .context(query.getContext()); - } - - public SelectQueryBuilder dataSource(String ds) - { - dataSource = new TableDataSource(ds); - return this; - } - - public SelectQueryBuilder dataSource(DataSource ds) - { - dataSource = ds; - return this; - } - - public SelectQueryBuilder intervals(QuerySegmentSpec q) - { - querySegmentSpec = q; - return this; - } - - public SelectQueryBuilder intervals(String s) - { - querySegmentSpec = new LegacySegmentSpec(s); - return this; - } - - public SelectQueryBuilder descending(boolean descending) - { - this.descending = descending; - return this; - } - - public SelectQueryBuilder context(Map c) - { - context = c; - return this; - } - - public SelectQueryBuilder filters(DimFilter f) - { - dimFilter = f; - return this; - } - - public SelectQueryBuilder granularity(Granularity g) - { - granularity = g; - return this; - } - - public SelectQueryBuilder dimensionSpecs(List d) - { - dimensions = d; - return this; - } - - public SelectQueryBuilder dimensions(List d) - { - dimensions = DefaultDimensionSpec.toSpec(d); - return this; - } - - public SelectQueryBuilder metrics(List m) - { - metrics = m; - return this; - } - - public SelectQueryBuilder virtualColumns(VirtualColumns vcs) - { - virtualColumns = vcs; - return this; - } - - public SelectQueryBuilder virtualColumns(List vcs) - { - return virtualColumns(VirtualColumns.create(vcs)); - } - - public SelectQueryBuilder virtualColumns(VirtualColumn... vcs) - { - return virtualColumns(VirtualColumns.create(Arrays.asList(vcs))); - } - - public SelectQueryBuilder pagingSpec(PagingSpec p) - { - pagingSpec = p; - return this; - } - } - - public static SelectQueryBuilder newSelectQueryBuilder() - { - return new SelectQueryBuilder(); - } - /** * A Builder for ScanQuery. *

diff --git a/processing/src/main/java/org/apache/druid/query/QueryConfig.java b/processing/src/main/java/org/apache/druid/query/QueryConfig.java index c3b73969f88d..b964710879cd 100644 --- a/processing/src/main/java/org/apache/druid/query/QueryConfig.java +++ b/processing/src/main/java/org/apache/druid/query/QueryConfig.java @@ -31,7 +31,6 @@ * @see org.apache.druid.query.search.SearchQueryConfig * @see org.apache.druid.query.topn.TopNQueryConfig * @see org.apache.druid.query.metadata.SegmentMetadataQueryConfig - * @see org.apache.druid.query.select.SelectQueryConfig * @see org.apache.druid.query.scan.ScanQueryConfig */ public class QueryConfig diff --git a/processing/src/main/java/org/apache/druid/query/QueryMetrics.java b/processing/src/main/java/org/apache/druid/query/QueryMetrics.java index e34d66196878..d770e7377adf 100644 --- a/processing/src/main/java/org/apache/druid/query/QueryMetrics.java +++ b/processing/src/main/java/org/apache/druid/query/QueryMetrics.java @@ -154,8 +154,7 @@ * dimensions than the default generic QueryMetrics. So those subinterfaces shouldn't be taken as direct examples for * following the plan specified above. * - * Refer {@link SearchQueryMetricsFactory} - * and {@link org.apache.druid.query.select.SelectQueryMetricsFactory} as an implementation example of this procedure. + * Refer {@link SearchQueryMetricsFactory} as an implementation example of this procedure. * * @param */ diff --git a/processing/src/main/java/org/apache/druid/query/dimension/DefaultDimensionSpec.java b/processing/src/main/java/org/apache/druid/query/dimension/DefaultDimensionSpec.java index ddc499c95e40..737d0ec8ccf3 100644 --- a/processing/src/main/java/org/apache/druid/query/dimension/DefaultDimensionSpec.java +++ b/processing/src/main/java/org/apache/druid/query/dimension/DefaultDimensionSpec.java @@ -29,10 +29,7 @@ import org.apache.druid.segment.vector.SingleValueDimensionVectorSelector; import javax.annotation.Nullable; -import java.util.Arrays; -import java.util.List; -import java.util.stream.Collectors; -import java.util.stream.StreamSupport; +import java.util.Objects; /** */ @@ -43,18 +40,6 @@ public static DefaultDimensionSpec of(String dimensionName) return new DefaultDimensionSpec(dimensionName, dimensionName); } - public static List toSpec(String... dimensionNames) - { - return toSpec(Arrays.asList(dimensionNames)); - } - - public static List toSpec(Iterable dimensionNames) - { - return StreamSupport.stream(dimensionNames.spliterator(), false) - .map(input -> new DefaultDimensionSpec(input, input)) - .collect(Collectors.toList()); - } - private static final byte CACHE_TYPE_ID = 0x0; private final String dimension; private final String outputName; @@ -174,13 +159,13 @@ public boolean equals(Object o) DefaultDimensionSpec that = (DefaultDimensionSpec) o; - if (dimension != null ? !dimension.equals(that.dimension) : that.dimension != null) { + if (!Objects.equals(dimension, that.dimension)) { return false; } - if (outputName != null ? !outputName.equals(that.outputName) : that.outputName != null) { + if (!Objects.equals(outputName, that.outputName)) { return false; } - if (outputType != null ? !outputType.equals(that.outputType) : that.outputType != null) { + if (!Objects.equals(outputType, that.outputType)) { return false; } diff --git a/processing/src/main/java/org/apache/druid/query/select/DefaultSelectQueryMetrics.java b/processing/src/main/java/org/apache/druid/query/select/DefaultSelectQueryMetrics.java deleted file mode 100644 index 80d92455a7d4..000000000000 --- a/processing/src/main/java/org/apache/druid/query/select/DefaultSelectQueryMetrics.java +++ /dev/null @@ -1,263 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.druid.query.select; - -import org.apache.druid.collections.bitmap.BitmapFactory; -import org.apache.druid.java.util.common.ISE; -import org.apache.druid.java.util.emitter.service.ServiceEmitter; -import org.apache.druid.query.BitmapResultFactory; -import org.apache.druid.query.Query; -import org.apache.druid.query.QueryMetrics; -import org.apache.druid.query.filter.Filter; -import org.joda.time.Interval; - -import java.util.List; - -/** - * This class is implemented with delegation to another QueryMetrics for compatibility, see "Making subinterfaces of - * QueryMetrics for emitting custom dimensions and/or metrics for specific query types" section in {@link QueryMetrics} - * javadoc. - */ -public class DefaultSelectQueryMetrics implements SelectQueryMetrics -{ - private QueryMetrics> delegateQueryMetrics; - - // queryMetrics.query(query) must already be called on the provided queryMetrics. - public DefaultSelectQueryMetrics(QueryMetrics> queryMetrics) - { - this.delegateQueryMetrics = queryMetrics; - } - - @Override - public void query(SelectQuery query) - { - // delegateQueryMetrics.query(query) must already be called on the provided queryMetrics. - } - - @Override - public void dataSource(SelectQuery query) - { - throw new ISE("Unsupported method in default query metrics implementation."); - } - - @Override - public void queryType(SelectQuery query) - { - throw new ISE("Unsupported method in default query metrics implementation."); - } - - @Override - public void interval(SelectQuery query) - { - throw new ISE("Unsupported method in default query metrics implementation."); - } - - @Override - public void hasFilters(SelectQuery query) - { - throw new ISE("Unsupported method in default query metrics implementation."); - } - - @Override - public void duration(SelectQuery query) - { - throw new ISE("Unsupported method in default query metrics implementation."); - } - - @Override - public void queryId(SelectQuery query) - { - throw new ISE("Unsupported method in default query metrics implementation."); - } - - @Override - public void sqlQueryId(SelectQuery query) - { - throw new ISE("Unsupported method in default query metrics implementation."); - } - - @Override - public void granularity(SelectQuery query) - { - // Don't emit by default - } - - @Override - public void context(SelectQuery query) - { - delegateQueryMetrics.context(query); - } - - @Override - public void server(String host) - { - delegateQueryMetrics.server(host); - } - - @Override - public void remoteAddress(String remoteAddress) - { - delegateQueryMetrics.remoteAddress(remoteAddress); - } - - @Override - public void status(String status) - { - delegateQueryMetrics.status(status); - } - - @Override - public void success(boolean success) - { - delegateQueryMetrics.success(success); - } - - @Override - public void segment(String segmentIdentifier) - { - delegateQueryMetrics.segment(segmentIdentifier); - } - - @Override - public void chunkInterval(Interval interval) - { - delegateQueryMetrics.chunkInterval(interval); - } - - @Override - public void preFilters(List preFilters) - { - delegateQueryMetrics.preFilters(preFilters); - } - - @Override - public void postFilters(List postFilters) - { - delegateQueryMetrics.postFilters(postFilters); - } - - @Override - public void identity(String identity) - { - delegateQueryMetrics.identity(identity); - } - - @Override - public void vectorized(final boolean vectorized) - { - delegateQueryMetrics.vectorized(vectorized); - } - - @Override - public BitmapResultFactory makeBitmapResultFactory(BitmapFactory factory) - { - return delegateQueryMetrics.makeBitmapResultFactory(factory); - } - - @Override - public QueryMetrics reportQueryTime(long timeNs) - { - return delegateQueryMetrics.reportQueryTime(timeNs); - } - - @Override - public QueryMetrics reportQueryBytes(long byteCount) - { - return delegateQueryMetrics.reportQueryBytes(byteCount); - } - - @Override - public QueryMetrics reportWaitTime(long timeNs) - { - return delegateQueryMetrics.reportWaitTime(timeNs); - } - - @Override - public QueryMetrics reportSegmentTime(long timeNs) - { - return delegateQueryMetrics.reportSegmentTime(timeNs); - } - - @Override - public QueryMetrics reportSegmentAndCacheTime(long timeNs) - { - return delegateQueryMetrics.reportSegmentAndCacheTime(timeNs); - } - - @Override - public QueryMetrics reportIntervalChunkTime(long timeNs) - { - return delegateQueryMetrics.reportIntervalChunkTime(timeNs); - } - - @Override - public QueryMetrics reportCpuTime(long timeNs) - { - return delegateQueryMetrics.reportCpuTime(timeNs); - } - - @Override - public QueryMetrics reportNodeTimeToFirstByte(long timeNs) - { - return delegateQueryMetrics.reportNodeTimeToFirstByte(timeNs); - } - - @Override - public QueryMetrics reportBackPressureTime(long timeNs) - { - return delegateQueryMetrics.reportBackPressureTime(timeNs); - } - - @Override - public QueryMetrics reportNodeTime(long timeNs) - { - return delegateQueryMetrics.reportNodeTime(timeNs); - } - - @Override - public QueryMetrics reportNodeBytes(long byteCount) - { - return delegateQueryMetrics.reportNodeBytes(byteCount); - } - - @Override - public QueryMetrics reportBitmapConstructionTime(long timeNs) - { - return delegateQueryMetrics.reportBitmapConstructionTime(timeNs); - } - - @Override - public QueryMetrics reportSegmentRows(long numRows) - { - return delegateQueryMetrics.reportSegmentRows(numRows); - } - - @Override - public QueryMetrics reportPreFilteredRows(long numRows) - { - return delegateQueryMetrics.reportPreFilteredRows(numRows); - } - - @Override - public void emit(ServiceEmitter emitter) - { - delegateQueryMetrics.emit(emitter); - } -} diff --git a/processing/src/main/java/org/apache/druid/query/select/DefaultSelectQueryMetricsFactory.java b/processing/src/main/java/org/apache/druid/query/select/DefaultSelectQueryMetricsFactory.java deleted file mode 100644 index 83384b2aa671..000000000000 --- a/processing/src/main/java/org/apache/druid/query/select/DefaultSelectQueryMetricsFactory.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.druid.query.select; - -import com.google.common.annotations.VisibleForTesting; -import com.google.inject.Inject; -import org.apache.druid.query.DefaultGenericQueryMetricsFactory; -import org.apache.druid.query.GenericQueryMetricsFactory; - -public class DefaultSelectQueryMetricsFactory implements SelectQueryMetricsFactory -{ - private static final SelectQueryMetricsFactory INSTANCE = - new DefaultSelectQueryMetricsFactory(DefaultGenericQueryMetricsFactory.instance()); - - private final GenericQueryMetricsFactory genericQueryMetricsFactory; - - @Inject - public DefaultSelectQueryMetricsFactory(GenericQueryMetricsFactory genericQueryMetricsFactory) - { - this.genericQueryMetricsFactory = genericQueryMetricsFactory; - } - - @VisibleForTesting - public static SelectQueryMetricsFactory instance() - { - return INSTANCE; - } - - @Override - public SelectQueryMetrics makeMetrics(SelectQuery query) - { - return new DefaultSelectQueryMetrics(genericQueryMetricsFactory.makeMetrics(query)); - } -} diff --git a/processing/src/main/java/org/apache/druid/query/select/EventHolder.java b/processing/src/main/java/org/apache/druid/query/select/EventHolder.java deleted file mode 100644 index ef5ee4400dfc..000000000000 --- a/processing/src/main/java/org/apache/druid/query/select/EventHolder.java +++ /dev/null @@ -1,128 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.druid.query.select; - -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.google.common.collect.Maps; -import org.apache.druid.java.util.common.DateTimes; -import org.apache.druid.java.util.common.ISE; -import org.joda.time.DateTime; - -import java.util.Map; - -/** - */ -public class EventHolder -{ - public static final String TIMESTAMP_KEY = "timestamp"; - - private final String segmentId; - private final int offset; - private final Map event; - - @JsonCreator - public EventHolder( - @JsonProperty("segmentId") String segmentId, - @JsonProperty("offset") int offset, - @JsonProperty("event") Map event - ) - { - this.segmentId = segmentId; - this.offset = offset; - this.event = event; - } - - public DateTime getTimestamp() - { - Object retVal = event.get(TIMESTAMP_KEY); - if (retVal instanceof Long) { - return DateTimes.utc((Long) retVal); - } else if (retVal instanceof String) { - return DateTimes.of((String) retVal); - } else if (retVal instanceof DateTime) { - return (DateTime) retVal; - } else { - throw new ISE("Do not understand format [%s]", retVal.getClass()); - } - } - - @JsonProperty - public String getSegmentId() - { - return segmentId; - } - - @JsonProperty - public int getOffset() - { - return offset; - } - - @JsonProperty - public Map getEvent() - { - return event; - } - - @Override - public boolean equals(Object o) - { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - - EventHolder that = (EventHolder) o; - - if (offset != that.offset) { - return false; - } - if (!Maps.difference(event, ((EventHolder) o).event).areEqual()) { - return false; - } - if (segmentId != null ? !segmentId.equals(that.segmentId) : that.segmentId != null) { - return false; - } - - return true; - } - - @Override - public int hashCode() - { - int result = segmentId != null ? segmentId.hashCode() : 0; - result = 31 * result + offset; - result = 31 * result + (event != null ? event.hashCode() : 0); - return result; - } - - @Override - public String toString() - { - return "EventHolder{" + - "segmentId='" + segmentId + '\'' + - ", offset=" + offset + - ", event=" + event + - '}'; - } -} diff --git a/processing/src/main/java/org/apache/druid/query/select/PagingOffset.java b/processing/src/main/java/org/apache/druid/query/select/PagingOffset.java deleted file mode 100644 index 3a72c1d46127..000000000000 --- a/processing/src/main/java/org/apache/druid/query/select/PagingOffset.java +++ /dev/null @@ -1,117 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.druid.query.select; - -import com.google.common.annotations.VisibleForTesting; - -/** - * offset iterator for select query - */ -public abstract class PagingOffset -{ - protected final int startOffset; - protected final int threshold; - - protected int counter; - - public PagingOffset(int startOffset, int threshold) - { - this.startOffset = startOffset; - this.threshold = threshold; - } - - public final int startOffset() - { - return startOffset; - } - - public abstract int startDelta(); - - public final int threshold() - { - return threshold; - } - - public final boolean hasNext() - { - return counter < threshold; - } - - public final void next() - { - counter++; - } - - public abstract int current(); - - private static class Ascending extends PagingOffset - { - public Ascending(int offset, int threshold) - { - super(offset, threshold); - } - - @Override - public final int startDelta() - { - return startOffset; - } - - @Override - public final int current() - { - return startOffset + counter; - } - } - - private static class Descending extends PagingOffset - { - public Descending(int offset, int threshold) - { - super(offset, threshold); - } - - @Override - public final int startDelta() - { - return -startOffset - 1; - } - - @Override - public final int current() - { - return startOffset - counter; - } - } - - public static PagingOffset of(int startOffset, int threshold) - { - return startOffset < 0 ? new Descending(startOffset, threshold) : new Ascending(startOffset, threshold); - } - - @VisibleForTesting - static int toOffset(int delta, boolean descending) - { - if (delta < 0) { - throw new IllegalArgumentException("Delta should not be negative"); - } - return descending ? -delta - 1 : delta; - } -} diff --git a/processing/src/main/java/org/apache/druid/query/select/PagingSpec.java b/processing/src/main/java/org/apache/druid/query/select/PagingSpec.java deleted file mode 100644 index 58c8628e05a3..000000000000 --- a/processing/src/main/java/org/apache/druid/query/select/PagingSpec.java +++ /dev/null @@ -1,194 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.druid.query.select; - -import com.fasterxml.jackson.annotation.JacksonInject; -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonProperty; -import org.apache.druid.java.util.common.StringUtils; - -import java.nio.ByteBuffer; -import java.util.HashMap; -import java.util.Map; - -/** - */ -public class PagingSpec -{ - public static PagingSpec newSpec(int threshold) - { - return new PagingSpec(null, threshold); - } - - public static Map merge(Iterable> cursors) - { - Map next = new HashMap<>(); - for (Map cursor : cursors) { - for (Map.Entry entry : cursor.entrySet()) { - next.put(entry.getKey(), entry.getValue()); - } - } - return next; - } - - public static Map next(Map cursor, boolean descending) - { - for (Map.Entry entry : cursor.entrySet()) { - entry.setValue(descending ? entry.getValue() - 1 : entry.getValue() + 1); - } - return cursor; - } - - private final Map pagingIdentifiers; - private final int threshold; - private final boolean fromNext; - - @JsonCreator - public PagingSpec( - @JsonProperty("pagingIdentifiers") Map pagingIdentifiers, - @JsonProperty("threshold") int threshold, - @JsonProperty("fromNext") Boolean fromNext, - @JacksonInject SelectQueryConfig config - ) - { - this.pagingIdentifiers = pagingIdentifiers == null ? new HashMap<>() : pagingIdentifiers; - this.threshold = threshold; - - boolean defaultFromNext = config.getEnableFromNextDefault(); - this.fromNext = fromNext == null ? defaultFromNext : fromNext; - } - - public PagingSpec(Map pagingIdentifiers, int threshold) - { - this(pagingIdentifiers, threshold, null, new SelectQueryConfig(true)); - } - - public PagingSpec(Map pagingIdentifiers, int threshold, Boolean fromNext) - { - this(pagingIdentifiers, threshold, fromNext, new SelectQueryConfig(true)); - } - - @JsonProperty - public Map getPagingIdentifiers() - { - return pagingIdentifiers; - } - - @JsonProperty - public int getThreshold() - { - return threshold; - } - - @JsonProperty - public boolean isFromNext() - { - return fromNext; - } - - public byte[] getCacheKey() - { - final byte[][] pagingKeys = new byte[pagingIdentifiers.size()][]; - final byte[][] pagingValues = new byte[pagingIdentifiers.size()][]; - - int index = 0; - int pagingKeysSize = 0; - int pagingValuesSize = 0; - for (Map.Entry entry : pagingIdentifiers.entrySet()) { - pagingKeys[index] = StringUtils.toUtf8(entry.getKey()); - pagingValues[index] = ByteBuffer.allocate(Integer.BYTES).putInt(entry.getValue()).array(); - pagingKeysSize += pagingKeys[index].length; - pagingValuesSize += Integer.BYTES; - index++; - } - - final byte[] thresholdBytes = ByteBuffer.allocate(Integer.BYTES).putInt(threshold).array(); - - final ByteBuffer queryCacheKey = ByteBuffer.allocate(pagingKeysSize + pagingValuesSize + thresholdBytes.length + 1); - - for (byte[] pagingKey : pagingKeys) { - queryCacheKey.put(pagingKey); - } - - for (byte[] pagingValue : pagingValues) { - queryCacheKey.put(pagingValue); - } - - queryCacheKey.put(thresholdBytes); - queryCacheKey.put(isFromNext() ? (byte) 0x01 : 0x00); - - return queryCacheKey.array(); - } - - public PagingOffset getOffset(String identifier, boolean descending) - { - Integer offset = pagingIdentifiers.get(identifier); - if (offset == null) { - offset = PagingOffset.toOffset(0, descending); - } else if (fromNext) { - offset = descending ? offset - 1 : offset + 1; - } - return PagingOffset.of(offset, threshold); - } - - @Override - public boolean equals(Object o) - { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - - PagingSpec that = (PagingSpec) o; - - if (fromNext != that.fromNext) { - return false; - } - if (threshold != that.threshold) { - return false; - } - if (!pagingIdentifiers.equals(that.pagingIdentifiers)) { - return false; - } - - return true; - } - - @Override - public int hashCode() - { - int result = pagingIdentifiers.hashCode(); - result = 31 * result + threshold; - result = 31 * result + (fromNext ? 1 : 0); - return result; - } - - @Override - public String toString() - { - return "PagingSpec{" + - "pagingIdentifiers=" + pagingIdentifiers + - ", threshold=" + threshold + - ", fromNext=" + fromNext + - '}'; - } -} diff --git a/processing/src/main/java/org/apache/druid/query/select/SelectBinaryFn.java b/processing/src/main/java/org/apache/druid/query/select/SelectBinaryFn.java deleted file mode 100644 index 9c758f35af75..000000000000 --- a/processing/src/main/java/org/apache/druid/query/select/SelectBinaryFn.java +++ /dev/null @@ -1,105 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.druid.query.select; - -import com.google.common.collect.Sets; -import org.apache.druid.java.util.common.granularity.AllGranularity; -import org.apache.druid.java.util.common.granularity.Granularity; -import org.apache.druid.query.Result; -import org.joda.time.DateTime; - -import java.util.List; -import java.util.Set; -import java.util.function.BinaryOperator; - -/** - */ -public class SelectBinaryFn implements BinaryOperator> -{ - private final Granularity gran; - private final PagingSpec pagingSpec; - private final boolean descending; - - public SelectBinaryFn(Granularity granularity, PagingSpec pagingSpec, boolean descending) - { - this.gran = granularity; - this.pagingSpec = pagingSpec; - this.descending = descending; - } - - @Override - public Result apply(Result arg1, Result arg2) - { - if (arg1 == null) { - return arg2; - } - - if (arg2 == null) { - return arg1; - } - - final List arg1Val = arg1.getValue().getEvents(); - final List arg2Val = arg2.getValue().getEvents(); - - if (arg1Val == null || arg1Val.isEmpty()) { - return arg2; - } - - if (arg2Val == null || arg2Val.isEmpty()) { - return arg1; - } - - final DateTime timestamp = (gran instanceof AllGranularity) - ? arg1.getTimestamp() - : gran.bucketStart(arg1.getTimestamp()); - - SelectResultValueBuilder builder = new SelectResultValueBuilder.MergeBuilder(timestamp, pagingSpec, descending); - - builder.addDimensions(mergeColumns(arg1.getValue().getDimensions(), arg2.getValue().getDimensions())); - builder.addMetrics(mergeColumns(arg1.getValue().getMetrics(), arg2.getValue().getMetrics())); - - for (EventHolder event : arg1Val) { - builder.addEntry(event); - } - - for (EventHolder event : arg2Val) { - builder.addEntry(event); - } - - return builder.build(); - } - - private Set mergeColumns(final Set arg1, final Set arg2) - { - if (arg1.isEmpty()) { - return arg2; - } - - if (arg2.isEmpty()) { - return arg1; - } - - if (arg1.equals(arg2)) { - return arg1; - } - - return Sets.union(arg1, arg2); - } -} diff --git a/processing/src/main/java/org/apache/druid/query/select/SelectQuery.java b/processing/src/main/java/org/apache/druid/query/select/SelectQuery.java index 7dee4056b63c..b3ef7189ef38 100644 --- a/processing/src/main/java/org/apache/druid/query/select/SelectQuery.java +++ b/processing/src/main/java/org/apache/druid/query/select/SelectQuery.java @@ -20,212 +20,151 @@ package org.apache.druid.query.select; import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.annotation.JsonTypeName; -import com.google.common.base.Preconditions; -import org.apache.druid.java.util.common.granularity.Granularities; +import com.google.common.collect.Ordering; import org.apache.druid.java.util.common.granularity.Granularity; -import org.apache.druid.query.BaseQuery; import org.apache.druid.query.DataSource; -import org.apache.druid.query.Druids; import org.apache.druid.query.Query; -import org.apache.druid.query.Result; -import org.apache.druid.query.dimension.DimensionSpec; +import org.apache.druid.query.QueryRunner; +import org.apache.druid.query.QuerySegmentWalker; import org.apache.druid.query.filter.DimFilter; import org.apache.druid.query.spec.QuerySegmentSpec; -import org.apache.druid.segment.VirtualColumns; +import org.joda.time.DateTimeZone; +import org.joda.time.Duration; +import org.joda.time.Interval; import java.util.List; import java.util.Map; -import java.util.Objects; -/** - */ -@JsonTypeName("select") -public class SelectQuery extends BaseQuery> +@Deprecated +public class SelectQuery implements Query { - private final DimFilter dimFilter; - private final List dimensions; - private final List metrics; - private final VirtualColumns virtualColumns; - private final PagingSpec pagingSpec; + static final String REMOVED_ERROR_MESSAGE = + "The 'select' query has been removed, use 'scan' instead. See " + + "https://druid.apache.org/docs/latest/querying/select-query.html for more details."; @JsonCreator - public SelectQuery( - @JsonProperty("dataSource") DataSource dataSource, - @JsonProperty("intervals") QuerySegmentSpec querySegmentSpec, - @JsonProperty("descending") boolean descending, - @JsonProperty("filter") DimFilter dimFilter, - @JsonProperty("granularity") Granularity granularity, - @JsonProperty("dimensions") List dimensions, - @JsonProperty("metrics") List metrics, - @JsonProperty("virtualColumns") VirtualColumns virtualColumns, - @JsonProperty("pagingSpec") PagingSpec pagingSpec, - @JsonProperty("context") Map context - ) + public SelectQuery() { - super(dataSource, querySegmentSpec, descending, context, Granularities.nullToAll(granularity)); - this.dimFilter = dimFilter; - this.dimensions = dimensions; - this.virtualColumns = VirtualColumns.nullToEmpty(virtualColumns); - this.metrics = metrics; - this.pagingSpec = pagingSpec; - - Preconditions.checkNotNull(pagingSpec, "must specify a pagingSpec"); - Preconditions.checkArgument(checkPagingSpec(pagingSpec, descending), "invalid pagingSpec"); + throw new RuntimeException(REMOVED_ERROR_MESSAGE); } - private boolean checkPagingSpec(PagingSpec pagingSpec, boolean descending) + @Override + public DataSource getDataSource() { - for (Integer value : pagingSpec.getPagingIdentifiers().values()) { - if (descending ^ (value < 0)) { - return false; - } - } - return pagingSpec.getThreshold() >= 0; + throw new RuntimeException(REMOVED_ERROR_MESSAGE); } @Override public boolean hasFilters() { - return dimFilter != null; + throw new RuntimeException(REMOVED_ERROR_MESSAGE); } @Override public DimFilter getFilter() { - return dimFilter; + throw new RuntimeException(REMOVED_ERROR_MESSAGE); } @Override public String getType() { - return Query.SELECT; + throw new RuntimeException(REMOVED_ERROR_MESSAGE); } - @JsonProperty("filter") - public DimFilter getDimensionsFilter() + @Override + public QueryRunner getRunner(QuerySegmentWalker walker) { - return dimFilter; + throw new RuntimeException(REMOVED_ERROR_MESSAGE); } - @JsonProperty - public List getDimensions() + @Override + public List getIntervals() { - return dimensions; + throw new RuntimeException(REMOVED_ERROR_MESSAGE); } - @JsonProperty - public PagingSpec getPagingSpec() + @Override + public Duration getDuration() { - return pagingSpec; + throw new RuntimeException(REMOVED_ERROR_MESSAGE); } - @JsonProperty - public List getMetrics() + @Override + public Granularity getGranularity() { - return metrics; + throw new RuntimeException(REMOVED_ERROR_MESSAGE); } - @JsonProperty - public VirtualColumns getVirtualColumns() + @Override + public DateTimeZone getTimezone() { - return virtualColumns; + throw new RuntimeException(REMOVED_ERROR_MESSAGE); } - public PagingOffset getPagingOffset(String identifier) + @Override + public Map getContext() { - return pagingSpec.getOffset(identifier, isDescending()); + throw new RuntimeException(REMOVED_ERROR_MESSAGE); } @Override - public SelectQuery withQuerySegmentSpec(QuerySegmentSpec querySegmentSpec) + public ContextType getContextValue(String key) { - return Druids.SelectQueryBuilder.copy(this).intervals(querySegmentSpec).build(); + throw new RuntimeException(REMOVED_ERROR_MESSAGE); } @Override - public Query> withDataSource(DataSource dataSource) + public ContextType getContextValue(String key, ContextType defaultValue) { - return Druids.SelectQueryBuilder.copy(this).dataSource(dataSource).build(); + throw new RuntimeException(REMOVED_ERROR_MESSAGE); } @Override - public SelectQuery withOverriddenContext(Map contextOverrides) + public boolean getContextBoolean(String key, boolean defaultValue) { - Map newContext = computeOverriddenContext(getContext(), contextOverrides); - return Druids.SelectQueryBuilder.copy(this).context(newContext).build(); + throw new RuntimeException(REMOVED_ERROR_MESSAGE); } - public SelectQuery withPagingSpec(PagingSpec pagingSpec) + @Override + public boolean isDescending() { - return Druids.SelectQueryBuilder.copy(this).pagingSpec(pagingSpec).build(); + throw new RuntimeException(REMOVED_ERROR_MESSAGE); } - public SelectQuery withDimFilter(DimFilter dimFilter) + @Override + public Ordering getResultOrdering() { - return Druids.SelectQueryBuilder.copy(this).filters(dimFilter).build(); + throw new RuntimeException(REMOVED_ERROR_MESSAGE); } @Override - public String toString() + public Query withOverriddenContext(Map contextOverride) { - return "SelectQuery{" + - "dataSource='" + getDataSource() + '\'' + - ", querySegmentSpec=" + getQuerySegmentSpec() + - ", descending=" + isDescending() + - ", dimFilter=" + dimFilter + - ", granularity=" + getGranularity() + - ", dimensions=" + dimensions + - ", metrics=" + metrics + - ", virtualColumns=" + virtualColumns + - ", pagingSpec=" + pagingSpec + - '}'; + throw new RuntimeException(REMOVED_ERROR_MESSAGE); } @Override - public boolean equals(Object o) + public Query withQuerySegmentSpec(QuerySegmentSpec spec) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - if (!super.equals(o)) { - return false; - } - - SelectQuery that = (SelectQuery) o; + throw new RuntimeException(REMOVED_ERROR_MESSAGE); + } - if (!Objects.equals(dimFilter, that.dimFilter)) { - return false; - } - if (!Objects.equals(dimensions, that.dimensions)) { - return false; - } - if (!Objects.equals(metrics, that.metrics)) { - return false; - } - if (!Objects.equals(virtualColumns, that.virtualColumns)) { - return false; - } - if (!Objects.equals(pagingSpec, that.pagingSpec)) { - return false; - } + @Override + public Query withId(String id) + { + throw new RuntimeException(REMOVED_ERROR_MESSAGE); + } - return true; + @Override + public String getId() + { + throw new RuntimeException(REMOVED_ERROR_MESSAGE); } @Override - public int hashCode() + public Query withDataSource(DataSource dataSource) { - int result = super.hashCode(); - result = 31 * result + (dimFilter != null ? dimFilter.hashCode() : 0); - result = 31 * result + (dimensions != null ? dimensions.hashCode() : 0); - result = 31 * result + (metrics != null ? metrics.hashCode() : 0); - result = 31 * result + (virtualColumns != null ? virtualColumns.hashCode() : 0); - result = 31 * result + (pagingSpec != null ? pagingSpec.hashCode() : 0); - return result; + throw new RuntimeException(REMOVED_ERROR_MESSAGE); } } diff --git a/processing/src/main/java/org/apache/druid/query/select/SelectQueryConfig.java b/processing/src/main/java/org/apache/druid/query/select/SelectQueryConfig.java deleted file mode 100644 index 386fc09387ee..000000000000 --- a/processing/src/main/java/org/apache/druid/query/select/SelectQueryConfig.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.druid.query.select; - -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonProperty; - -public class SelectQueryConfig -{ - public static String ENABLE_FROM_NEXT_DEFAULT = "enableFromNextDefault"; - - @JsonProperty - private boolean enableFromNextDefault = true; - - @JsonCreator - public SelectQueryConfig( - @JsonProperty("enableFromNextDefault") Boolean enableFromNextDefault - ) - { - if (enableFromNextDefault != null) { - this.enableFromNextDefault = enableFromNextDefault.booleanValue(); - } - } - - public boolean getEnableFromNextDefault() - { - return enableFromNextDefault; - } - - public void setEnableFromNextDefault(boolean enableFromNextDefault) - { - this.enableFromNextDefault = enableFromNextDefault; - } -} diff --git a/processing/src/main/java/org/apache/druid/query/select/SelectQueryEngine.java b/processing/src/main/java/org/apache/druid/query/select/SelectQueryEngine.java deleted file mode 100644 index 848eace2a267..000000000000 --- a/processing/src/main/java/org/apache/druid/query/select/SelectQueryEngine.java +++ /dev/null @@ -1,321 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.druid.query.select; - -import com.google.common.base.Function; -import com.google.common.base.Preconditions; -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; -import org.apache.druid.java.util.common.DateTimes; -import org.apache.druid.java.util.common.IAE; -import org.apache.druid.java.util.common.ISE; -import org.apache.druid.java.util.common.guava.Sequence; -import org.apache.druid.query.ColumnSelectorPlus; -import org.apache.druid.query.QueryRunnerHelper; -import org.apache.druid.query.Result; -import org.apache.druid.query.dimension.ColumnSelectorStrategy; -import org.apache.druid.query.dimension.ColumnSelectorStrategyFactory; -import org.apache.druid.query.dimension.DefaultDimensionSpec; -import org.apache.druid.query.dimension.DimensionSpec; -import org.apache.druid.query.filter.Filter; -import org.apache.druid.segment.BaseDoubleColumnValueSelector; -import org.apache.druid.segment.BaseFloatColumnValueSelector; -import org.apache.druid.segment.BaseLongColumnValueSelector; -import org.apache.druid.segment.BaseObjectColumnValueSelector; -import org.apache.druid.segment.ColumnValueSelector; -import org.apache.druid.segment.Cursor; -import org.apache.druid.segment.DimensionHandlerUtils; -import org.apache.druid.segment.DimensionSelector; -import org.apache.druid.segment.Segment; -import org.apache.druid.segment.StorageAdapter; -import org.apache.druid.segment.column.ColumnCapabilities; -import org.apache.druid.segment.column.ColumnHolder; -import org.apache.druid.segment.column.ValueType; -import org.apache.druid.segment.data.IndexedInts; -import org.apache.druid.segment.filter.Filters; -import org.joda.time.Interval; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -/** - */ -public class SelectQueryEngine -{ - private static final SelectStrategyFactory STRATEGY_FACTORY = new SelectStrategyFactory(); - - public static class SelectStrategyFactory implements ColumnSelectorStrategyFactory - { - @Override - public SelectColumnSelectorStrategy makeColumnSelectorStrategy( - ColumnCapabilities capabilities, - ColumnValueSelector selector - ) - { - ValueType type = capabilities.getType(); - switch (type) { - case STRING: - return new StringSelectColumnSelectorStrategy(); - case LONG: - return new LongSelectColumnSelectorStrategy(); - case FLOAT: - return new FloatSelectColumnSelectorStrategy(); - case DOUBLE: - return new DoubleSelectColumnSelectorStrategy(); - default: - throw new IAE("Cannot create query type helper from invalid type [%s]", type); - } - } - } - - public interface SelectColumnSelectorStrategy extends ColumnSelectorStrategy - { - /** - * Read the current row from selector and add the row values for a dimension to the result map. - * - * Multi-valued rows should be added to the result as a List, single value rows should be added as a single object. - * - * @param outputName Output name for this dimension in the select query being served - * @param selector Dimension value selector - * @param resultMap Row value map for the current row being retrieved by the select query - */ - void addRowValuesToSelectResult( - String outputName, - ValueSelectorType selector, - Map resultMap - ); - } - - public static class StringSelectColumnSelectorStrategy implements SelectColumnSelectorStrategy - { - @Override - public void addRowValuesToSelectResult(String outputName, DimensionSelector selector, Map resultMap) - { - final IndexedInts row = selector.getRow(); - int rowSize = row.size(); - if (rowSize == 0) { - resultMap.put(outputName, null); - } else if (rowSize == 1) { - final String dimVal = selector.lookupName(row.get(0)); - resultMap.put(outputName, dimVal); - } else { - List dimVals = new ArrayList<>(rowSize); - for (int i = 0; i < rowSize; ++i) { - dimVals.add(selector.lookupName(row.get(i))); - } - resultMap.put(outputName, dimVals); - } - } - } - - public static class LongSelectColumnSelectorStrategy - implements SelectColumnSelectorStrategy - { - - @Override - public void addRowValuesToSelectResult( - String outputName, - BaseLongColumnValueSelector selector, - Map resultMap - ) - { - if (selector == null) { - resultMap.put(outputName, null); - } else { - resultMap.put(outputName, selector.getLong()); - } - } - } - - public static class FloatSelectColumnSelectorStrategy - implements SelectColumnSelectorStrategy - { - @Override - public void addRowValuesToSelectResult( - String outputName, - BaseFloatColumnValueSelector selector, - Map resultMap - ) - { - if (selector == null) { - resultMap.put(outputName, null); - } else { - resultMap.put(outputName, selector.getFloat()); - } - } - } - public static class DoubleSelectColumnSelectorStrategy - implements SelectColumnSelectorStrategy - { - @Override - public void addRowValuesToSelectResult( - String outputName, - BaseDoubleColumnValueSelector selector, - Map resultMap - ) - { - if (selector == null) { - resultMap.put(outputName, null); - } else { - resultMap.put(outputName, selector.getDouble()); - } - } - } - - public Sequence> process(final SelectQuery query, final Segment segment) - { - final StorageAdapter adapter = segment.asStorageAdapter(); - - if (adapter == null) { - throw new ISE( - "Null storage adapter found. Probably trying to issue a query against a segment being memory unmapped." - ); - } - - Preconditions.checkArgument( - query.getDataSource().getNames().size() == 1, - "At the point where this code is called, only one data source should exist. Data sources: %s", - query.getDataSource().getNames() - ); - - final Iterable dims; - if (query.getDimensions() == null || query.getDimensions().isEmpty()) { - dims = DefaultDimensionSpec.toSpec(adapter.getAvailableDimensions()); - } else { - dims = query.getDimensions(); - } - - final Iterable metrics; - if (query.getMetrics() == null || query.getMetrics().isEmpty()) { - metrics = adapter.getAvailableMetrics(); - } else { - metrics = query.getMetrics(); - } - List intervals = query.getQuerySegmentSpec().getIntervals(); - Preconditions.checkArgument(intervals.size() == 1, "Can only handle a single interval, got[%s]", intervals); - - // should be rewritten with given interval - final String segmentId = segment.getId().withInterval(intervals.get(0)).toString(); - - final Filter filter = Filters.convertToCNFFromQueryContext(query, Filters.toFilter(query.getDimensionsFilter())); - - return QueryRunnerHelper.makeCursorBasedQuery( - adapter, - query.getQuerySegmentSpec().getIntervals(), - filter, - query.getVirtualColumns(), - query.isDescending(), - query.getGranularity(), - new Function>() - { - @Override - public Result apply(Cursor cursor) - { - final SelectResultValueBuilder builder = new SelectResultValueBuilder( - cursor.getTime(), - query.getPagingSpec(), - query.isDescending() - ); - - final BaseLongColumnValueSelector timestampColumnSelector = - cursor.getColumnSelectorFactory().makeColumnValueSelector(ColumnHolder.TIME_COLUMN_NAME); - - final List> selectorPlusList = Arrays.asList( - DimensionHandlerUtils.createColumnSelectorPluses( - STRATEGY_FACTORY, - Lists.newArrayList(dims), - cursor.getColumnSelectorFactory() - ) - ); - - for (DimensionSpec dimSpec : dims) { - builder.addDimension(dimSpec.getOutputName()); - } - - final Map> metSelectors = new HashMap<>(); - for (String metric : metrics) { - final BaseObjectColumnValueSelector metricSelector = - cursor.getColumnSelectorFactory().makeColumnValueSelector(metric); - metSelectors.put(metric, metricSelector); - builder.addMetric(metric); - } - - final PagingOffset offset = query.getPagingOffset(segmentId); - - cursor.advanceTo(offset.startDelta()); - - int lastOffset = offset.startOffset(); - for (; !cursor.isDone() && offset.hasNext(); cursor.advance(), offset.next()) { - final Map theEvent = singleEvent( - EventHolder.TIMESTAMP_KEY, - timestampColumnSelector, - selectorPlusList, - metSelectors - ); - - builder.addEntry( - new EventHolder( - segmentId, - lastOffset = offset.current(), - theEvent - ) - ); - } - - builder.finished(segmentId, lastOffset); - - return builder.build(); - } - } - ); - } - - public static Map singleEvent( - String timestampKey, - BaseLongColumnValueSelector timestampColumnSelector, - List> selectorPlusList, - Map> metSelectors - ) - { - final Map theEvent = Maps.newLinkedHashMap(); - theEvent.put(timestampKey, DateTimes.utc(timestampColumnSelector.getLong())); - - for (ColumnSelectorPlus selectorPlus : selectorPlusList) { - selectorPlus - .getColumnSelectorStrategy() - .addRowValuesToSelectResult(selectorPlus.getOutputName(), selectorPlus.getSelector(), theEvent); - } - - for (Map.Entry> metSelector : metSelectors.entrySet()) { - final String metric = metSelector.getKey(); - final BaseObjectColumnValueSelector selector = metSelector.getValue(); - - if (selector == null) { - theEvent.put(metric, null); - } else { - theEvent.put(metric, selector.getObject()); - } - } - return theEvent; - } -} diff --git a/processing/src/main/java/org/apache/druid/query/select/SelectQueryMetrics.java b/processing/src/main/java/org/apache/druid/query/select/SelectQueryMetrics.java deleted file mode 100644 index 790c01631826..000000000000 --- a/processing/src/main/java/org/apache/druid/query/select/SelectQueryMetrics.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.druid.query.select; - -import org.apache.druid.query.QueryMetrics; - -public interface SelectQueryMetrics extends QueryMetrics -{ - /** - * Sets the granularity of {@link SelectQuery#getGranularity()} of the given query as dimension. - * - * This method is going to be used in "full" metrics impl, - * see https://github.com/apache/incubator-druid/pull/4570#issuecomment-319458229 - */ - @SuppressWarnings("unused") - void granularity(SelectQuery query); -} diff --git a/processing/src/main/java/org/apache/druid/query/select/SelectQueryMetricsFactory.java b/processing/src/main/java/org/apache/druid/query/select/SelectQueryMetricsFactory.java deleted file mode 100644 index 6b0d7fdb3768..000000000000 --- a/processing/src/main/java/org/apache/druid/query/select/SelectQueryMetricsFactory.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.druid.query.select; - -/** - * Implementations could be injected using - * - * PolyBind - * .optionBinder(binder, Key.get(SelectQueryMetricsFactory.class)) - * .addBinding("myCustomSelectQueryMetricsFactory") - * .to(MyCustomSelectQueryMetricsFactory.class); - * - * And then setting property: - * druid.query.select.queryMetricsFactory=myCustomSelectQueryMetricsFactory - */ -public interface SelectQueryMetricsFactory -{ - SelectQueryMetrics makeMetrics(SelectQuery selectQuery); -} diff --git a/processing/src/main/java/org/apache/druid/query/select/SelectQueryQueryToolChest.java b/processing/src/main/java/org/apache/druid/query/select/SelectQueryQueryToolChest.java deleted file mode 100644 index 7ff4dfd94812..000000000000 --- a/processing/src/main/java/org/apache/druid/query/select/SelectQueryQueryToolChest.java +++ /dev/null @@ -1,403 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.druid.query.select; - -import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.common.base.Function; -import com.google.common.base.Functions; -import com.google.common.base.Preconditions; -import com.google.common.collect.Iterables; -import com.google.common.collect.Lists; -import com.google.inject.Inject; -import org.apache.druid.java.util.common.StringUtils; -import org.apache.druid.java.util.common.granularity.Granularity; -import org.apache.druid.java.util.common.guava.Comparators; -import org.apache.druid.java.util.common.guava.Sequence; -import org.apache.druid.query.CacheStrategy; -import org.apache.druid.query.IntervalChunkingQueryRunnerDecorator; -import org.apache.druid.query.Query; -import org.apache.druid.query.QueryPlus; -import org.apache.druid.query.QueryRunner; -import org.apache.druid.query.QueryToolChest; -import org.apache.druid.query.Result; -import org.apache.druid.query.ResultGranularTimestampComparator; -import org.apache.druid.query.aggregation.MetricManipulationFn; -import org.apache.druid.query.context.ResponseContext; -import org.apache.druid.query.dimension.DimensionSpec; -import org.apache.druid.query.filter.DimFilter; -import org.apache.druid.timeline.LogicalSegment; -import org.apache.druid.timeline.SegmentId; -import org.joda.time.DateTime; -import org.joda.time.Interval; - -import java.nio.ByteBuffer; -import java.util.Arrays; -import java.util.Collections; -import java.util.Comparator; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.TreeMap; -import java.util.TreeSet; -import java.util.function.BinaryOperator; - -/** - */ -public class SelectQueryQueryToolChest extends QueryToolChest, SelectQuery> -{ - private static final byte SELECT_QUERY = 0x16; - private static final TypeReference OBJECT_TYPE_REFERENCE = - new TypeReference() - { - }; - private static final TypeReference> TYPE_REFERENCE = - new TypeReference>() - { - }; - - private final ObjectMapper jsonMapper; - @Deprecated - private final IntervalChunkingQueryRunnerDecorator intervalChunkingQueryRunnerDecorator; - private final SelectQueryMetricsFactory queryMetricsFactory; - - public SelectQueryQueryToolChest( - ObjectMapper jsonMapper, - IntervalChunkingQueryRunnerDecorator intervalChunkingQueryRunnerDecorator - ) - { - this(jsonMapper, intervalChunkingQueryRunnerDecorator, DefaultSelectQueryMetricsFactory.instance()); - } - - @Inject - public SelectQueryQueryToolChest( - ObjectMapper jsonMapper, - IntervalChunkingQueryRunnerDecorator intervalChunkingQueryRunnerDecorator, - SelectQueryMetricsFactory queryMetricsFactory - ) - { - this.jsonMapper = jsonMapper; - this.intervalChunkingQueryRunnerDecorator = intervalChunkingQueryRunnerDecorator; - this.queryMetricsFactory = queryMetricsFactory; - } - - @Override - public BinaryOperator> createMergeFn( - Query> query - ) - { - final SelectQuery selectQuery = (SelectQuery) query; - return new SelectBinaryFn(selectQuery.getGranularity(), selectQuery.getPagingSpec(), selectQuery.isDescending()); - } - - @Override - public Comparator> createResultComparator(Query> query) - { - return ResultGranularTimestampComparator.create(query.getGranularity(), query.isDescending()); - } - - @Override - public SelectQueryMetrics makeMetrics(SelectQuery query) - { - SelectQueryMetrics queryMetrics = queryMetricsFactory.makeMetrics(query); - queryMetrics.query(query); - return queryMetrics; - } - - @Override - public Function, Result> makePreComputeManipulatorFn( - final SelectQuery query, - final MetricManipulationFn fn - ) - { - return Functions.identity(); - } - - @Override - public TypeReference> getResultTypeReference() - { - return TYPE_REFERENCE; - } - - @Override - public CacheStrategy, Object, SelectQuery> getCacheStrategy(final SelectQuery query) - { - - return new CacheStrategy, Object, SelectQuery>() - { - private final List dimensionSpecs = - query.getDimensions() != null ? query.getDimensions() : Collections.emptyList(); - private final List dimOutputNames = dimensionSpecs.size() > 0 ? - Lists.transform(dimensionSpecs, DimensionSpec::getOutputName) : Collections.emptyList(); - - @Override - public boolean isCacheable(SelectQuery query, boolean willMergeRunners) - { - return true; - } - - @Override - public byte[] computeCacheKey(SelectQuery query) - { - final DimFilter dimFilter = query.getDimensionsFilter(); - final byte[] filterBytes = dimFilter == null ? new byte[]{} : dimFilter.getCacheKey(); - final byte[] granularityBytes = query.getGranularity().getCacheKey(); - - final List dimensionSpecs = - query.getDimensions() != null ? query.getDimensions() : Collections.emptyList(); - final byte[][] dimensionsBytes = new byte[dimensionSpecs.size()][]; - int dimensionsBytesSize = 0; - int index = 0; - for (DimensionSpec dimension : dimensionSpecs) { - dimensionsBytes[index] = dimension.getCacheKey(); - dimensionsBytesSize += dimensionsBytes[index].length; - ++index; - } - - final Set metrics = new TreeSet<>(); - if (query.getMetrics() != null) { - metrics.addAll(query.getMetrics()); - } - - final byte[][] metricBytes = new byte[metrics.size()][]; - int metricBytesSize = 0; - index = 0; - for (String metric : metrics) { - metricBytes[index] = StringUtils.toUtf8(metric); - metricBytesSize += metricBytes[index].length; - ++index; - } - - final byte[] virtualColumnsCacheKey = query.getVirtualColumns().getCacheKey(); - final byte isDescendingByte = query.isDescending() ? (byte) 1 : 0; - - final ByteBuffer queryCacheKey = ByteBuffer - .allocate( - 2 - + granularityBytes.length - + filterBytes.length - + query.getPagingSpec().getCacheKey().length - + dimensionsBytesSize - + metricBytesSize - + virtualColumnsCacheKey.length - ) - .put(SELECT_QUERY) - .put(granularityBytes) - .put(filterBytes) - .put(query.getPagingSpec().getCacheKey()) - .put(isDescendingByte); - - for (byte[] dimensionsByte : dimensionsBytes) { - queryCacheKey.put(dimensionsByte); - } - - for (byte[] metricByte : metricBytes) { - queryCacheKey.put(metricByte); - } - - queryCacheKey.put(virtualColumnsCacheKey); - - return queryCacheKey.array(); - } - - @Override - public byte[] computeResultLevelCacheKey(SelectQuery query) - { - return computeCacheKey(query); - } - - @Override - public TypeReference getCacheObjectClazz() - { - return OBJECT_TYPE_REFERENCE; - } - - @Override - public Function, Object> prepareForCache(boolean isResultLevelCache) - { - return new Function, Object>() - { - @Override - public Object apply(final Result input) - { - if (!dimOutputNames.isEmpty()) { - return Arrays.asList( - input.getTimestamp().getMillis(), - input.getValue().getPagingIdentifiers(), - input.getValue().getDimensions(), - input.getValue().getMetrics(), - input.getValue().getEvents(), - dimOutputNames - ); - } - return Arrays.asList( - input.getTimestamp().getMillis(), - input.getValue().getPagingIdentifiers(), - input.getValue().getDimensions(), - input.getValue().getMetrics(), - input.getValue().getEvents() - ); - } - }; - } - - @Override - public Function> pullFromCache(boolean isResultLevelCache) - { - return new Function>() - { - private final Granularity granularity = query.getGranularity(); - - @Override - public Result apply(Object input) - { - List results = (List) input; - Iterator resultIter = results.iterator(); - - DateTime timestamp = granularity.toDateTime(((Number) resultIter.next()).longValue()); - - Map pageIdentifier = jsonMapper.convertValue( - resultIter.next(), - new TypeReference>() {} - ); - Set dimensionSet = jsonMapper.convertValue(resultIter.next(), new TypeReference>() {}); - Set metricSet = jsonMapper.convertValue(resultIter.next(), new TypeReference>() {}); - List eventHolders = jsonMapper.convertValue( - resultIter.next(), - new TypeReference>() {} - ); - // check the condition that outputName of cached result should be updated - if (resultIter.hasNext()) { - List cachedOutputNames = (List) resultIter.next(); - Preconditions.checkArgument( - cachedOutputNames.size() == dimOutputNames.size(), - "Cache hit but different number of dimensions??" - ); - for (int idx = 0; idx < dimOutputNames.size(); idx++) { - if (!cachedOutputNames.get(idx).equals(dimOutputNames.get(idx))) { - // rename outputName in the EventHolder - for (EventHolder eventHolder : eventHolders) { - Object obj = eventHolder.getEvent().remove(cachedOutputNames.get(idx)); - if (obj != null) { - eventHolder.getEvent().put(dimOutputNames.get(idx), obj); - } - } - } - } - } - - return new Result<>( - timestamp, - new SelectResultValue(pageIdentifier, dimensionSet, metricSet, eventHolders) - ); - } - }; - } - }; - } - - @Override - public QueryRunner> preMergeQueryDecoration(final QueryRunner> runner) - { - return intervalChunkingQueryRunnerDecorator.decorate( - new QueryRunner>() - { - @Override - public Sequence> run( - QueryPlus> queryPlus, - ResponseContext responseContext - ) - { - SelectQuery selectQuery = (SelectQuery) queryPlus.getQuery(); - if (selectQuery.getDimensionsFilter() != null) { - selectQuery = selectQuery.withDimFilter(selectQuery.getDimensionsFilter().optimize()); - queryPlus = queryPlus.withQuery(selectQuery); - } - return runner.run(queryPlus, responseContext); - } - }, this); - } - - @Override - public List filterSegments(SelectQuery query, List segments) - { - // at the point where this code is called, only one datasource should exist. - final String dataSource = Iterables.getOnlyElement(query.getDataSource().getNames()); - - PagingSpec pagingSpec = query.getPagingSpec(); - Map paging = pagingSpec.getPagingIdentifiers(); - if (paging == null || paging.isEmpty()) { - return segments; - } - - final Granularity granularity = query.getGranularity(); - - TreeMap granularThresholds = new TreeMap<>(); - - // A paged select query using a UnionDataSource will return pagingIdentifiers from segments in more than one - // dataSource which confuses subsequent queries and causes a failure. To avoid this, filter only the paging keys - // that are applicable to this dataSource so that each dataSource in a union query gets the appropriate keys. - paging - .keySet() - .stream() - .filter(identifier -> SegmentId.tryParse(dataSource, identifier) != null) - .map(SegmentId.makeIntervalExtractor(dataSource)) - .sorted(query.isDescending() ? Comparators.intervalsByEndThenStart() - : Comparators.intervalsByStartThenEnd()) - .forEach(interval -> { - if (query.isDescending()) { - long granularEnd = granularity.bucketStart(interval.getEnd()).getMillis(); - Long currentEnd = granularThresholds.get(granularEnd); - if (currentEnd == null || interval.getEndMillis() > currentEnd) { - granularThresholds.put(granularEnd, interval.getEndMillis()); - } - } else { - long granularStart = granularity.bucketStart(interval.getStart()).getMillis(); - Long currentStart = granularThresholds.get(granularStart); - if (currentStart == null || interval.getStartMillis() < currentStart) { - granularThresholds.put(granularStart, interval.getStartMillis()); - } - } - }); - - List queryIntervals = Lists.newArrayList(segments); - - Iterator it = queryIntervals.iterator(); - if (query.isDescending()) { - while (it.hasNext()) { - Interval interval = it.next().getInterval(); - Map.Entry ceiling = granularThresholds.ceilingEntry(granularity.bucketStart(interval.getEnd()).getMillis()); - if (ceiling == null || interval.getStartMillis() >= ceiling.getValue()) { - it.remove(); - } - } - } else { - while (it.hasNext()) { - Interval interval = it.next().getInterval(); - Map.Entry floor = granularThresholds.floorEntry(granularity.bucketStart(interval.getStart()).getMillis()); - if (floor == null || interval.getEndMillis() <= floor.getValue()) { - it.remove(); - } - } - } - return queryIntervals; - } -} diff --git a/processing/src/main/java/org/apache/druid/query/select/SelectQueryRunnerFactory.java b/processing/src/main/java/org/apache/druid/query/select/SelectQueryRunnerFactory.java deleted file mode 100644 index 9145f09fee7a..000000000000 --- a/processing/src/main/java/org/apache/druid/query/select/SelectQueryRunnerFactory.java +++ /dev/null @@ -1,105 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.druid.query.select; - -import com.google.inject.Inject; -import org.apache.druid.java.util.common.ISE; -import org.apache.druid.java.util.common.guava.Sequence; -import org.apache.druid.query.ChainedExecutionQueryRunner; -import org.apache.druid.query.Query; -import org.apache.druid.query.QueryPlus; -import org.apache.druid.query.QueryRunner; -import org.apache.druid.query.QueryRunnerFactory; -import org.apache.druid.query.QueryToolChest; -import org.apache.druid.query.QueryWatcher; -import org.apache.druid.query.Result; -import org.apache.druid.query.context.ResponseContext; -import org.apache.druid.segment.Segment; - -import java.util.concurrent.ExecutorService; - -/** - */ -public class SelectQueryRunnerFactory - implements QueryRunnerFactory, SelectQuery> -{ - private final SelectQueryQueryToolChest toolChest; - private final SelectQueryEngine engine; - private final QueryWatcher queryWatcher; - - @Inject - public SelectQueryRunnerFactory( - SelectQueryQueryToolChest toolChest, - SelectQueryEngine engine, - QueryWatcher queryWatcher - ) - { - this.toolChest = toolChest; - this.engine = engine; - this.queryWatcher = queryWatcher; - } - - @Override - public QueryRunner> createRunner(final Segment segment) - { - return new SelectQueryRunner(engine, segment); - } - - @Override - public QueryRunner> mergeRunners( - ExecutorService queryExecutor, - Iterable>> queryRunners - ) - { - return new ChainedExecutionQueryRunner<>(queryExecutor, queryWatcher, queryRunners); - } - - @Override - public QueryToolChest, SelectQuery> getToolchest() - { - return toolChest; - } - - private static class SelectQueryRunner implements QueryRunner> - { - private final SelectQueryEngine engine; - private final Segment segment; - - private SelectQueryRunner(SelectQueryEngine engine, Segment segment) - { - this.engine = engine; - this.segment = segment; - } - - @Override - public Sequence> run( - QueryPlus> queryPlus, - ResponseContext responseContext - ) - { - Query> input = queryPlus.getQuery(); - if (!(input instanceof SelectQuery)) { - throw new ISE("Got a [%s] which isn't a %s", input.getClass(), SelectQuery.class); - } - - return engine.process((SelectQuery) input, segment); - } - } -} diff --git a/processing/src/main/java/org/apache/druid/query/select/SelectResultValue.java b/processing/src/main/java/org/apache/druid/query/select/SelectResultValue.java deleted file mode 100644 index fb138664137a..000000000000 --- a/processing/src/main/java/org/apache/druid/query/select/SelectResultValue.java +++ /dev/null @@ -1,136 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.druid.query.select; - -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonProperty; - -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Set; - -/** - */ -public class SelectResultValue implements Iterable -{ - private final Map pagingIdentifiers; - private final Set dimensions; - private final Set metrics; - private final List events; - - @JsonCreator - public SelectResultValue( - @JsonProperty("pagingIdentifiers") Map pagingIdentifiers, - @JsonProperty("dimensions") Set dimensions, - @JsonProperty("metrics") Set metrics, - @JsonProperty("events") List events - ) - { - this.pagingIdentifiers = pagingIdentifiers; - this.dimensions = dimensions; - this.metrics = metrics; - this.events = events; - } - - @JsonProperty - public Map getPagingIdentifiers() - { - return pagingIdentifiers; - } - - @JsonProperty - public Set getDimensions() - { - return dimensions; - } - - @JsonProperty - public Set getMetrics() - { - return metrics; - } - - @JsonProperty - public List getEvents() - { - return events; - } - - @Override - public Iterator iterator() - { - return events.iterator(); - } - - @Override - public boolean equals(Object o) - { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - - SelectResultValue that = (SelectResultValue) o; - - if (events != null ? !events.equals(that.events) : that.events != null) { - return false; - } - - if (dimensions != null ? !dimensions.equals(that.dimensions) : that.dimensions != null) { - return false; - } - - if (metrics != null ? !metrics.equals(that.metrics) : that.metrics != null) { - return false; - } - - if (pagingIdentifiers != null - ? !pagingIdentifiers.equals(that.pagingIdentifiers) - : that.pagingIdentifiers != null) { - return false; - } - - return true; - } - - @Override - public int hashCode() - { - int result = pagingIdentifiers != null ? pagingIdentifiers.hashCode() : 0; - result = 31 * result + (dimensions != null ? dimensions.hashCode() : 0); - result = 31 * result + (metrics != null ? metrics.hashCode() : 0); - result = 31 * result + (events != null ? events.hashCode() : 0); - return result; - } - - @Override - public String toString() - { - return "SelectResultValue{" + - "pagingIdentifiers=" + pagingIdentifiers + - ", dimensions=" + dimensions + - ", metrics=" + metrics + - ", events=" + events + - '}'; - } -} diff --git a/processing/src/main/java/org/apache/druid/query/select/SelectResultValueBuilder.java b/processing/src/main/java/org/apache/druid/query/select/SelectResultValueBuilder.java deleted file mode 100644 index 646349af01d7..000000000000 --- a/processing/src/main/java/org/apache/druid/query/select/SelectResultValueBuilder.java +++ /dev/null @@ -1,153 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.druid.query.select; - -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; -import com.google.common.collect.MinMaxPriorityQueue; -import com.google.common.primitives.Longs; -import org.apache.druid.query.Result; -import org.joda.time.DateTime; - -import java.util.ArrayDeque; -import java.util.Comparator; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Queue; -import java.util.Set; - -/** - */ -public class SelectResultValueBuilder -{ - private static final Comparator COMPARATOR = new Comparator() - { - @Override - public int compare(EventHolder o1, EventHolder o2) - { - int retVal = Longs.compare(o1.getTimestamp().getMillis(), o2.getTimestamp().getMillis()); - - if (retVal == 0) { - retVal = o1.getSegmentId().compareTo(o2.getSegmentId()); - } - - if (retVal == 0) { - retVal = Integer.compare(o1.getOffset(), o2.getOffset()); - } - - return retVal; - } - }; - - protected final DateTime timestamp; - protected final PagingSpec pagingSpec; - protected final boolean descending; - protected Set dimensions; - protected Set metrics; - - protected final Queue pQueue; - protected final Map pagingIdentifiers; - - public SelectResultValueBuilder(DateTime timestamp, PagingSpec pagingSpec, boolean descending) - { - this.timestamp = timestamp; - this.pagingSpec = pagingSpec; - this.descending = descending; - this.dimensions = new HashSet<>(); - this.metrics = new HashSet<>(); - this.pagingIdentifiers = Maps.newLinkedHashMap(); - this.pQueue = instantiatePQueue(); - } - - public void addEntry(EventHolder event) - { - pQueue.add(event); - } - - public void finished(String segmentId, int lastOffset) - { - pagingIdentifiers.put(segmentId, lastOffset); - } - - public void addDimension(String dimension) - { - dimensions.add(dimension); - } - - public void addDimensions(Set dimensions) - { - this.dimensions.addAll(dimensions); - } - - public void addMetric(String metric) - { - metrics.add(metric); - } - - public void addMetrics(Set metrics) - { - this.metrics.addAll(metrics); - } - - public Result build() - { - return new Result<>(timestamp, new SelectResultValue(pagingIdentifiers, dimensions, metrics, getEventHolders())); - } - - protected List getEventHolders() - { - return Lists.newArrayList(pQueue); - } - - protected Queue instantiatePQueue() - { - return new ArrayDeque<>(); - } - - public static class MergeBuilder extends SelectResultValueBuilder - { - public MergeBuilder(DateTime timestamp, PagingSpec pagingSpec, boolean descending) - { - super(timestamp, pagingSpec, descending); - } - - @Override - protected Queue instantiatePQueue() - { - int threshold = pagingSpec.getThreshold(); - return MinMaxPriorityQueue.orderedBy(descending ? COMPARATOR.reversed() : COMPARATOR) - .maximumSize(threshold > 0 ? threshold : Integer.MAX_VALUE) - .create(); - } - - @Override - protected List getEventHolders() - { - final List values = Lists.newArrayListWithCapacity(pQueue.size()); - while (!pQueue.isEmpty()) { - EventHolder event = pQueue.remove(); - pagingIdentifiers.put(event.getSegmentId(), event.getOffset()); - values.add(event); - } - return values; - } - } -} diff --git a/processing/src/main/java/org/apache/druid/segment/Cursor.java b/processing/src/main/java/org/apache/druid/segment/Cursor.java index 645caee895f7..925c7387e534 100644 --- a/processing/src/main/java/org/apache/druid/segment/Cursor.java +++ b/processing/src/main/java/org/apache/druid/segment/Cursor.java @@ -39,7 +39,6 @@ public interface Cursor DateTime getTime(); void advance(); void advanceUninterruptibly(); - void advanceTo(int offset); boolean isDone(); boolean isDoneOrInterrupted(); void reset(); diff --git a/processing/src/main/java/org/apache/druid/segment/QueryableIndexCursorSequenceBuilder.java b/processing/src/main/java/org/apache/druid/segment/QueryableIndexCursorSequenceBuilder.java index 9855e62797c7..dfd2fa624a97 100644 --- a/processing/src/main/java/org/apache/druid/segment/QueryableIndexCursorSequenceBuilder.java +++ b/processing/src/main/java/org/apache/druid/segment/QueryableIndexCursorSequenceBuilder.java @@ -429,16 +429,6 @@ public void advanceUninterruptibly() cursorOffset.increment(); } - @Override - public void advanceTo(int offset) - { - int count = 0; - while (count < offset && !isDone()) { - advance(); - count++; - } - } - @Override public boolean isDone() { diff --git a/processing/src/main/java/org/apache/druid/segment/incremental/IncrementalIndexStorageAdapter.java b/processing/src/main/java/org/apache/druid/segment/incremental/IncrementalIndexStorageAdapter.java index 26835cc2cb2a..8dd3b28f5c83 100644 --- a/processing/src/main/java/org/apache/druid/segment/incremental/IncrementalIndexStorageAdapter.java +++ b/processing/src/main/java/org/apache/druid/segment/incremental/IncrementalIndexStorageAdapter.java @@ -344,16 +344,6 @@ public void advanceUninterruptibly() done = true; } - @Override - public void advanceTo(int offset) - { - int count = 0; - while (count < offset && !isDone()) { - advance(); - count++; - } - } - @Override public boolean isDone() { diff --git a/processing/src/test/java/org/apache/druid/query/aggregation/AggregationTestHelper.java b/processing/src/test/java/org/apache/druid/query/aggregation/AggregationTestHelper.java index af936d1454f6..32aa79910c96 100644 --- a/processing/src/test/java/org/apache/druid/query/aggregation/AggregationTestHelper.java +++ b/processing/src/test/java/org/apache/druid/query/aggregation/AggregationTestHelper.java @@ -23,12 +23,10 @@ import com.fasterxml.jackson.core.JsonToken; import com.fasterxml.jackson.core.ObjectCodec; import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.InjectableValues; import com.fasterxml.jackson.databind.Module; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Function; import com.google.common.base.Supplier; -import com.google.common.base.Suppliers; import com.google.common.collect.Lists; import com.google.common.io.Closeables; import org.apache.commons.io.IOUtils; @@ -59,10 +57,6 @@ import org.apache.druid.query.groupby.GroupByQueryRunnerFactory; import org.apache.druid.query.groupby.GroupByQueryRunnerTest; import org.apache.druid.query.groupby.ResultRow; -import org.apache.druid.query.select.SelectQueryConfig; -import org.apache.druid.query.select.SelectQueryEngine; -import org.apache.druid.query.select.SelectQueryQueryToolChest; -import org.apache.druid.query.select.SelectQueryRunnerFactory; import org.apache.druid.query.timeseries.TimeseriesQueryEngine; import org.apache.druid.query.timeseries.TimeseriesQueryQueryToolChest; import org.apache.druid.query.timeseries.TimeseriesQueryRunnerFactory; @@ -176,60 +170,6 @@ public int columnCacheSizeBytes() ); } - public static AggregationTestHelper createSelectQueryAggregationTestHelper( - List jsonModulesToRegister, - TemporaryFolder tempFolder - ) - { - ObjectMapper mapper = TestHelper.makeJsonMapper(); - mapper.setInjectableValues( - new InjectableValues.Std().addValue( - SelectQueryConfig.class, - new SelectQueryConfig(true) - ) - ); - - Supplier configSupplier = Suppliers.ofInstance(new SelectQueryConfig(true)); - - SelectQueryQueryToolChest toolchest = new SelectQueryQueryToolChest( - TestHelper.makeJsonMapper(), - QueryRunnerTestHelper.noopIntervalChunkingQueryRunnerDecorator() - ); - - SelectQueryRunnerFactory factory = new SelectQueryRunnerFactory( - new SelectQueryQueryToolChest( - TestHelper.makeJsonMapper(), - QueryRunnerTestHelper.noopIntervalChunkingQueryRunnerDecorator() - ), - new SelectQueryEngine( - ), - QueryRunnerTestHelper.NOOP_QUERYWATCHER - ); - - IndexIO indexIO = new IndexIO( - mapper, - new ColumnConfig() - { - @Override - public int columnCacheSizeBytes() - { - return 0; - } - } - ); - - return new AggregationTestHelper( - mapper, - new IndexMergerV9(mapper, indexIO, OffHeapMemorySegmentWriteOutMediumFactory.instance()), - indexIO, - toolchest, - factory, - tempFolder, - jsonModulesToRegister, - Closer.create() - ); - } - public static AggregationTestHelper createTimeseriesQueryAggregationTestHelper( List jsonModulesToRegister, TemporaryFolder tempFolder diff --git a/processing/src/test/java/org/apache/druid/query/scan/MultiSegmentScanQueryTest.java b/processing/src/test/java/org/apache/druid/query/scan/MultiSegmentScanQueryTest.java index e664803e4cb8..3b5a9252d0fe 100644 --- a/processing/src/test/java/org/apache/druid/query/scan/MultiSegmentScanQueryTest.java +++ b/processing/src/test/java/org/apache/druid/query/scan/MultiSegmentScanQueryTest.java @@ -25,6 +25,7 @@ import org.apache.commons.io.IOUtils; import org.apache.commons.lang.StringUtils; import org.apache.druid.java.util.common.DateTimes; +import org.apache.druid.java.util.common.Intervals; import org.apache.druid.java.util.common.concurrent.Execs; import org.apache.druid.java.util.common.granularity.Granularities; import org.apache.druid.java.util.common.guava.MergeSequence; @@ -38,7 +39,8 @@ import org.apache.druid.query.QueryRunnerTestHelper; import org.apache.druid.query.TableDataSource; import org.apache.druid.query.context.ResponseContext; -import org.apache.druid.query.select.SelectQueryRunnerTest; +import org.apache.druid.query.spec.LegacySegmentSpec; +import org.apache.druid.query.spec.QuerySegmentSpec; import org.apache.druid.segment.IncrementalIndexSegment; import org.apache.druid.segment.Segment; import org.apache.druid.segment.TestIndex; @@ -75,6 +77,8 @@ public class MultiSegmentScanQueryTest new ScanQueryEngine(), new ScanQueryConfig() ); + private static final Interval I_0112_0114 = Intervals.of("2011-01-12/2011-01-14"); + public static final QuerySegmentSpec I_0112_0114_SPEC = new LegacySegmentSpec(I_0112_0114); // time modified version of druid.sample.numeric.tsv public static final String[] V_0112 = { @@ -182,7 +186,7 @@ private Druids.ScanQueryBuilder newBuilder() { return Druids.newScanQueryBuilder() .dataSource(new TableDataSource(QueryRunnerTestHelper.DATA_SOURCE)) - .intervals(SelectQueryRunnerTest.I_0112_0114_SPEC) + .intervals(I_0112_0114_SPEC) .batchSize(batchSize) .columns(Collections.emptyList()) .legacy(false) diff --git a/processing/src/test/java/org/apache/druid/query/select/DefaultSelectQueryMetricsTest.java b/processing/src/test/java/org/apache/druid/query/select/DefaultSelectQueryMetricsTest.java deleted file mode 100644 index 3cb569ddc421..000000000000 --- a/processing/src/test/java/org/apache/druid/query/select/DefaultSelectQueryMetricsTest.java +++ /dev/null @@ -1,99 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.druid.query.select; - -import org.apache.druid.java.util.emitter.service.ServiceEmitter; -import org.apache.druid.query.CachingEmitter; -import org.apache.druid.query.DefaultQueryMetricsTest; -import org.apache.druid.query.DruidMetrics; -import org.apache.druid.query.Druids; -import org.apache.druid.query.QueryRunnerTestHelper; -import org.joda.time.Interval; -import org.junit.Assert; -import org.junit.Test; - -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; - -public class DefaultSelectQueryMetricsTest -{ - - /** - * Tests that passed a query {@link DefaultSelectQueryMetrics} produces events with a certain set of dimensions. - */ - @Test - public void testDefaultSelectQueryMetricsQuery() - { - CachingEmitter cachingEmitter = new CachingEmitter(); - ServiceEmitter serviceEmitter = new ServiceEmitter("", "", cachingEmitter); - SelectQuery query = Druids - .newSelectQueryBuilder() - .dataSource(QueryRunnerTestHelper.DATA_SOURCE) - .granularity(QueryRunnerTestHelper.DAY_GRAN) - .intervals(QueryRunnerTestHelper.FULL_ON_INTERVAL_SPEC) - .descending(true) - .pagingSpec(PagingSpec.newSpec(1)) - .build(); - - SelectQueryMetrics queryMetrics = DefaultSelectQueryMetricsFactory.instance().makeMetrics(query); - - queryMetrics.query(query); - - queryMetrics.reportQueryTime(0).emit(serviceEmitter); - Map actualEvent = cachingEmitter.getLastEmittedEvent().toMap(); - Assert.assertEquals(12, actualEvent.size()); - Assert.assertTrue(actualEvent.containsKey("feed")); - Assert.assertTrue(actualEvent.containsKey("timestamp")); - Assert.assertEquals("", actualEvent.get("host")); - Assert.assertEquals("", actualEvent.get("service")); - Assert.assertEquals(QueryRunnerTestHelper.DATA_SOURCE, actualEvent.get(DruidMetrics.DATASOURCE)); - Assert.assertEquals(query.getType(), actualEvent.get(DruidMetrics.TYPE)); - List expectedIntervals = QueryRunnerTestHelper.FULL_ON_INTERVAL_SPEC.getIntervals(); - List expectedStringIntervals = - expectedIntervals.stream().map(Interval::toString).collect(Collectors.toList()); - Assert.assertEquals(expectedStringIntervals, actualEvent.get(DruidMetrics.INTERVAL)); - Assert.assertEquals("false", actualEvent.get("hasFilters")); - Assert.assertEquals(expectedIntervals.get(0).toDuration().toString(), actualEvent.get("duration")); - Assert.assertEquals("", actualEvent.get(DruidMetrics.ID)); - - // Metric - Assert.assertEquals("query/time", actualEvent.get("metric")); - Assert.assertEquals(0L, actualEvent.get("value")); - } - - @Test - public void testDefaultSelectQueryMetricsMetricNamesAndUnits() - { - CachingEmitter cachingEmitter = new CachingEmitter(); - ServiceEmitter serviceEmitter = new ServiceEmitter("", "", cachingEmitter); - SelectQuery query = Druids - .newSelectQueryBuilder() - .dataSource(QueryRunnerTestHelper.DATA_SOURCE) - .granularity(QueryRunnerTestHelper.DAY_GRAN) - .intervals(QueryRunnerTestHelper.FULL_ON_INTERVAL_SPEC) - .descending(true) - .pagingSpec(PagingSpec.newSpec(1)) - .build(); - - SelectQueryMetrics queryMetrics = DefaultSelectQueryMetricsFactory.instance().makeMetrics(query); - DefaultQueryMetricsTest.testQueryMetricsDefaultMetricNamesAndUnits(cachingEmitter, serviceEmitter, queryMetrics); - } -} diff --git a/processing/src/test/java/org/apache/druid/query/select/MultiSegmentSelectQueryTest.java b/processing/src/test/java/org/apache/druid/query/select/MultiSegmentSelectQueryTest.java deleted file mode 100644 index 8534f9061a84..000000000000 --- a/processing/src/test/java/org/apache/druid/query/select/MultiSegmentSelectQueryTest.java +++ /dev/null @@ -1,356 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.druid.query.select; - -import com.google.common.base.Supplier; -import com.google.common.base.Suppliers; -import com.google.common.collect.ImmutableList; -import com.google.common.io.CharSource; -import org.apache.commons.io.IOUtils; -import org.apache.commons.lang.StringUtils; -import org.apache.druid.jackson.DefaultObjectMapper; -import org.apache.druid.java.util.common.DateTimes; -import org.apache.druid.java.util.common.Intervals; -import org.apache.druid.java.util.common.granularity.Granularities; -import org.apache.druid.query.Druids; -import org.apache.druid.query.QueryPlus; -import org.apache.druid.query.QueryRunner; -import org.apache.druid.query.QueryRunnerFactory; -import org.apache.druid.query.QueryRunnerTestHelper; -import org.apache.druid.query.Result; -import org.apache.druid.query.TableDataSource; -import org.apache.druid.query.UnionDataSource; -import org.apache.druid.query.UnionQueryRunner; -import org.apache.druid.query.dimension.DefaultDimensionSpec; -import org.apache.druid.query.ordering.StringComparators; -import org.apache.druid.segment.IncrementalIndexSegment; -import org.apache.druid.segment.ReferenceCountingSegment; -import org.apache.druid.segment.Segment; -import org.apache.druid.segment.TestIndex; -import org.apache.druid.segment.incremental.IncrementalIndex; -import org.apache.druid.segment.incremental.IncrementalIndexSchema; -import org.apache.druid.timeline.SegmentId; -import org.apache.druid.timeline.TimelineObjectHolder; -import org.apache.druid.timeline.VersionedIntervalTimeline; -import org.apache.druid.timeline.partition.NoneShardSpec; -import org.apache.druid.timeline.partition.SingleElementPartitionChunk; -import org.joda.time.Interval; -import org.junit.AfterClass; -import org.junit.Assert; -import org.junit.BeforeClass; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; -import java.util.Map; - -/** - */ -@RunWith(Parameterized.class) -public class MultiSegmentSelectQueryTest -{ - private static final Supplier CONFIG_SUPPLIER = Suppliers.ofInstance(new SelectQueryConfig(true)); - - private static final SelectQueryQueryToolChest TOOL_CHEST = new SelectQueryQueryToolChest( - new DefaultObjectMapper(), - QueryRunnerTestHelper.noopIntervalChunkingQueryRunnerDecorator() - ); - - private static final QueryRunnerFactory FACTORY = new SelectQueryRunnerFactory( - TOOL_CHEST, - new SelectQueryEngine(), - QueryRunnerTestHelper.NOOP_QUERYWATCHER - ); - - // time modified version of druid.sample.numeric.tsv - public static final String[] V_0112 = { - "2011-01-12T00:00:00.000Z\tspot\tautomotive\t1000\t10000.0\t10000.0\t100000\tpreferred\tapreferred\t100.000000", - "2011-01-12T01:00:00.000Z\tspot\tbusiness\t1100\t11000.0\t11000.0\t110000\tpreferred\tbpreferred\t100.000000", - "2011-01-12T02:00:00.000Z\tspot\tentertainment\t1200\t12000.0\t12000.0\t120000\tpreferred\tepreferred\t100.000000", - "2011-01-12T03:00:00.000Z\tspot\thealth\t1300\t13000.0\t13000.0\t130000\tpreferred\thpreferred\t100.000000", - "2011-01-12T04:00:00.000Z\tspot\tmezzanine\t1400\t14000.0\t14000.0\t140000\tpreferred\tmpreferred\t100.000000", - "2011-01-12T05:00:00.000Z\tspot\tnews\t1500\t15000.0\t15000.0\t150000\tpreferred\tnpreferred\t100.000000", - "2011-01-12T06:00:00.000Z\tspot\tpremium\t1600\t16000.0\t16000.0\t160000\tpreferred\tppreferred\t100.000000", - "2011-01-12T07:00:00.000Z\tspot\ttechnology\t1700\t17000.0\t17000.0\t170000\tpreferred\ttpreferred\t100.000000", - "2011-01-12T08:00:00.000Z\tspot\ttravel\t1800\t18000.0\t18000.0\t180000\tpreferred\ttpreferred\t100.000000", - "2011-01-12T09:00:00.000Z\ttotal_market\tmezzanine\t1400\t14000.0\t14000.0\t140000\tpreferred\tmpreferred\t1000.000000", - "2011-01-12T10:00:00.000Z\ttotal_market\tpremium\t1600\t16000.0\t16000.0\t160000\tpreferred\tppreferred\t1000.000000", - "2011-01-12T11:00:00.000Z\tupfront\tmezzanine\t1400\t14000.0\t14000.0\t140000\tpreferred\tmpreferred\t800.000000\tvalue", - "2011-01-12T12:00:00.000Z\tupfront\tpremium\t1600\t16000.0\t16000.0\t160000\tpreferred\tppreferred\t800.000000\tvalue" - }; - - public static final String[] V_0113 = { - "2011-01-13T00:00:00.000Z\tspot\tautomotive\t1000\t10000.0\t10000.0\t100000\tpreferred\tapreferred\t94.874713", - "2011-01-13T01:00:00.000Z\tspot\tbusiness\t1100\t11000.0\t11000.0\t110000\tpreferred\tbpreferred\t103.629399", - "2011-01-13T02:00:00.000Z\tspot\tentertainment\t1200\t12000.0\t12000.0\t120000\tpreferred\tepreferred\t110.087299", - "2011-01-13T03:00:00.000Z\tspot\thealth\t1300\t13000.0\t13000.0\t130000\tpreferred\thpreferred\t114.947403", - "2011-01-13T04:00:00.000Z\tspot\tmezzanine\t1400\t14000.0\t14000.0\t140000\tpreferred\tmpreferred\t104.465767", - "2011-01-13T05:00:00.000Z\tspot\tnews\t1500\t15000.0\t15000.0\t150000\tpreferred\tnpreferred\t102.851683", - "2011-01-13T06:00:00.000Z\tspot\tpremium\t1600\t16000.0\t16000.0\t160000\tpreferred\tppreferred\t108.863011", - "2011-01-13T07:00:00.000Z\tspot\ttechnology\t1700\t17000.0\t17000.0\t170000\tpreferred\ttpreferred\t111.356672", - "2011-01-13T08:00:00.000Z\tspot\ttravel\t1800\t18000.0\t18000.0\t180000\tpreferred\ttpreferred\t106.236928", - "2011-01-13T09:00:00.000Z\ttotal_market\tmezzanine\t1400\t14000.0\t14000.0\t140000\tpreferred\tmpreferred\t1040.945505", - "2011-01-13T10:00:00.000Z\ttotal_market\tpremium\t1600\t16000.0\t16000.0\t160000\tpreferred\tppreferred\t1689.012875", - "2011-01-13T11:00:00.000Z\tupfront\tmezzanine\t1400\t14000.0\t14000.0\t140000\tpreferred\tmpreferred\t826.060182\tvalue", - "2011-01-13T12:00:00.000Z\tupfront\tpremium\t1600\t16000.0\t16000.0\t160000\tpreferred\tppreferred\t1564.617729\tvalue" - }; - - public static final String[] V_OVERRIDE = { - "2011-01-12T04:00:00.000Z\tspot\tautomotive\t1000\t10000.0\t10000.0\t100000\tpreferred\tapreferred\t999.000000", - "2011-01-12T05:00:00.000Z\tspot\tbusiness\t1100\t11000.0\t11000.0\t110000\tpreferred\tbpreferred\t999.000000", - "2011-01-12T06:00:00.000Z\tspot\tentertainment\t1200\t12000.0\t12000.0\t120000\tpreferred\tepreferred\t999.000000", - "2011-01-12T07:00:00.000Z\tspot\thealth\t1300\t13000.0\t13000.0\t130000\tpreferred\thpreferred\t999.000000" - }; - - private static Segment segment0; - private static Segment segment1; - private static Segment segment_override; // this makes segment0 split into three logical segments - - private static List segmentIdentifiers; - - private static QueryRunner runner; - - @BeforeClass - public static void setup() throws IOException - { - CharSource v_0112 = CharSource.wrap(StringUtils.join(V_0112, "\n")); - CharSource v_0113 = CharSource.wrap(StringUtils.join(V_0113, "\n")); - CharSource v_override = CharSource.wrap(StringUtils.join(V_OVERRIDE, "\n")); - - IncrementalIndex index0 = TestIndex.loadIncrementalIndex(newIndex("2011-01-12T00:00:00.000Z"), v_0112); - IncrementalIndex index1 = TestIndex.loadIncrementalIndex(newIndex("2011-01-13T00:00:00.000Z"), v_0113); - IncrementalIndex index2 = TestIndex.loadIncrementalIndex(newIndex("2011-01-12T04:00:00.000Z"), v_override); - - segment0 = new IncrementalIndexSegment(index0, makeIdentifier(index0, "v1")); - segment1 = new IncrementalIndexSegment(index1, makeIdentifier(index1, "v1")); - segment_override = new IncrementalIndexSegment(index2, makeIdentifier(index2, "v2")); - - VersionedIntervalTimeline timeline = new VersionedIntervalTimeline<>(StringComparators.LEXICOGRAPHIC); - timeline.add( - index0.getInterval(), - "v1", - new SingleElementPartitionChunk<>(ReferenceCountingSegment.wrapRootGenerationSegment(segment0)) - ); - timeline.add( - index1.getInterval(), - "v1", - new SingleElementPartitionChunk<>(ReferenceCountingSegment.wrapRootGenerationSegment(segment1)) - ); - timeline.add( - index2.getInterval(), - "v2", - new SingleElementPartitionChunk<>(ReferenceCountingSegment.wrapRootGenerationSegment(segment_override)) - ); - - segmentIdentifiers = new ArrayList<>(); - for (TimelineObjectHolder holder : timeline.lookup(Intervals.of("2011-01-12/2011-01-14"))) { - segmentIdentifiers.add(makeIdentifier(holder.getInterval(), holder.getVersion()).toString()); - } - - runner = QueryRunnerTestHelper.makeFilteringQueryRunner(timeline, FACTORY); - } - - private static SegmentId makeIdentifier(IncrementalIndex index, String version) - { - return makeIdentifier(index.getInterval(), version); - } - - private static SegmentId makeIdentifier(Interval interval, String version) - { - return SegmentId.of(QueryRunnerTestHelper.DATA_SOURCE, interval, version, NoneShardSpec.instance()); - } - - private static IncrementalIndex newIndex(String minTimeStamp) - { - return newIndex(minTimeStamp, 10000); - } - - private static IncrementalIndex newIndex(String minTimeStamp, int maxRowCount) - { - final IncrementalIndexSchema schema = new IncrementalIndexSchema.Builder() - .withMinTimestamp(DateTimes.of(minTimeStamp).getMillis()) - .withQueryGranularity(Granularities.HOUR) - .withMetrics(TestIndex.METRIC_AGGS) - .build(); - return new IncrementalIndex.Builder() - .setIndexSchema(schema) - .setMaxRowCount(maxRowCount) - .buildOnheap(); - } - - @AfterClass - public static void clear() - { - IOUtils.closeQuietly(segment0); - IOUtils.closeQuietly(segment1); - IOUtils.closeQuietly(segment_override); - } - - @Parameterized.Parameters(name = "fromNext={0}") - public static Iterable constructorFeeder() - { - return QueryRunnerTestHelper.cartesian(Arrays.asList(false, true)); - } - - private final boolean fromNext; - - public MultiSegmentSelectQueryTest(boolean fromNext) - { - this.fromNext = fromNext; - } - - private Druids.SelectQueryBuilder newBuilder() - { - return Druids.newSelectQueryBuilder() - .dataSource(new TableDataSource(QueryRunnerTestHelper.DATA_SOURCE)) - .intervals(SelectQueryRunnerTest.I_0112_0114_SPEC) - .granularity(QueryRunnerTestHelper.ALL_GRAN) - .dimensionSpecs(DefaultDimensionSpec.toSpec(QueryRunnerTestHelper.DIMENSIONS)) - .pagingSpec(PagingSpec.newSpec(3)); - } - - @Test - public void testAllGranularity() - { - runAllGranularityTest( - newBuilder().build(), - new int[][]{ - {2, -1, -1, -1, 3}, {3, 1, -1, -1, 3}, {-1, 3, 0, -1, 3}, {-1, -1, 3, -1, 3}, {-1, -1, 4, 1, 3}, - {-1, -1, -1, 4, 3}, {-1, -1, -1, 7, 3}, {-1, -1, -1, 10, 3}, {-1, -1, -1, 12, 2}, {-1, -1, -1, 13, 0} - } - ); - - runAllGranularityTest( - newBuilder().descending(true).build(), - new int[][]{ - {0, 0, 0, -3, 3}, {0, 0, 0, -6, 3}, {0, 0, 0, -9, 3}, {0, 0, 0, -12, 3}, {0, 0, -2, -13, 3}, - {0, 0, -5, 0, 3}, {0, -3, 0, 0, 3}, {-2, -4, 0, 0, 3}, {-4, 0, 0, 0, 2}, {-5, 0, 0, 0, 0} - } - ); - } - - private void runAllGranularityTest(SelectQuery query, int[][] expectedOffsets) - { - for (int[] expected : expectedOffsets) { - List> results = runner.run(QueryPlus.wrap(query)).toList(); - Assert.assertEquals(1, results.size()); - - SelectResultValue value = results.get(0).getValue(); - Map pagingIdentifiers = value.getPagingIdentifiers(); - - Map merged = PagingSpec.merge(Collections.singletonList(pagingIdentifiers)); - - for (int i = 0; i < 4; i++) { - if (query.isDescending() ^ expected[i] >= 0) { - Assert.assertEquals(expected[i], pagingIdentifiers.get(segmentIdentifiers.get(i)).intValue()); - } - } - Assert.assertEquals(expected[4], value.getEvents().size()); - - query = query.withPagingSpec(toNextCursor(merged, query, 3)); - } - } - - @Test - public void testDayGranularity() - { - runDayGranularityTest( - newBuilder().granularity(QueryRunnerTestHelper.DAY_GRAN).build(), - new int[][]{ - {2, -1, -1, 2, 3, 0, 0, 3}, {3, 1, -1, 5, 1, 2, 0, 3}, {-1, 3, 0, 8, 0, 2, 1, 3}, - {-1, -1, 3, 11, 0, 0, 3, 3}, {-1, -1, 4, 12, 0, 0, 1, 1}, {-1, -1, 5, 13, 0, 0, 0, 0} - } - ); - - runDayGranularityTest( - newBuilder().granularity(QueryRunnerTestHelper.DAY_GRAN).descending(true).build(), - new int[][]{ - {0, 0, -3, -3, 0, 0, 3, 3}, {0, -1, -5, -6, 0, 1, 2, 3}, {0, -4, 0, -9, 0, 3, 0, 3}, - {-3, 0, 0, -12, 3, 0, 0, 3}, {-4, 0, 0, -13, 1, 0, 0, 1}, {-5, 0, 0, -14, 0, 0, 0, 0} - } - ); - } - - private void runDayGranularityTest(SelectQuery query, int[][] expectedOffsets) - { - for (int[] expected : expectedOffsets) { - List> results = runner.run(QueryPlus.wrap(query)).toList(); - Assert.assertEquals(2, results.size()); - - SelectResultValue value0 = results.get(0).getValue(); - SelectResultValue value1 = results.get(1).getValue(); - - Map pagingIdentifiers0 = value0.getPagingIdentifiers(); - Map pagingIdentifiers1 = value1.getPagingIdentifiers(); - - Map merged = PagingSpec.merge(Arrays.asList(pagingIdentifiers0, pagingIdentifiers1)); - - for (int i = 0; i < 4; i++) { - if (query.isDescending() ^ expected[i] >= 0) { - Assert.assertEquals(expected[i], merged.get(segmentIdentifiers.get(i)).intValue()); - } - } - - query = query.withPagingSpec(toNextCursor(merged, query, 3)); - } - } - - @Test - public void testPagingIdentifiersForUnionDatasource() - { - Druids.SelectQueryBuilder selectQueryBuilder = Druids - .newSelectQueryBuilder() - .dataSource( - new UnionDataSource( - ImmutableList.of( - new TableDataSource(QueryRunnerTestHelper.DATA_SOURCE), - new TableDataSource("testing-2") - ) - ) - ) - .intervals(SelectQueryRunnerTest.I_0112_0114_SPEC) - .granularity(QueryRunnerTestHelper.ALL_GRAN) - .dimensionSpecs(DefaultDimensionSpec.toSpec(QueryRunnerTestHelper.DIMENSIONS)) - .pagingSpec(PagingSpec.newSpec(3)); - - SelectQuery query = selectQueryBuilder.build(); - QueryRunner unionQueryRunner = new UnionQueryRunner(runner); - - List> results = unionQueryRunner.run(QueryPlus.wrap(query)).toList(); - - Map pagingIdentifiers = results.get(0).getValue().getPagingIdentifiers(); - query = query.withPagingSpec(toNextCursor(PagingSpec.merge(Collections.singletonList(pagingIdentifiers)), query, 3)); - - unionQueryRunner.run(QueryPlus.wrap(query)).toList(); - } - - private PagingSpec toNextCursor(Map merged, SelectQuery query, int threshold) - { - if (!fromNext) { - merged = PagingSpec.next(merged, query.isDescending()); - } - return new PagingSpec(merged, threshold, fromNext); - } -} diff --git a/processing/src/test/java/org/apache/druid/query/select/PagingOffsetTest.java b/processing/src/test/java/org/apache/druid/query/select/PagingOffsetTest.java deleted file mode 100644 index 9f6f26484d5f..000000000000 --- a/processing/src/test/java/org/apache/druid/query/select/PagingOffsetTest.java +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.druid.query.select; - -import com.google.common.primitives.Ints; -import org.junit.Assert; -import org.junit.Test; - -import java.util.ArrayList; -import java.util.List; - -public class PagingOffsetTest -{ - @Test - public void testZeroThreshold() - { - PagingOffset offset = PagingOffset.of(PagingOffset.toOffset(3, false), 0); - Assert.assertEquals(3, offset.startOffset()); - Assert.assertEquals(3, offset.startDelta()); - Assert.assertArrayEquals(new int[]{}, toArray(offset)); - - offset = PagingOffset.of(PagingOffset.toOffset(3, true), 0); - Assert.assertEquals(-4, offset.startOffset()); - Assert.assertEquals(3, offset.startDelta()); - Assert.assertArrayEquals(new int[]{}, toArray(offset)); - } - - @Test - public void testAscending() - { - PagingOffset offset = PagingOffset.of(PagingOffset.toOffset(3, false), 3); - Assert.assertEquals(3, offset.startOffset()); - Assert.assertEquals(3, offset.startDelta()); - Assert.assertArrayEquals(new int[]{3, 4, 5}, toArray(offset)); - } - - @Test - public void testDescending() - { - PagingOffset offset = PagingOffset.of(PagingOffset.toOffset(3, true), 3); - Assert.assertEquals(-4, offset.startOffset()); - Assert.assertEquals(3, offset.startDelta()); - Assert.assertArrayEquals(new int[]{-4, -5, -6}, toArray(offset)); - } - - private int[] toArray(PagingOffset offset) - { - List ints = new ArrayList<>(); - for (; offset.hasNext(); offset.next()) { - ints.add(offset.current()); - } - return Ints.toArray(ints); - } -} diff --git a/processing/src/test/java/org/apache/druid/query/select/SelectBinaryFnTest.java b/processing/src/test/java/org/apache/druid/query/select/SelectBinaryFnTest.java deleted file mode 100644 index 19bf08e95389..000000000000 --- a/processing/src/test/java/org/apache/druid/query/select/SelectBinaryFnTest.java +++ /dev/null @@ -1,277 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.druid.query.select; - -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Maps; -import com.google.common.collect.Sets; -import org.apache.druid.java.util.common.DateTimes; -import org.apache.druid.java.util.common.ISE; -import org.apache.druid.java.util.common.granularity.Granularities; -import org.apache.druid.query.Result; -import org.junit.Assert; -import org.junit.Test; - -import java.util.Arrays; -import java.util.Collections; -import java.util.Iterator; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Set; - -/** - */ -public class SelectBinaryFnTest -{ - private static final String SEGMENT_ID1 = "testSegment1"; - - private static final String SEGMENT_ID2 = "testSegment2"; - - @Test - public void testApply() - { - SelectBinaryFn binaryFn = new SelectBinaryFn(Granularities.ALL, new PagingSpec(null, 5), false); - - Result res1 = new Result<>( - DateTimes.of("2013-01-01"), - new SelectResultValue( - ImmutableMap.of(), - Sets.newHashSet("first", "fourth"), - Sets.newHashSet("sixth"), - Arrays.asList( - new EventHolder( - SEGMENT_ID1, - 0, - ImmutableMap.of( - EventHolder.TIMESTAMP_KEY, - DateTimes.of("2013-01-01T00"), - "dim", - "first" - ) - ), - new EventHolder( - SEGMENT_ID1, - 1, - ImmutableMap.of( - EventHolder.TIMESTAMP_KEY, - DateTimes.of("2013-01-01T03"), - "dim", - "fourth" - ) - ), - new EventHolder( - SEGMENT_ID1, - 2, - ImmutableMap.of( - EventHolder.TIMESTAMP_KEY, - DateTimes.of("2013-01-01T05"), - "dim", - "sixth" - ) - ) - ) - ) - ); - - - Result res2 = new Result<>( - DateTimes.of("2013-01-01"), - new SelectResultValue( - ImmutableMap.of(), - Sets.newHashSet("second", "third"), - Sets.newHashSet("fifth"), - Arrays.asList( - new EventHolder( - SEGMENT_ID2, - 0, - ImmutableMap.of( - EventHolder.TIMESTAMP_KEY, - DateTimes.of("2013-01-01T00"), - "dim", - "second" - ) - ), - new EventHolder( - SEGMENT_ID2, - 1, - ImmutableMap.of( - EventHolder.TIMESTAMP_KEY, - DateTimes.of("2013-01-01T02"), - "dim", - "third" - ) - ), - new EventHolder( - SEGMENT_ID2, - 2, - ImmutableMap.of( - EventHolder.TIMESTAMP_KEY, - DateTimes.of("2013-01-01T04"), - "dim", - "fifth" - ) - ) - ) - ) - ); - - Result merged = binaryFn.apply(res1, res2); - - Assert.assertEquals(res1.getTimestamp(), merged.getTimestamp()); - - LinkedHashMap expectedPageIds = Maps.newLinkedHashMap(); - expectedPageIds.put(SEGMENT_ID1, 1); - expectedPageIds.put(SEGMENT_ID2, 2); - - Iterator exSegmentIter = expectedPageIds.keySet().iterator(); - Iterator acSegmentIter = merged.getValue().getPagingIdentifiers().keySet().iterator(); - - verifyIters(exSegmentIter, acSegmentIter); - - Iterator exOffsetIter = expectedPageIds.values().iterator(); - Iterator acOffsetIter = merged.getValue().getPagingIdentifiers().values().iterator(); - - verifyIters(exOffsetIter, acOffsetIter); - - List exEvents = Arrays.asList( - new EventHolder( - SEGMENT_ID1, - 0, - ImmutableMap.of( - EventHolder.TIMESTAMP_KEY, - DateTimes.of("2013-01-01T00"), "dim", "first" - ) - ), - new EventHolder( - SEGMENT_ID2, - 0, - ImmutableMap.of( - EventHolder.TIMESTAMP_KEY, - DateTimes.of("2013-01-01T00"), - "dim", - "second" - ) - ), - new EventHolder( - SEGMENT_ID2, - 1, - ImmutableMap.of( - EventHolder.TIMESTAMP_KEY, - DateTimes.of("2013-01-01T02"), - "dim", - "third" - ) - ), - new EventHolder( - SEGMENT_ID1, - 1, - ImmutableMap.of( - EventHolder.TIMESTAMP_KEY, - DateTimes.of("2013-01-01T03"), - "dim", - "fourth" - ) - ), - new EventHolder( - SEGMENT_ID2, - 2, - ImmutableMap.of( - EventHolder.TIMESTAMP_KEY, - DateTimes.of("2013-01-01T04"), - "dim", - "fifth" - ) - ) - ); - - List acEvents = merged.getValue().getEvents(); - - - verifyEvents(exEvents, acEvents); - } - - @Test - public void testColumnMerge() - { - SelectBinaryFn binaryFn = new SelectBinaryFn(Granularities.ALL, new PagingSpec(null, 5), false); - - Result res1 = new Result<>( - DateTimes.of("2013-01-01"), - new SelectResultValue( - ImmutableMap.of(), - Sets.newHashSet("first", "second", "fourth"), - Sets.newHashSet("eight", "nineth"), - Collections.singletonList( - new EventHolder( - SEGMENT_ID1, - 0, - ImmutableMap.of(EventHolder.TIMESTAMP_KEY, DateTimes.of("2013-01-01T00"), "dim", "first") - ) - ) - ) - ); - - Result res2 = new Result<>( - DateTimes.of("2013-01-01"), - new SelectResultValue( - ImmutableMap.of(), - Sets.newHashSet("third", "second", "fifth"), - Sets.newHashSet("seventh"), - Collections.singletonList( - new EventHolder( - SEGMENT_ID2, - 0, - ImmutableMap.of(EventHolder.TIMESTAMP_KEY, DateTimes.of("2013-01-01T00"), "dim", "second") - ) - ) - ) - ); - - Result merged = binaryFn.apply(res1, res2); - - Set exDimensions = Sets.newHashSet("first", "second", "fourth", "third", "fifth"); - Set exMetrics = Sets.newHashSet("eight", "nineth", "seventh"); - - Set acDimensions = merged.getValue().getDimensions(); - Set acMetrics = merged.getValue().getMetrics(); - - Assert.assertEquals(exDimensions, acDimensions); - Assert.assertEquals(exMetrics, acMetrics); - } - - private void verifyIters(Iterator iter1, Iterator iter2) - { - while (iter1.hasNext()) { - Assert.assertEquals(iter1.next(), iter2.next()); - } - - if (iter2.hasNext()) { - throw new ISE("This should be empty!"); - } - } - - private void verifyEvents(List events1, List events2) - { - Iterator ex = events1.iterator(); - Iterator ac = events2.iterator(); - - verifyIters(ex, ac); - } -} diff --git a/processing/src/test/java/org/apache/druid/query/select/SelectQueryConfigTest.java b/processing/src/test/java/org/apache/druid/query/select/SelectQueryConfigTest.java deleted file mode 100644 index b47bbcf3164a..000000000000 --- a/processing/src/test/java/org/apache/druid/query/select/SelectQueryConfigTest.java +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.druid.query.select; - -import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.common.collect.ImmutableMap; -import org.apache.druid.jackson.DefaultObjectMapper; -import org.junit.Assert; -import org.junit.Test; - -public class SelectQueryConfigTest -{ - private final ObjectMapper MAPPER = new DefaultObjectMapper(); - - private final ImmutableMap CONFIG_MAP = ImmutableMap - .builder() - .put(SelectQueryConfig.ENABLE_FROM_NEXT_DEFAULT, "false") - .build(); - - private final ImmutableMap CONFIG_MAP2 = ImmutableMap - .builder() - .put(SelectQueryConfig.ENABLE_FROM_NEXT_DEFAULT, "true") - .build(); - - private final ImmutableMap CONFIG_MAP_EMPTY = ImmutableMap - .builder() - .build(); - - @Test - public void testSerde() - { - final SelectQueryConfig config = MAPPER.convertValue(CONFIG_MAP, SelectQueryConfig.class); - Assert.assertEquals(false, config.getEnableFromNextDefault()); - - final SelectQueryConfig config2 = MAPPER.convertValue(CONFIG_MAP2, SelectQueryConfig.class); - Assert.assertEquals(true, config2.getEnableFromNextDefault()); - - final SelectQueryConfig configEmpty = MAPPER.convertValue(CONFIG_MAP_EMPTY, SelectQueryConfig.class); - Assert.assertEquals(true, configEmpty.getEnableFromNextDefault()); - } -} diff --git a/processing/src/test/java/org/apache/druid/query/select/SelectQueryQueryToolChestTest.java b/processing/src/test/java/org/apache/druid/query/select/SelectQueryQueryToolChestTest.java deleted file mode 100644 index 540bdc3722fd..000000000000 --- a/processing/src/test/java/org/apache/druid/query/select/SelectQueryQueryToolChestTest.java +++ /dev/null @@ -1,72 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.druid.query.select; - -import com.google.common.base.Supplier; -import com.google.common.base.Suppliers; -import org.apache.druid.jackson.DefaultObjectMapper; -import org.apache.druid.query.CacheStrategy; -import org.apache.druid.query.Druids; -import org.apache.druid.query.QueryRunnerTestHelper; -import org.apache.druid.query.Result; -import org.junit.Assert; -import org.junit.Test; - -import java.util.Arrays; -import java.util.Collections; - -public class SelectQueryQueryToolChestTest -{ - private static final Supplier CONFIG_SUPPLIER = Suppliers.ofInstance(new SelectQueryConfig(true)); - - private static final SelectQueryQueryToolChest TOOL_CHEST = new SelectQueryQueryToolChest( - new DefaultObjectMapper(), - QueryRunnerTestHelper.noopIntervalChunkingQueryRunnerDecorator() - ); - - @Test - public void testComputeCacheKeyWithDifferentSortOrer() - { - final SelectQuery query1 = Druids.newSelectQueryBuilder() - .dataSource("dummy") - .dimensions(Collections.singletonList("testDim")) - .intervals(SelectQueryRunnerTest.I_0112_0114_SPEC) - .granularity(QueryRunnerTestHelper.ALL_GRAN) - .pagingSpec(PagingSpec.newSpec(3)) - .descending(false) - .build(); - - final SelectQuery query2 = Druids.newSelectQueryBuilder() - .dataSource("dummy") - .dimensions(Collections.singletonList("testDim")) - .intervals(SelectQueryRunnerTest.I_0112_0114_SPEC) - .granularity(QueryRunnerTestHelper.ALL_GRAN) - .pagingSpec(PagingSpec.newSpec(3)) - .descending(true) - .build(); - - final CacheStrategy, Object, SelectQuery> strategy1 = TOOL_CHEST.getCacheStrategy(query1); - Assert.assertNotNull(strategy1); - final CacheStrategy, Object, SelectQuery> strategy2 = TOOL_CHEST.getCacheStrategy(query2); - Assert.assertNotNull(strategy2); - - Assert.assertFalse(Arrays.equals(strategy1.computeCacheKey(query1), strategy2.computeCacheKey(query2))); - } -} diff --git a/processing/src/test/java/org/apache/druid/query/select/SelectQueryRunnerTest.java b/processing/src/test/java/org/apache/druid/query/select/SelectQueryRunnerTest.java deleted file mode 100644 index e4f8a393ce24..000000000000 --- a/processing/src/test/java/org/apache/druid/query/select/SelectQueryRunnerTest.java +++ /dev/null @@ -1,1095 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.druid.query.select; - -import com.google.common.base.Function; -import com.google.common.base.Supplier; -import com.google.common.base.Suppliers; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Iterables; -import com.google.common.collect.Lists; -import com.google.common.collect.ObjectArrays; -import com.google.common.collect.Sets; -import org.apache.druid.jackson.DefaultObjectMapper; -import org.apache.druid.java.util.common.DateTimes; -import org.apache.druid.java.util.common.ISE; -import org.apache.druid.java.util.common.Intervals; -import org.apache.druid.js.JavaScriptConfig; -import org.apache.druid.query.Druids; -import org.apache.druid.query.QueryPlus; -import org.apache.druid.query.QueryRunner; -import org.apache.druid.query.QueryRunnerTestHelper; -import org.apache.druid.query.Result; -import org.apache.druid.query.TableDataSource; -import org.apache.druid.query.dimension.DefaultDimensionSpec; -import org.apache.druid.query.dimension.DimensionSpec; -import org.apache.druid.query.dimension.ExtractionDimensionSpec; -import org.apache.druid.query.expression.TestExprMacroTable; -import org.apache.druid.query.extraction.ExtractionFn; -import org.apache.druid.query.extraction.JavaScriptExtractionFn; -import org.apache.druid.query.extraction.MapLookupExtractor; -import org.apache.druid.query.filter.AndDimFilter; -import org.apache.druid.query.filter.BoundDimFilter; -import org.apache.druid.query.filter.SelectorDimFilter; -import org.apache.druid.query.lookup.LookupExtractionFn; -import org.apache.druid.query.ordering.StringComparators; -import org.apache.druid.query.spec.LegacySegmentSpec; -import org.apache.druid.query.spec.QuerySegmentSpec; -import org.apache.druid.segment.column.ColumnHolder; -import org.apache.druid.segment.column.ValueType; -import org.apache.druid.segment.virtual.ExpressionVirtualColumn; -import org.apache.druid.timeline.SegmentId; -import org.joda.time.DateTime; -import org.joda.time.Interval; -import org.joda.time.chrono.ISOChronology; -import org.junit.Assert; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashMap; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Set; - -/** - */ -@RunWith(Parameterized.class) -public class SelectQueryRunnerTest -{ - // copied from druid.sample.numeric.tsv - public static final String[] V_0112 = { - "2011-01-12T00:00:00.000Z\tspot\tautomotive\t1000\t10000.0\t100000\tpreferred\tapreferred\t100.000000", - "2011-01-12T00:00:00.000Z\tspot\tbusiness\t1100\t11000.0\t110000\tpreferred\tbpreferred\t100.000000", - "2011-01-12T00:00:00.000Z\tspot\tentertainment\t1200\t12000.0\t120000\tpreferred\tepreferred\t100.000000", - "2011-01-12T00:00:00.000Z\tspot\thealth\t1300\t13000.0\t130000\tpreferred\thpreferred\t100.000000", - "2011-01-12T00:00:00.000Z\tspot\tmezzanine\t1400\t14000.0\t140000\tpreferred\tmpreferred\t100.000000", - "2011-01-12T00:00:00.000Z\tspot\tnews\t1500\t15000.0\t150000\tpreferred\tnpreferred\t100.000000", - "2011-01-12T00:00:00.000Z\tspot\tpremium\t1600\t16000.0\t160000\tpreferred\tppreferred\t100.000000", - "2011-01-12T00:00:00.000Z\tspot\ttechnology\t1700\t17000.0\t170000\tpreferred\ttpreferred\t100.000000", - "2011-01-12T00:00:00.000Z\tspot\ttravel\t1800\t18000.0\t180000\tpreferred\ttpreferred\t100.000000", - "2011-01-12T00:00:00.000Z\ttotal_market\tmezzanine\t1400\t14000.0\t140000\tpreferred\tmpreferred\t1000.000000", - "2011-01-12T00:00:00.000Z\ttotal_market\tpremium\t1600\t16000.0\t160000\tpreferred\tppreferred\t1000.000000", - "2011-01-12T00:00:00.000Z\tupfront\tmezzanine\t1400\t14000.0\t140000\tpreferred\tmpreferred\t800.000000\tvalue", - "2011-01-12T00:00:00.000Z\tupfront\tpremium\t1600\t16000.0\t160000\tpreferred\tppreferred\t800.000000\tvalue" - }; - public static final String[] V_0113 = { - "2011-01-13T00:00:00.000Z\tspot\tautomotive\t1000\t10000.0\t100000\tpreferred\tapreferred\t94.874713", - "2011-01-13T00:00:00.000Z\tspot\tbusiness\t1100\t11000.0\t110000\tpreferred\tbpreferred\t103.629399", - "2011-01-13T00:00:00.000Z\tspot\tentertainment\t1200\t12000.0\t120000\tpreferred\tepreferred\t110.087299", - "2011-01-13T00:00:00.000Z\tspot\thealth\t1300\t13000.0\t130000\tpreferred\thpreferred\t114.947403", - "2011-01-13T00:00:00.000Z\tspot\tmezzanine\t1400\t14000.0\t140000\tpreferred\tmpreferred\t104.465767", - "2011-01-13T00:00:00.000Z\tspot\tnews\t1500\t15000.0\t150000\tpreferred\tnpreferred\t102.851683", - "2011-01-13T00:00:00.000Z\tspot\tpremium\t1600\t16000.0\t160000\tpreferred\tppreferred\t108.863011", - "2011-01-13T00:00:00.000Z\tspot\ttechnology\t1700\t17000.0\t170000\tpreferred\ttpreferred\t111.356672", - "2011-01-13T00:00:00.000Z\tspot\ttravel\t1800\t18000.0\t180000\tpreferred\ttpreferred\t106.236928", - "2011-01-13T00:00:00.000Z\ttotal_market\tmezzanine\t1400\t14000.0\t140000\tpreferred\tmpreferred\t1040.945505", - "2011-01-13T00:00:00.000Z\ttotal_market\tpremium\t1600\t16000.0\t160000\tpreferred\tppreferred\t1689.012875", - "2011-01-13T00:00:00.000Z\tupfront\tmezzanine\t1400\t14000.0\t140000\tpreferred\tmpreferred\t826.060182\tvalue", - "2011-01-13T00:00:00.000Z\tupfront\tpremium\t1600\t16000.0\t160000\tpreferred\tppreferred\t1564.617729\tvalue" - }; - - private static final Interval I_0112_0114 = Intervals.of("2011-01-12/2011-01-14"); - public static final QuerySegmentSpec I_0112_0114_SPEC = new LegacySegmentSpec(I_0112_0114); - private static final SegmentId SEGMENT_ID_I_0112_0114 = QueryRunnerTestHelper.SEGMENT_ID.withInterval(I_0112_0114); - - private static final String SEGMENT_ID_STRING = SEGMENT_ID_I_0112_0114.toString(); - - public static final String[] V_0112_0114 = ObjectArrays.concat(V_0112, V_0113, String.class); - - private static final boolean DEFAULT_FROM_NEXT = true; - private static final SelectQueryConfig CONFIG = new SelectQueryConfig(true); - - static { - CONFIG.setEnableFromNextDefault(DEFAULT_FROM_NEXT); - } - - private static final Supplier CONFIG_SUPPLIER = Suppliers.ofInstance(CONFIG); - - private static final SelectQueryQueryToolChest TOOL_CHEST = new SelectQueryQueryToolChest( - new DefaultObjectMapper(), - QueryRunnerTestHelper.noopIntervalChunkingQueryRunnerDecorator() - ); - - @Parameterized.Parameters(name = "{0}:descending={1}") - public static Iterable constructorFeeder() - { - return QueryRunnerTestHelper.cartesian( - QueryRunnerTestHelper.makeQueryRunners( - new SelectQueryRunnerFactory( - TOOL_CHEST, - new SelectQueryEngine(), - QueryRunnerTestHelper.NOOP_QUERYWATCHER - ) - ), - Arrays.asList(false, true) - ); - } - private final QueryRunner runner; - private final boolean descending; - - public SelectQueryRunnerTest(QueryRunner runner, boolean descending) - { - this.runner = runner; - this.descending = descending; - } - - private Druids.SelectQueryBuilder newTestQuery() - { - return Druids.newSelectQueryBuilder() - .dataSource(new TableDataSource(QueryRunnerTestHelper.DATA_SOURCE)) - .dimensionSpecs(DefaultDimensionSpec.toSpec(Collections.emptyList())) - .metrics(Collections.emptyList()) - .intervals(QueryRunnerTestHelper.FULL_ON_INTERVAL_SPEC) - .granularity(QueryRunnerTestHelper.ALL_GRAN) - .pagingSpec(PagingSpec.newSpec(3)) - .descending(descending); - } - - @Test - public void testFullOnSelect() - { - SelectQuery query = newTestQuery() - .intervals(I_0112_0114_SPEC) - .build(); - - Iterable> results = runner.run(QueryPlus.wrap(query)).toList(); - - PagingOffset offset = query.getPagingOffset(SEGMENT_ID_STRING); - List> expectedResults = toExpected( - SEGMENT_ID_STRING, - toFullEvents(V_0112_0114), - Lists.newArrayList( - "market", - "quality", - "qualityLong", - "qualityFloat", - "qualityDouble", - "qualityNumericString", - "placement", - "placementish", - "partial_null_column", - "null_column" - ), - Lists.newArrayList( - "index", - "quality_uniques", - "indexMin", - "indexMaxPlusTen", - "indexFloat", - "indexMaxFloat", - "indexMinFloat" - ), - offset.startOffset(), - offset.threshold() - ); - verify(expectedResults, populateNullColumnAtLastForQueryableIndexCase(results, "null_column")); - } - - @Test - public void testSequentialPaging() - { - int[] asc = {2, 5, 8, 11, 14, 17, 20, 23, 25}; - int[] dsc = {-3, -6, -9, -12, -15, -18, -21, -24, -26}; - int[] expected = descending ? dsc : asc; - - SelectQuery query = newTestQuery().intervals(I_0112_0114_SPEC).build(); - for (int offset : expected) { - List> results = runner.run(QueryPlus.wrap(query)).toList(); - - Assert.assertEquals(1, results.size()); - - SelectResultValue result = results.get(0).getValue(); - Map pagingIdentifiers = result.getPagingIdentifiers(); - Assert.assertEquals(offset, pagingIdentifiers.get(SEGMENT_ID_I_0112_0114.toString()).intValue()); - - Map next = PagingSpec.next(pagingIdentifiers, descending); - query = query.withPagingSpec(new PagingSpec(next, 3, false)); - } - - query = newTestQuery().intervals(I_0112_0114_SPEC).build(); - for (int offset : expected) { - List> results = runner.run(QueryPlus.wrap(query)).toList(); - - Assert.assertEquals(1, results.size()); - - SelectResultValue result = results.get(0).getValue(); - Map pagingIdentifiers = result.getPagingIdentifiers(); - Assert.assertEquals(offset, pagingIdentifiers.get(SEGMENT_ID_I_0112_0114.toString()).intValue()); - - // use identifier as-is but with fromNext=true - query = query.withPagingSpec(new PagingSpec(pagingIdentifiers, 3, true)); - } - } - - @Test - public void testFullOnSelectWithDimensionSpec() - { - Map map = new HashMap<>(); - map.put("automotive", "automotive0"); - map.put("business", "business0"); - map.put("entertainment", "entertainment0"); - map.put("health", "health0"); - map.put("mezzanine", "mezzanine0"); - map.put("news", "news0"); - map.put("premium", "premium0"); - map.put("technology", "technology0"); - map.put("travel", "travel0"); - - SelectQuery query = newTestQuery() - .dimensionSpecs( - Arrays.asList( - new DefaultDimensionSpec(QueryRunnerTestHelper.MARKET_DIMENSION, "mar"), - new ExtractionDimensionSpec( - QueryRunnerTestHelper.QUALITY_DIMENSION, - "qual", - new LookupExtractionFn(new MapLookupExtractor(map, true), false, null, true, false) - ), - new DefaultDimensionSpec(QueryRunnerTestHelper.PLACEMENT_DIMENSION, "place") - ) - ) - .build(); - - Iterable> results = runner.run(QueryPlus.wrap(query)).toList(); - - String segmentIdInThisQuery = QueryRunnerTestHelper.SEGMENT_ID.toString(); - - List> expectedResultsAsc = Collections.singletonList( - new Result( - DateTimes.of("2011-01-12T00:00:00.000Z"), - new SelectResultValue( - ImmutableMap.of(segmentIdInThisQuery, 2), - Sets.newHashSet("mar", "qual", "place"), - Sets.newHashSet("index", "quality_uniques", "indexMin", "indexMaxPlusTen", "indexMinFloat", - "indexFloat", "indexMaxFloat" - ), - Arrays.asList( - new EventHolder( - segmentIdInThisQuery, - 0, - new ImmutableMap.Builder() - .put(EventHolder.TIMESTAMP_KEY, DateTimes.of("2011-01-12T00:00:00.000Z")) - .put("mar", "spot") - .put("qual", "automotive0") - .put("place", "preferred") - .put(QueryRunnerTestHelper.INDEX_METRIC, 100.000000F) - .build() - ), - new EventHolder( - segmentIdInThisQuery, - 1, - new ImmutableMap.Builder() - .put(EventHolder.TIMESTAMP_KEY, DateTimes.of("2011-01-12T00:00:00.000Z")) - .put("mar", "spot") - .put("qual", "business0") - .put("place", "preferred") - .put(QueryRunnerTestHelper.INDEX_METRIC, 100.000000F) - .build() - ), - new EventHolder( - segmentIdInThisQuery, - 2, - new ImmutableMap.Builder() - .put(EventHolder.TIMESTAMP_KEY, DateTimes.of("2011-01-12T00:00:00.000Z")) - .put("mar", "spot") - .put("qual", "entertainment0") - .put("place", "preferred") - .put(QueryRunnerTestHelper.INDEX_METRIC, 100.000000F) - .build() - ) - ) - ) - ) - ); - - List> expectedResultsDsc = Collections.singletonList( - new Result( - DateTimes.of("2011-01-12T00:00:00.000Z"), - new SelectResultValue( - ImmutableMap.of(segmentIdInThisQuery, -3), - Sets.newHashSet("mar", "qual", "place"), - Sets.newHashSet("index", "quality_uniques", "indexMin", "indexMaxPlusTen", "indexMinFloat", - "indexFloat", "indexMaxFloat" - ), - Arrays.asList( - new EventHolder( - segmentIdInThisQuery, - -1, - new ImmutableMap.Builder() - .put(EventHolder.TIMESTAMP_KEY, DateTimes.of("2011-04-15T00:00:00.000Z")) - .put("mar", "upfront") - .put("qual", "premium0") - .put("place", "preferred") - .put(QueryRunnerTestHelper.INDEX_METRIC, 780.27197265625F) - .build() - ), - new EventHolder( - segmentIdInThisQuery, - -2, - new ImmutableMap.Builder() - .put(EventHolder.TIMESTAMP_KEY, DateTimes.of("2011-04-15T00:00:00.000Z")) - .put("mar", "upfront") - .put("qual", "mezzanine0") - .put("place", "preferred") - .put(QueryRunnerTestHelper.INDEX_METRIC, 962.731201171875F) - .build() - ), - new EventHolder( - segmentIdInThisQuery, - -3, - new ImmutableMap.Builder() - .put(EventHolder.TIMESTAMP_KEY, DateTimes.of("2011-04-15T00:00:00.000Z")) - .put("mar", "total_market") - .put("qual", "premium0") - .put("place", "preferred") - .put(QueryRunnerTestHelper.INDEX_METRIC, 1029.0570068359375F) - .build() - ) - ) - ) - ) - ); - - verify(descending ? expectedResultsDsc : expectedResultsAsc, results); - } - - @Test - public void testSelectWithDimsAndMets() - { - SelectQuery query = newTestQuery() - .intervals(I_0112_0114_SPEC) - .dimensionSpecs(DefaultDimensionSpec.toSpec(QueryRunnerTestHelper.MARKET_DIMENSION)) - .metrics(Collections.singletonList(QueryRunnerTestHelper.INDEX_METRIC)) - .build(); - - Iterable> results = runner.run(QueryPlus.wrap(query)).toList(); - - PagingOffset offset = query.getPagingOffset(SEGMENT_ID_STRING); - List> expectedResults = toExpected( - SEGMENT_ID_STRING, - toEvents( - new String[]{ - EventHolder.TIMESTAMP_KEY + ":TIME", - QueryRunnerTestHelper.MARKET_DIMENSION + ":STRING", - null, - null, - null, - null, - null, - null, - QueryRunnerTestHelper.INDEX_METRIC + ":FLOAT" - }, - V_0112_0114 - ), - Collections.singletonList("market"), - Collections.singletonList("index"), - offset.startOffset(), - offset.threshold() - ); - verify(expectedResults, results); - } - - @Test - public void testSelectPagination() - { - SelectQuery query = newTestQuery() - .intervals(I_0112_0114_SPEC) - .dimensionSpecs(DefaultDimensionSpec.toSpec(QueryRunnerTestHelper.QUALITY_DIMENSION)) - .metrics(Collections.singletonList(QueryRunnerTestHelper.INDEX_METRIC)) - .pagingSpec(new PagingSpec(toPagingIdentifier(3, descending), 3)) - .build(); - - Iterable> results = runner.run(QueryPlus.wrap(query)).toList(); - - PagingOffset offset = query.getPagingOffset(SEGMENT_ID_STRING); - List> expectedResults = toExpected( - SEGMENT_ID_STRING, - toEvents( - new String[]{ - EventHolder.TIMESTAMP_KEY + ":TIME", - "foo:NULL", - "foo2:NULL" - }, - V_0112_0114 - ), - Collections.singletonList("quality"), - Collections.singletonList("index"), - offset.startOffset(), - offset.threshold() - ); - verify(expectedResults, results); - } - - @Test - public void testFullOnSelectWithFilter() - { - // startDelta + threshold pairs - for (int[] param : new int[][]{{3, 3}, {0, 1}, {5, 5}, {2, 7}, {3, 0}}) { - SelectQuery query = newTestQuery() - .intervals(I_0112_0114_SPEC) - .filters(new SelectorDimFilter(QueryRunnerTestHelper.MARKET_DIMENSION, "spot", null)) - .granularity(QueryRunnerTestHelper.DAY_GRAN) - .dimensionSpecs(DefaultDimensionSpec.toSpec(QueryRunnerTestHelper.QUALITY_DIMENSION)) - .metrics(Collections.singletonList(QueryRunnerTestHelper.INDEX_METRIC)) - .pagingSpec(new PagingSpec(toPagingIdentifier(param[0], descending), param[1])) - .build(); - - Iterable> results = runner.run(QueryPlus.wrap(query)).toList(); - - final List>> events = toEvents( - new String[]{ - EventHolder.TIMESTAMP_KEY + ":TIME", - null, - QueryRunnerTestHelper.QUALITY_DIMENSION + ":STRING", - null, - null, - QueryRunnerTestHelper.INDEX_METRIC + ":FLOAT" - }, - // filtered values with day granularity - new String[]{ - "2011-01-12T00:00:00.000Z\tspot\tautomotive\tpreferred\tapreferred\t100.000000", - "2011-01-12T00:00:00.000Z\tspot\tbusiness\tpreferred\tbpreferred\t100.000000", - "2011-01-12T00:00:00.000Z\tspot\tentertainment\tpreferred\tepreferred\t100.000000", - "2011-01-12T00:00:00.000Z\tspot\thealth\tpreferred\thpreferred\t100.000000", - "2011-01-12T00:00:00.000Z\tspot\tmezzanine\tpreferred\tmpreferred\t100.000000", - "2011-01-12T00:00:00.000Z\tspot\tnews\tpreferred\tnpreferred\t100.000000", - "2011-01-12T00:00:00.000Z\tspot\tpremium\tpreferred\tppreferred\t100.000000", - "2011-01-12T00:00:00.000Z\tspot\ttechnology\tpreferred\ttpreferred\t100.000000", - "2011-01-12T00:00:00.000Z\tspot\ttravel\tpreferred\ttpreferred\t100.000000" - }, - new String[]{ - "2011-01-13T00:00:00.000Z\tspot\tautomotive\tpreferred\tapreferred\t94.874713", - "2011-01-13T00:00:00.000Z\tspot\tbusiness\tpreferred\tbpreferred\t103.629399", - "2011-01-13T00:00:00.000Z\tspot\tentertainment\tpreferred\tepreferred\t110.087299", - "2011-01-13T00:00:00.000Z\tspot\thealth\tpreferred\thpreferred\t114.947403", - "2011-01-13T00:00:00.000Z\tspot\tmezzanine\tpreferred\tmpreferred\t104.465767", - "2011-01-13T00:00:00.000Z\tspot\tnews\tpreferred\tnpreferred\t102.851683", - "2011-01-13T00:00:00.000Z\tspot\tpremium\tpreferred\tppreferred\t108.863011", - "2011-01-13T00:00:00.000Z\tspot\ttechnology\tpreferred\ttpreferred\t111.356672", - "2011-01-13T00:00:00.000Z\tspot\ttravel\tpreferred\ttpreferred\t106.236928" - } - ); - - PagingOffset offset = query.getPagingOffset(SEGMENT_ID_STRING); - List> expectedResults = toExpected( - SEGMENT_ID_STRING, - events, - Collections.singletonList("quality"), - Collections.singletonList("index"), - offset.startOffset(), - offset.threshold() - ); - verify(expectedResults, results); - } - } - - @Test - public void testFullOnSelectWithFilterOnVirtualColumn() - { - Interval interval = Intervals.of("2011-01-13/2011-01-14"); - SelectQuery query = newTestQuery() - .intervals(new LegacySegmentSpec(interval)) - .filters( - new AndDimFilter( - Arrays.asList( - new SelectorDimFilter(QueryRunnerTestHelper.MARKET_DIMENSION, "spot", null), - new BoundDimFilter("expr", "11.1", null, false, false, null, null, StringComparators.NUMERIC) - ) - ) - ) - .granularity(QueryRunnerTestHelper.ALL_GRAN) - .dimensionSpecs(DefaultDimensionSpec.toSpec(QueryRunnerTestHelper.QUALITY_DIMENSION)) - .metrics(Collections.singletonList(QueryRunnerTestHelper.INDEX_METRIC)) - .pagingSpec(new PagingSpec(null, 10, true)) - .virtualColumns( - new ExpressionVirtualColumn("expr", "index / 10.0", ValueType.FLOAT, TestExprMacroTable.INSTANCE) - ) - .build(); - - Iterable> results = runner.run(QueryPlus.wrap(query)).toList(); - - final List>> events = toEvents( - new String[]{ - EventHolder.TIMESTAMP_KEY + ":TIME", - null, - QueryRunnerTestHelper.QUALITY_DIMENSION + ":STRING", - null, - null, - QueryRunnerTestHelper.INDEX_METRIC + ":FLOAT" - }, - // filtered values with all granularity - new String[]{ - "2011-01-13T00:00:00.000Z\tspot\thealth\tpreferred\thpreferred\t114.947403", - "2011-01-13T00:00:00.000Z\tspot\ttechnology\tpreferred\ttpreferred\t111.356672" - } - ); - - String segmentIdInThisQuery = QueryRunnerTestHelper.SEGMENT_ID.withInterval(interval).toString(); - PagingOffset offset = query.getPagingOffset(segmentIdInThisQuery); - List> expectedResults = toExpected( - segmentIdInThisQuery, - events, - Collections.singletonList("quality"), - Collections.singletonList("index"), - offset.startOffset(), - offset.threshold() - ); - verify(expectedResults, results); - } - - @Test - public void testSelectWithFilterLookupExtractionFn() - { - - Map extractionMap = new HashMap<>(); - extractionMap.put("total_market", "replaced"); - MapLookupExtractor mapLookupExtractor = new MapLookupExtractor(extractionMap, false); - LookupExtractionFn lookupExtractionFn = new LookupExtractionFn(mapLookupExtractor, false, null, true, true); - SelectQuery query = newTestQuery() - .intervals(I_0112_0114_SPEC) - .filters(new SelectorDimFilter(QueryRunnerTestHelper.MARKET_DIMENSION, "replaced", lookupExtractionFn)) - .granularity(QueryRunnerTestHelper.DAY_GRAN) - .dimensionSpecs(DefaultDimensionSpec.toSpec(QueryRunnerTestHelper.QUALITY_DIMENSION)) - .metrics(Collections.singletonList(QueryRunnerTestHelper.INDEX_METRIC)) - .build(); - - Iterable> results = runner.run(QueryPlus.wrap(query)).toList(); - Iterable> resultsOptimize = TOOL_CHEST - .postMergeQueryDecoration(TOOL_CHEST.mergeResults(TOOL_CHEST.preMergeQueryDecoration(runner))) - .run(QueryPlus.wrap(query)) - .toList(); - - final List>> events = toEvents( - new String[]{ - EventHolder.TIMESTAMP_KEY + ":TIME", - null, - QueryRunnerTestHelper.QUALITY_DIMENSION + ":STRING", - null, - null, - QueryRunnerTestHelper.INDEX_METRIC + ":FLOAT" - }, - // filtered values with day granularity - new String[]{ - "2011-01-12T00:00:00.000Z\ttotal_market\tmezzanine\tpreferred\tmpreferred\t1000.000000", - "2011-01-12T00:00:00.000Z\ttotal_market\tpremium\tpreferred\tppreferred\t1000.000000" - }, - new String[]{ - "2011-01-13T00:00:00.000Z\ttotal_market\tmezzanine\tpreferred\tmpreferred\t1040.945505", - "2011-01-13T00:00:00.000Z\ttotal_market\tpremium\tpreferred\tppreferred\t1689.012875" - } - ); - - PagingOffset offset = query.getPagingOffset(SEGMENT_ID_STRING); - List> expectedResults = toExpected( - SEGMENT_ID_STRING, - events, - Collections.singletonList(QueryRunnerTestHelper.QUALITY_DIMENSION), - Collections.singletonList(QueryRunnerTestHelper.INDEX_METRIC), - offset.startOffset(), - offset.threshold() - ); - - verify(expectedResults, results); - verify(expectedResults, resultsOptimize); - } - - @Test - public void testFullSelectNoResults() - { - SelectQuery query = newTestQuery() - .intervals(I_0112_0114_SPEC) - .filters( - new AndDimFilter( - Arrays.asList( - new SelectorDimFilter(QueryRunnerTestHelper.MARKET_DIMENSION, "spot", null), - new SelectorDimFilter(QueryRunnerTestHelper.MARKET_DIMENSION, "foo", null) - ) - ) - ) - .build(); - - Iterable> results = runner.run(QueryPlus.wrap(query)).toList(); - - List> expectedResults = Collections.singletonList( - new Result( - DateTimes.of("2011-01-12T00:00:00.000Z"), - new SelectResultValue( - ImmutableMap.of(), - Sets.newHashSet( - "market", - "quality", - "qualityLong", - "qualityFloat", - "qualityDouble", - "qualityNumericString", - "placement", - "placementish", - "partial_null_column", - "null_column" - ), - Sets.newHashSet( - "index", - "quality_uniques", - "indexMin", - "indexMaxPlusTen", - "indexMinFloat", - "indexFloat", - "indexMaxFloat" - ), - new ArrayList<>() - ) - ) - ); - - verify(expectedResults, populateNullColumnAtLastForQueryableIndexCase(results, "null_column")); - } - - @Test - public void testFullSelectNoDimensionAndMetric() - { - SelectQuery query = newTestQuery() - .intervals(I_0112_0114_SPEC) - .dimensionSpecs(DefaultDimensionSpec.toSpec("foo")) - .metrics(Collections.singletonList("foo2")) - .build(); - - Iterable> results = runner.run(QueryPlus.wrap(query)).toList(); - - final List>> events = toEvents( - new String[]{ - EventHolder.TIMESTAMP_KEY + ":TIME", - "foo:NULL", - "foo2:NULL" - }, - V_0112_0114 - ); - - PagingOffset offset = query.getPagingOffset(SEGMENT_ID_STRING); - List> expectedResults = toExpected( - SEGMENT_ID_STRING, - events, - Collections.singletonList("foo"), - Collections.singletonList("foo2"), - offset.startOffset(), - offset.threshold() - ); - verify(expectedResults, results); - } - - @Test - public void testFullOnSelectWithLongAndFloat() - { - List dimSpecs = Arrays.asList( - new DefaultDimensionSpec(QueryRunnerTestHelper.INDEX_METRIC, "floatIndex", ValueType.FLOAT), - new DefaultDimensionSpec(ColumnHolder.TIME_COLUMN_NAME, "longTime", ValueType.LONG) - ); - - SelectQuery query = newTestQuery() - .dimensionSpecs(dimSpecs) - .metrics(Arrays.asList(ColumnHolder.TIME_COLUMN_NAME, "index")) - .intervals(I_0112_0114_SPEC) - .build(); - - Iterable> results = runner.run(QueryPlus.wrap(query)).toList(); - - List> expectedResultsAsc = Collections.singletonList( - new Result( - DateTimes.of("2011-01-12T00:00:00.000Z"), - new SelectResultValue( - ImmutableMap.of(SEGMENT_ID_STRING, 2), - Sets.newHashSet("null_column", "floatIndex", "longTime"), - Sets.newHashSet("__time", "index"), - Arrays.asList( - new EventHolder( - SEGMENT_ID_STRING, - 0, - new ImmutableMap.Builder() - .put(EventHolder.TIMESTAMP_KEY, DateTimes.of("2011-01-12T00:00:00.000Z")) - .put("longTime", 1294790400000L) - .put("floatIndex", 100.0f) - .put(QueryRunnerTestHelper.INDEX_METRIC, 100.000000F) - .put(ColumnHolder.TIME_COLUMN_NAME, 1294790400000L) - .build() - ), - new EventHolder( - SEGMENT_ID_STRING, - 1, - new ImmutableMap.Builder() - .put(EventHolder.TIMESTAMP_KEY, DateTimes.of("2011-01-12T00:00:00.000Z")) - .put("longTime", 1294790400000L) - .put("floatIndex", 100.0f) - .put(QueryRunnerTestHelper.INDEX_METRIC, 100.000000F) - .put(ColumnHolder.TIME_COLUMN_NAME, 1294790400000L) - .build() - ), - new EventHolder( - SEGMENT_ID_STRING, - 2, - new ImmutableMap.Builder() - .put(EventHolder.TIMESTAMP_KEY, DateTimes.of("2011-01-12T00:00:00.000Z")) - .put("longTime", 1294790400000L) - .put("floatIndex", 100.0f) - .put(QueryRunnerTestHelper.INDEX_METRIC, 100.000000F) - .put(ColumnHolder.TIME_COLUMN_NAME, 1294790400000L) - .build() - ) - ) - ) - ) - ); - - List> expectedResultsDsc = Collections.singletonList( - new Result( - DateTimes.of("2011-01-12T00:00:00.000Z"), - new SelectResultValue( - ImmutableMap.of(SEGMENT_ID_STRING, -3), - Sets.newHashSet("null_column", "floatIndex", "longTime"), - Sets.newHashSet("__time", "index"), - Arrays.asList( - new EventHolder( - SEGMENT_ID_STRING, - -1, - new ImmutableMap.Builder() - .put(EventHolder.TIMESTAMP_KEY, DateTimes.of("2011-01-13T00:00:00.000Z")) - .put("longTime", 1294876800000L) - .put("floatIndex", 1564.6177f) - .put(QueryRunnerTestHelper.INDEX_METRIC, 1564.6177f) - .put(ColumnHolder.TIME_COLUMN_NAME, 1294876800000L) - .build() - ), - new EventHolder( - SEGMENT_ID_STRING, - -2, - new ImmutableMap.Builder() - .put(EventHolder.TIMESTAMP_KEY, DateTimes.of("2011-01-13T00:00:00.000Z")) - .put("longTime", 1294876800000L) - .put("floatIndex", 826.0602f) - .put(QueryRunnerTestHelper.INDEX_METRIC, 826.0602f) - .put(ColumnHolder.TIME_COLUMN_NAME, 1294876800000L) - .build() - ), - new EventHolder( - SEGMENT_ID_STRING, - -3, - new ImmutableMap.Builder() - .put(EventHolder.TIMESTAMP_KEY, DateTimes.of("2011-01-13T00:00:00.000Z")) - .put("longTime", 1294876800000L) - .put("floatIndex", 1689.0128f) - .put(QueryRunnerTestHelper.INDEX_METRIC, 1689.0128f) - .put(ColumnHolder.TIME_COLUMN_NAME, 1294876800000L) - .build() - ) - ) - ) - ) - ); - - verify(descending ? expectedResultsDsc : expectedResultsAsc, populateNullColumnAtLastForQueryableIndexCase(results, "null_column")); - } - - @Test - public void testFullOnSelectWithLongAndFloatWithExFn() - { - String jsFn = "function(str) { return 'super-' + str; }"; - ExtractionFn jsExtractionFn = new JavaScriptExtractionFn(jsFn, false, JavaScriptConfig.getEnabledInstance()); - - List dimSpecs = Arrays.asList( - new ExtractionDimensionSpec(QueryRunnerTestHelper.INDEX_METRIC, "floatIndex", jsExtractionFn), - new ExtractionDimensionSpec(ColumnHolder.TIME_COLUMN_NAME, "longTime", jsExtractionFn) - ); - - SelectQuery query = newTestQuery() - .dimensionSpecs(dimSpecs) - .metrics(Arrays.asList(ColumnHolder.TIME_COLUMN_NAME, "index")) - .intervals(I_0112_0114_SPEC) - .build(); - - Iterable> results = runner.run(QueryPlus.wrap(query)).toList(); - - List> expectedResultsAsc = Collections.singletonList( - new Result( - DateTimes.of("2011-01-12T00:00:00.000Z"), - new SelectResultValue( - ImmutableMap.of(SEGMENT_ID_STRING, 2), - Sets.newHashSet("null_column", "floatIndex", "longTime"), - Sets.newHashSet("__time", "index"), - Arrays.asList( - new EventHolder( - SEGMENT_ID_STRING, - 0, - new ImmutableMap.Builder() - .put(EventHolder.TIMESTAMP_KEY, DateTimes.of("2011-01-12T00:00:00.000Z")) - .put("longTime", "super-1294790400000") - .put("floatIndex", "super-100") - .put(QueryRunnerTestHelper.INDEX_METRIC, 100.000000F) - .put(ColumnHolder.TIME_COLUMN_NAME, 1294790400000L) - .build() - ), - new EventHolder( - SEGMENT_ID_STRING, - 1, - new ImmutableMap.Builder() - .put(EventHolder.TIMESTAMP_KEY, DateTimes.of("2011-01-12T00:00:00.000Z")) - .put("longTime", "super-1294790400000") - .put("floatIndex", "super-100") - .put(QueryRunnerTestHelper.INDEX_METRIC, 100.000000F) - .put(ColumnHolder.TIME_COLUMN_NAME, 1294790400000L) - .build() - ), - new EventHolder( - SEGMENT_ID_STRING, - 2, - new ImmutableMap.Builder() - .put(EventHolder.TIMESTAMP_KEY, DateTimes.of("2011-01-12T00:00:00.000Z")) - .put("longTime", "super-1294790400000") - .put("floatIndex", "super-100") - .put(QueryRunnerTestHelper.INDEX_METRIC, 100.000000F) - .put(ColumnHolder.TIME_COLUMN_NAME, 1294790400000L) - .build() - ) - ) - ) - ) - ); - - List> expectedResultsDsc = Collections.singletonList( - new Result( - DateTimes.of("2011-01-12T00:00:00.000Z"), - new SelectResultValue( - ImmutableMap.of(SEGMENT_ID_STRING, -3), - Sets.newHashSet("null_column", "floatIndex", "longTime"), - Sets.newHashSet("__time", "index"), - Arrays.asList( - new EventHolder( - SEGMENT_ID_STRING, - -1, - new ImmutableMap.Builder() - .put(EventHolder.TIMESTAMP_KEY, DateTimes.of("2011-01-13T00:00:00.000Z")) - .put("longTime", "super-1294876800000") - .put("floatIndex", "super-1564.617729") - .put(QueryRunnerTestHelper.INDEX_METRIC, 1564.6177f) - .put(ColumnHolder.TIME_COLUMN_NAME, 1294876800000L) - .build() - ), - new EventHolder( - SEGMENT_ID_STRING, - -2, - new ImmutableMap.Builder() - .put(EventHolder.TIMESTAMP_KEY, DateTimes.of("2011-01-13T00:00:00.000Z")) - .put("longTime", "super-1294876800000") - .put("floatIndex", "super-826.060182") - .put(QueryRunnerTestHelper.INDEX_METRIC, 826.0602f) - .put(ColumnHolder.TIME_COLUMN_NAME, 1294876800000L) - .build() - ), - new EventHolder( - SEGMENT_ID_STRING, - -3, - new ImmutableMap.Builder() - .put(EventHolder.TIMESTAMP_KEY, DateTimes.of("2011-01-13T00:00:00.000Z")) - .put("longTime", "super-1294876800000") - .put("floatIndex", "super-1689.012875") - .put(QueryRunnerTestHelper.INDEX_METRIC, 1689.0128f) - .put(ColumnHolder.TIME_COLUMN_NAME, 1294876800000L) - .build() - ) - ) - ) - ) - ); - - verify(descending ? expectedResultsDsc : expectedResultsAsc, populateNullColumnAtLastForQueryableIndexCase(results, "null_column")); - } - - private Map toPagingIdentifier(int startDelta, boolean descending) - { - return ImmutableMap.of(SEGMENT_ID_STRING, PagingOffset.toOffset(startDelta, descending)); - } - - private List>> toFullEvents(final String[]... valueSet) - { - return toEvents(new String[]{EventHolder.TIMESTAMP_KEY + ":TIME", - QueryRunnerTestHelper.MARKET_DIMENSION + ":STRING", - QueryRunnerTestHelper.QUALITY_DIMENSION + ":STRING", - "qualityLong" + ":LONG", - "qualityFloat" + ":FLOAT", - "qualityNumericString" + ":STRING", - QueryRunnerTestHelper.PLACEMENT_DIMENSION + ":STRING", - QueryRunnerTestHelper.PLACEMENTISH_DIMENSION + ":STRINGS", - QueryRunnerTestHelper.INDEX_METRIC + ":FLOAT", - QueryRunnerTestHelper.PARTIAL_NULL_DIMENSION + ":STRING"}, - valueSet); - } - - private List>> toEvents(final String[] dimSpecs, final String[]... valueSet) - { - List>> events = new ArrayList<>(); - for (String[] values : valueSet) { - events.add( - Lists.newArrayList( - Iterables.transform( - Arrays.asList(values), new Function>() - { - @Override - public Map apply(String input) - { - Map event = new HashMap<>(); - String[] values = input.split("\\t"); - for (int i = 0; i < dimSpecs.length; i++) { - if (dimSpecs[i] == null || i >= dimSpecs.length || i >= values.length) { - continue; - } - String[] specs = dimSpecs[i].split(":"); - event.put( - specs[0], - specs.length == 1 || specs[1].equals("STRING") ? values[i] : - specs[1].equals("TIME") ? DateTimes.of(values[i]) : - specs[1].equals("FLOAT") ? Float.valueOf(values[i]) : - specs[1].equals("DOUBLE") ? Double.valueOf(values[i]) : - specs[1].equals("LONG") ? Long.valueOf(values[i]) : - specs[1].equals("NULL") ? null : - specs[1].equals("STRINGS") ? Arrays.asList(values[i].split("\u0001")) : - values[i] - ); - } - return event; - } - } - ) - ) - ); - } - return events; - } - - private List> toExpected( - String segmentId, - List>> targets, - List dimensions, - List metrics, - final int offset, - final int threshold - ) - { - if (offset < 0) { - targets = Lists.reverse(targets); - } - List> expected = Lists.newArrayListWithExpectedSize(targets.size()); - for (List> group : targets) { - List holders = Lists.newArrayListWithExpectedSize(threshold); - int newOffset = offset; - if (offset < 0) { - int start = group.size() + offset; - int end = Math.max(-1, start - threshold); - for (int i = start; i > end; i--) { - holders.add(new EventHolder(segmentId, newOffset--, group.get(i))); - } - } else { - int end = Math.min(group.size(), offset + threshold); - for (int i = offset; i < end; i++) { - holders.add(new EventHolder(segmentId, newOffset++, group.get(i))); - } - } - int lastOffset = holders.isEmpty() ? offset : holders.get(holders.size() - 1).getOffset(); - expected.add( - new Result( - new DateTime(group.get(0).get(EventHolder.TIMESTAMP_KEY), ISOChronology.getInstanceUTC()), - new SelectResultValue( - ImmutableMap.of(segmentId, lastOffset), - Sets.newHashSet(dimensions), - Sets.newHashSet(metrics), - holders) - ) - ); - } - return expected; - } - - private static void verify( - Iterable> expectedResults, - Iterable> actualResults - ) - { - Iterator> expectedIter = expectedResults.iterator(); - Iterator> actualIter = actualResults.iterator(); - - while (expectedIter.hasNext()) { - Result expected = expectedIter.next(); - Result actual = actualIter.next(); - - Assert.assertEquals(expected.getTimestamp(), actual.getTimestamp()); - - for (Map.Entry entry : expected.getValue().getPagingIdentifiers().entrySet()) { - Assert.assertEquals(entry.getValue(), actual.getValue().getPagingIdentifiers().get(entry.getKey())); - } - - Assert.assertEquals(expected.getValue().getDimensions(), actual.getValue().getDimensions()); - Assert.assertEquals(expected.getValue().getMetrics(), actual.getValue().getMetrics()); - - Iterator expectedEvts = expected.getValue().getEvents().iterator(); - Iterator actualEvts = actual.getValue().getEvents().iterator(); - - while (expectedEvts.hasNext()) { - EventHolder exHolder = expectedEvts.next(); - EventHolder acHolder = actualEvts.next(); - - Assert.assertEquals(exHolder.getTimestamp(), acHolder.getTimestamp()); - Assert.assertEquals(exHolder.getOffset(), acHolder.getOffset()); - - for (Map.Entry ex : exHolder.getEvent().entrySet()) { - Object actVal = acHolder.getEvent().get(ex.getKey()); - - // work around for current II limitations - if (acHolder.getEvent().get(ex.getKey()) instanceof Double) { - actVal = ((Double) actVal).floatValue(); - } - Assert.assertEquals("invalid value for " + ex.getKey(), ex.getValue(), actVal); - } - } - - if (actualEvts.hasNext()) { - throw new ISE("This event iterator should be exhausted!"); - } - } - - if (actualIter.hasNext()) { - throw new ISE("This iterator should be exhausted!"); - } - } - - private static Iterable> populateNullColumnAtLastForQueryableIndexCase(Iterable> results, String columnName) - { - // A Queryable index does not have the null column when it has loaded a index. - for (Result value : results) { - Set dimensions = value.getValue().getDimensions(); - if (dimensions.contains(columnName)) { - break; - } - dimensions.add(columnName); - } - - return results; - } - -} diff --git a/processing/src/test/java/org/apache/druid/query/select/SelectQuerySpecTest.java b/processing/src/test/java/org/apache/druid/query/select/SelectQuerySpecTest.java deleted file mode 100644 index ba37c28b8f1a..000000000000 --- a/processing/src/test/java/org/apache/druid/query/select/SelectQuerySpecTest.java +++ /dev/null @@ -1,155 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.druid.query.select; - -import com.fasterxml.jackson.databind.InjectableValues; -import com.fasterxml.jackson.databind.ObjectMapper; -import org.apache.druid.jackson.DefaultObjectMapper; -import org.apache.druid.java.util.common.Intervals; -import org.apache.druid.query.QueryRunnerTestHelper; -import org.apache.druid.query.TableDataSource; -import org.apache.druid.query.dimension.DefaultDimensionSpec; -import org.apache.druid.query.spec.LegacySegmentSpec; -import org.junit.Assert; -import org.junit.Test; - -import java.util.Arrays; -import java.util.Collections; - -/** - */ -public class SelectQuerySpecTest -{ - private final ObjectMapper objectMapper = new DefaultObjectMapper(); - - { - objectMapper.setInjectableValues( - new InjectableValues.Std().addValue(SelectQueryConfig.class, new SelectQueryConfig(true)) - ); - } - - @Test - public void testSerializationLegacyString() throws Exception - { - String legacy = - "{\"queryType\":\"select\",\"dataSource\":{\"type\":\"table\",\"name\":\"testing\"}," - + "\"intervals\":{\"type\":\"LegacySegmentSpec\",\"intervals\":[\"2011-01-12T00:00:00.000Z/2011-01-14T00:00:00.000Z\"]}," - + "\"descending\":true," - + "\"filter\":null," - + "\"granularity\":{\"type\":\"all\"}," - + "\"dimensions\":[\"market\",\"quality\"]," - + "\"metrics\":[\"index\"]," - + "\"virtualColumns\":null," - + "\"pagingSpec\":{\"pagingIdentifiers\":{},\"threshold\":3}," - + "\"context\":null}"; - - String current = - "{\"queryType\":\"select\",\"dataSource\":{\"type\":\"table\",\"name\":\"testing\"}," - + "\"intervals\":{\"type\":\"LegacySegmentSpec\",\"intervals\":[\"2011-01-12T00:00:00.000Z/2011-01-14T00:00:00.000Z\"]}," - + "\"descending\":true," - + "\"filter\":null," - + "\"granularity\":{\"type\":\"all\"}," - + "\"dimensions\":" - + "[{\"type\":\"default\",\"dimension\":\"market\",\"outputName\":\"market\",\"outputType\":\"STRING\"}," - + "{\"type\":\"default\",\"dimension\":\"quality\",\"outputName\":\"quality\",\"outputType\":\"STRING\"}]," - + "\"metrics\":[\"index\"]," - + "\"virtualColumns\":[]," - + "\"pagingSpec\":{\"pagingIdentifiers\":{},\"threshold\":3,\"fromNext\":true}," - + "\"context\":null}"; - - SelectQuery query = new SelectQuery( - new TableDataSource(QueryRunnerTestHelper.DATA_SOURCE), - new LegacySegmentSpec(Intervals.of("2011-01-12/2011-01-14")), - true, - null, - QueryRunnerTestHelper.ALL_GRAN, - DefaultDimensionSpec.toSpec(Arrays.asList("market", "quality")), - Collections.singletonList("index"), - null, - new PagingSpec(null, 3, null), - null - ); - - String actual = objectMapper.writeValueAsString(query); - Assert.assertEquals(current, actual); - Assert.assertEquals(query, objectMapper.readValue(actual, SelectQuery.class)); - Assert.assertEquals(query, objectMapper.readValue(legacy, SelectQuery.class)); - } - - @Test - public void testPagingSpecFromNext() throws Exception - { - String baseQueryJson = - "{\"queryType\":\"select\",\"dataSource\":{\"type\":\"table\",\"name\":\"testing\"}," - + "\"intervals\":{\"type\":\"LegacySegmentSpec\",\"intervals\":[\"2011-01-12T00:00:00.000Z/2011-01-14T00:00:00.000Z\"]}," - + "\"descending\":true," - + "\"filter\":null," - + "\"granularity\":{\"type\":\"all\"}," - + "\"dimensions\":" - + "[{\"type\":\"default\",\"dimension\":\"market\",\"outputName\":\"market\",\"outputType\":\"STRING\"}," - + "{\"type\":\"default\",\"dimension\":\"quality\",\"outputName\":\"quality\",\"outputType\":\"STRING\"}]," - + "\"metrics\":[\"index\"]," - + "\"virtualColumns\":[],"; - - String withFalse = - baseQueryJson - + "\"pagingSpec\":{\"pagingIdentifiers\":{},\"threshold\":3,\"fromNext\":false}," - + "\"context\":null}"; - - String withTrue = - baseQueryJson - + "\"pagingSpec\":{\"pagingIdentifiers\":{},\"threshold\":3,\"fromNext\":true}," - + "\"context\":null}"; - - SelectQuery queryWithNull = new SelectQuery( - new TableDataSource(QueryRunnerTestHelper.DATA_SOURCE), - new LegacySegmentSpec(Intervals.of("2011-01-12/2011-01-14")), - true, - null, - QueryRunnerTestHelper.ALL_GRAN, - DefaultDimensionSpec.toSpec(Arrays.asList("market", "quality")), - Collections.singletonList("index"), - null, - new PagingSpec(null, 3, null), - null - ); - - SelectQuery queryWithFalse = queryWithNull.withPagingSpec( - new PagingSpec(null, 3, false) - ); - - SelectQuery queryWithTrue = queryWithNull.withPagingSpec( - new PagingSpec(null, 3, true) - ); - - String actualWithNull = objectMapper.writeValueAsString(queryWithNull); - Assert.assertEquals(withTrue, actualWithNull); - - String actualWithFalse = objectMapper.writeValueAsString(queryWithFalse); - Assert.assertEquals(withFalse, actualWithFalse); - - String actualWithTrue = objectMapper.writeValueAsString(queryWithTrue); - Assert.assertEquals(withTrue, actualWithTrue); - - Assert.assertEquals(queryWithNull, objectMapper.readValue(actualWithNull, SelectQuery.class)); - Assert.assertEquals(queryWithFalse, objectMapper.readValue(actualWithFalse, SelectQuery.class)); - Assert.assertEquals(queryWithTrue, objectMapper.readValue(actualWithTrue, SelectQuery.class)); - } -} diff --git a/processing/src/test/java/org/apache/druid/query/select/SelectQueryTest.java b/processing/src/test/java/org/apache/druid/query/select/SelectQueryTest.java new file mode 100644 index 000000000000..d3814007428e --- /dev/null +++ b/processing/src/test/java/org/apache/druid/query/select/SelectQueryTest.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.select; + +import com.fasterxml.jackson.databind.JsonMappingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.druid.jackson.DefaultObjectMapper; +import org.apache.druid.java.util.common.StringUtils; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; + +public class SelectQueryTest +{ + private static final String SOME_QUERY_THAT_IS_DOOMED_TO_FAIL = + "{\n" + + " \"queryType\": \"select\",\n" + + " \"dataSource\": \"wikipedia\",\n" + + " \"descending\": \"false\",\n" + + " \"dimensions\":[],\n" + + " \"metrics\":[],\n" + + " \"granularity\": \"all\",\n" + + " \"intervals\": [\n" + + " \"2013-01-01/2013-01-02\"\n" + + " ],\n" + + " \"pagingSpec\":{\"pagingIdentifiers\": {}, \"threshold\":5}\n" + + " }"; + + private final ObjectMapper objectMapper = new DefaultObjectMapper(); + + @Rule + public ExpectedException expectedException = ExpectedException.none(); + + @Test + public void testSerde() throws Exception + { + final String exceptionMessage = + StringUtils.format( + "Instantiation of [simple type, class org.apache.druid.query.select.SelectQuery] value failed: %s", + SelectQuery.REMOVED_ERROR_MESSAGE + ); + expectedException.expect(JsonMappingException.class); + expectedException.expectMessage(exceptionMessage); + objectMapper.readValue(SOME_QUERY_THAT_IS_DOOMED_TO_FAIL, SelectQuery.class); + } +} diff --git a/server/src/main/java/org/apache/druid/client/cache/CacheConfig.java b/server/src/main/java/org/apache/druid/client/cache/CacheConfig.java index 3450057c6bf2..5b68466ca4e9 100644 --- a/server/src/main/java/org/apache/druid/client/cache/CacheConfig.java +++ b/server/src/main/java/org/apache/druid/client/cache/CacheConfig.java @@ -55,7 +55,7 @@ public class CacheConfig private int maxEntrySize = 1_000_000; @JsonProperty - private List unCacheable = ImmutableList.of(Query.SELECT); + private List unCacheable = ImmutableList.of(); @JsonProperty private int resultLevelCacheLimit = Integer.MAX_VALUE; diff --git a/server/src/main/java/org/apache/druid/guice/QueryRunnerFactoryModule.java b/server/src/main/java/org/apache/druid/guice/QueryRunnerFactoryModule.java index 53939aeb3797..6071973cc37a 100644 --- a/server/src/main/java/org/apache/druid/guice/QueryRunnerFactoryModule.java +++ b/server/src/main/java/org/apache/druid/guice/QueryRunnerFactoryModule.java @@ -36,8 +36,6 @@ import org.apache.druid.query.scan.ScanQueryRunnerFactory; import org.apache.druid.query.search.SearchQuery; import org.apache.druid.query.search.SearchQueryRunnerFactory; -import org.apache.druid.query.select.SelectQuery; -import org.apache.druid.query.select.SelectQueryRunnerFactory; import org.apache.druid.query.timeboundary.TimeBoundaryQuery; import org.apache.druid.query.timeboundary.TimeBoundaryQueryRunnerFactory; import org.apache.druid.query.timeseries.TimeseriesQuery; @@ -60,7 +58,6 @@ public class QueryRunnerFactoryModule extends QueryToolChestModule .put(SegmentMetadataQuery.class, SegmentMetadataQueryRunnerFactory.class) .put(GroupByQuery.class, GroupByQueryRunnerFactory.class) .put(ScanQuery.class, ScanQueryRunnerFactory.class) - .put(SelectQuery.class, SelectQueryRunnerFactory.class) .put(TopNQuery.class, TopNQueryRunnerFactory.class) .put(DataSourceMetadataQuery.class, DataSourceMetadataQueryRunnerFactory.class) .build(); diff --git a/server/src/main/java/org/apache/druid/guice/QueryToolChestModule.java b/server/src/main/java/org/apache/druid/guice/QueryToolChestModule.java index 1bb0d59ae98d..2fb685c31c97 100644 --- a/server/src/main/java/org/apache/druid/guice/QueryToolChestModule.java +++ b/server/src/main/java/org/apache/druid/guice/QueryToolChestModule.java @@ -49,11 +49,6 @@ import org.apache.druid.query.search.SearchQueryConfig; import org.apache.druid.query.search.SearchQueryMetricsFactory; import org.apache.druid.query.search.SearchQueryQueryToolChest; -import org.apache.druid.query.select.DefaultSelectQueryMetricsFactory; -import org.apache.druid.query.select.SelectQuery; -import org.apache.druid.query.select.SelectQueryConfig; -import org.apache.druid.query.select.SelectQueryMetricsFactory; -import org.apache.druid.query.select.SelectQueryQueryToolChest; import org.apache.druid.query.timeboundary.TimeBoundaryQuery; import org.apache.druid.query.timeboundary.TimeBoundaryQueryQueryToolChest; import org.apache.druid.query.timeseries.DefaultTimeseriesQueryMetricsFactory; @@ -76,7 +71,6 @@ public class QueryToolChestModule implements Module public static final String GROUPBY_QUERY_METRICS_FACTORY_PROPERTY = "druid.query.groupBy.queryMetricsFactory"; public static final String TIMESERIES_QUERY_METRICS_FACTORY_PROPERTY = "druid.query.timeseries.queryMetricsFactory"; public static final String TOPN_QUERY_METRICS_FACTORY_PROPERTY = "druid.query.topN.queryMetricsFactory"; - public static final String SELECT_QUERY_METRICS_FACTORY_PROPERTY = "druid.query.select.queryMetricsFactory"; public static final String SEARCH_QUERY_METRICS_FACTORY_PROPERTY = "druid.query.search.queryMetricsFactory"; public final Map, Class> mappings = @@ -87,7 +81,6 @@ public class QueryToolChestModule implements Module .put(SegmentMetadataQuery.class, SegmentMetadataQueryQueryToolChest.class) .put(GroupByQuery.class, GroupByQueryQueryToolChest.class) .put(ScanQuery.class, ScanQueryQueryToolChest.class) - .put(SelectQuery.class, SelectQueryQueryToolChest.class) .put(TopNQuery.class, TopNQueryQueryToolChest.class) .put(DataSourceMetadataQuery.class, DataSourceQueryQueryToolChest.class) .build(); @@ -109,7 +102,6 @@ public void configure(Binder binder) JsonConfigProvider.bind(binder, "druid.query.search", SearchQueryConfig.class); JsonConfigProvider.bind(binder, "druid.query.topN", TopNQueryConfig.class); JsonConfigProvider.bind(binder, "druid.query.segmentMetadata", SegmentMetadataQueryConfig.class); - JsonConfigProvider.bind(binder, "druid.query.select", SelectQueryConfig.class); JsonConfigProvider.bind(binder, "druid.query.scan", ScanQueryConfig.class); PolyBind.createChoice( @@ -156,17 +148,6 @@ public void configure(Binder binder) .addBinding("default") .to(DefaultTopNQueryMetricsFactory.class); - PolyBind.createChoice( - binder, - SELECT_QUERY_METRICS_FACTORY_PROPERTY, - Key.get(SelectQueryMetricsFactory.class), - Key.get(DefaultSelectQueryMetricsFactory.class) - ); - PolyBind - .optionBinder(binder, Key.get(SelectQueryMetricsFactory.class)) - .addBinding("default") - .to(DefaultSelectQueryMetricsFactory.class); - PolyBind.createChoice( binder, SEARCH_QUERY_METRICS_FACTORY_PROPERTY, diff --git a/server/src/main/java/org/apache/druid/segment/realtime/firehose/IngestSegmentFirehose.java b/server/src/main/java/org/apache/druid/segment/realtime/firehose/IngestSegmentFirehose.java index 477a55037f10..c0064a25f64c 100644 --- a/server/src/main/java/org/apache/druid/segment/realtime/firehose/IngestSegmentFirehose.java +++ b/server/src/main/java/org/apache/druid/segment/realtime/firehose/IngestSegmentFirehose.java @@ -25,6 +25,7 @@ import org.apache.druid.data.input.Firehose; import org.apache.druid.data.input.InputRow; import org.apache.druid.data.input.MapBasedInputRow; +import org.apache.druid.data.input.impl.TimestampSpec; import org.apache.druid.java.util.common.DateTimes; import org.apache.druid.java.util.common.granularity.Granularities; import org.apache.druid.java.util.common.guava.Sequence; @@ -33,7 +34,6 @@ import org.apache.druid.java.util.common.guava.Yielders; import org.apache.druid.query.dimension.DefaultDimensionSpec; import org.apache.druid.query.filter.DimFilter; -import org.apache.druid.query.select.EventHolder; import org.apache.druid.segment.BaseLongColumnValueSelector; import org.apache.druid.segment.BaseObjectColumnValueSelector; import org.apache.druid.segment.Cursor; @@ -132,7 +132,7 @@ public InputRow next() { final Map theEvent = Maps.newLinkedHashMap(); final long timestamp = timestampColumnSelector.getLong(); - theEvent.put(EventHolder.TIMESTAMP_KEY, DateTimes.utc(timestamp)); + theEvent.put(TimestampSpec.DEFAULT_COLUMN, DateTimes.utc(timestamp)); for (Map.Entry dimSelector : dimSelectors.entrySet()) { diff --git a/server/src/test/java/org/apache/druid/client/CachingClusteredClientFunctionalityTest.java b/server/src/test/java/org/apache/druid/client/CachingClusteredClientFunctionalityTest.java index 9a7bb900a5fa..84534d28e7d4 100644 --- a/server/src/test/java/org/apache/druid/client/CachingClusteredClientFunctionalityTest.java +++ b/server/src/test/java/org/apache/druid/client/CachingClusteredClientFunctionalityTest.java @@ -20,8 +20,6 @@ package org.apache.druid.client; import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.common.base.Supplier; -import com.google.common.base.Suppliers; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; import com.google.common.collect.Ordering; @@ -48,7 +46,6 @@ import org.apache.druid.query.QueryToolChestWarehouse; import org.apache.druid.query.aggregation.CountAggregatorFactory; import org.apache.druid.query.context.ResponseContext; -import org.apache.druid.query.select.SelectQueryConfig; import org.apache.druid.server.coordination.ServerType; import org.apache.druid.timeline.DataSegment; import org.apache.druid.timeline.VersionedIntervalTimeline; @@ -74,11 +71,8 @@ public class CachingClusteredClientFunctionalityTest { private static final ObjectMapper OBJECT_MAPPER = CachingClusteredClientTestUtils.createObjectMapper(); - private static final Supplier SELECT_CONFIG_SUPPLIER = Suppliers.ofInstance( - new SelectQueryConfig(true) - ); private static final Pair WAREHOUSE_AND_CLOSER = CachingClusteredClientTestUtils - .createWarehouse(OBJECT_MAPPER, SELECT_CONFIG_SUPPLIER); + .createWarehouse(OBJECT_MAPPER); private static final QueryToolChestWarehouse WAREHOUSE = WAREHOUSE_AND_CLOSER.lhs; private static final Closer RESOURCE_CLOSER = WAREHOUSE_AND_CLOSER.rhs; diff --git a/server/src/test/java/org/apache/druid/client/CachingClusteredClientTest.java b/server/src/test/java/org/apache/druid/client/CachingClusteredClientTest.java index 25d1d7efbb28..242795084917 100644 --- a/server/src/test/java/org/apache/druid/client/CachingClusteredClientTest.java +++ b/server/src/test/java/org/apache/druid/client/CachingClusteredClientTest.java @@ -24,14 +24,11 @@ import com.fasterxml.jackson.databind.annotation.JsonSerialize; import com.google.common.base.Function; import com.google.common.base.Preconditions; -import com.google.common.base.Supplier; -import com.google.common.base.Suppliers; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Iterables; import com.google.common.collect.Lists; import com.google.common.collect.Ordering; -import com.google.common.collect.Sets; import com.google.common.hash.HashFunction; import com.google.common.hash.Hashing; import com.google.common.util.concurrent.ForwardingListeningExecutorService; @@ -106,12 +103,6 @@ import org.apache.druid.query.search.SearchQueryConfig; import org.apache.druid.query.search.SearchQueryQueryToolChest; import org.apache.druid.query.search.SearchResultValue; -import org.apache.druid.query.select.EventHolder; -import org.apache.druid.query.select.PagingSpec; -import org.apache.druid.query.select.SelectQuery; -import org.apache.druid.query.select.SelectQueryConfig; -import org.apache.druid.query.select.SelectQueryQueryToolChest; -import org.apache.druid.query.select.SelectResultValue; import org.apache.druid.query.spec.MultipleIntervalSegmentSpec; import org.apache.druid.query.spec.MultipleSpecificSegmentSpec; import org.apache.druid.query.timeboundary.TimeBoundaryQuery; @@ -158,7 +149,6 @@ import java.util.List; import java.util.Map; import java.util.Random; -import java.util.Set; import java.util.TreeMap; import java.util.concurrent.Callable; import java.util.concurrent.ConcurrentLinkedDeque; @@ -254,11 +244,8 @@ public class CachingClusteredClientTest private static final DateTimeZone TIMEZONE = DateTimes.inferTzFromString("America/Los_Angeles"); private static final Granularity PT1H_TZ_GRANULARITY = new PeriodGranularity(new Period("PT1H"), null, TIMEZONE); private static final String TOP_DIM = "a_dim"; - private static final Supplier SELECT_CONFIG_SUPPLIER = Suppliers.ofInstance( - new SelectQueryConfig(true) - ); private static final Pair WAREHOUSE_AND_CLOSER = CachingClusteredClientTestUtils - .createWarehouse(JSON_MAPPER, SELECT_CONFIG_SUPPLIER); + .createWarehouse(JSON_MAPPER); private static final QueryToolChestWarehouse WAREHOUSE = WAREHOUSE_AND_CLOSER.lhs; private static final Closer RESOURCE_CLOSER = WAREHOUSE_AND_CLOSER.rhs; @@ -1255,183 +1242,6 @@ public void testSearchCachingRenamedOutput() ); } - @Test - public void testSelectCaching() - { - final Set dimensions = Sets.newHashSet("a"); - final Set metrics = Sets.newHashSet("rows"); - - Druids.SelectQueryBuilder builder = Druids.newSelectQueryBuilder() - .dataSource(DATA_SOURCE) - .intervals(SEG_SPEC) - .filters(DIM_FILTER) - .granularity(GRANULARITY) - .dimensions(Collections.singletonList("a")) - .metrics(Collections.singletonList("rows")) - .pagingSpec(new PagingSpec(null, 3)) - .context(CONTEXT); - - testQueryCaching( - getDefaultQueryRunner(), - builder.build(), - Intervals.of("2011-01-01/2011-01-02"), - makeSelectResults(dimensions, metrics, DateTimes.of("2011-01-01"), ImmutableMap.of("a", "b", "rows", 1)), - - Intervals.of("2011-01-02/2011-01-03"), - makeSelectResults(dimensions, metrics, DateTimes.of("2011-01-02"), ImmutableMap.of("a", "c", "rows", 5)), - - Intervals.of("2011-01-05/2011-01-10"), - makeSelectResults( - dimensions, - metrics, - DateTimes.of("2011-01-05"), - DateTimes.of("2011-01-06"), - DateTimes.of("2011-01-07"), ImmutableMap.of("a", "f", "rows", 7), ImmutableMap.of("a", "ff"), - DateTimes.of("2011-01-08"), ImmutableMap.of("a", "g", "rows", 8), - DateTimes.of("2011-01-09"), ImmutableMap.of("a", "h", "rows", 9) - ), - - Intervals.of("2011-01-05/2011-01-10"), - makeSelectResults( - dimensions, - metrics, - DateTimes.of("2011-01-05T01"), ImmutableMap.of("a", "d", "rows", 5), - DateTimes.of("2011-01-06T01"), ImmutableMap.of("a", "e", "rows", 6), - DateTimes.of("2011-01-07T01"), ImmutableMap.of("a", "f", "rows", 7), - DateTimes.of("2011-01-08T01"), ImmutableMap.of("a", "g", "rows", 8), - DateTimes.of("2011-01-09T01"), ImmutableMap.of("a", "h", "rows", 9) - ) - ); - - QueryRunner runner = new FinalizeResultsQueryRunner( - getDefaultQueryRunner(), - new SelectQueryQueryToolChest( - JSON_MAPPER, - QueryRunnerTestHelper.noopIntervalChunkingQueryRunnerDecorator() - ) - ); - TestHelper.assertExpectedResults( - makeSelectResults( - dimensions, - metrics, - DateTimes.of("2011-01-01"), ImmutableMap.of("a", "b", "rows", 1), - DateTimes.of("2011-01-02"), ImmutableMap.of("a", "c", "rows", 5), - DateTimes.of("2011-01-05"), - DateTimes.of("2011-01-05T01"), ImmutableMap.of("a", "d", "rows", 5), - DateTimes.of("2011-01-06"), - DateTimes.of("2011-01-06T01"), ImmutableMap.of("a", "e", "rows", 6), - DateTimes.of("2011-01-07"), ImmutableMap.of("a", "f", "rows", 7), ImmutableMap.of("a", "ff"), - DateTimes.of("2011-01-07T01"), ImmutableMap.of("a", "f", "rows", 7), - DateTimes.of("2011-01-08"), ImmutableMap.of("a", "g", "rows", 8), - DateTimes.of("2011-01-08T01"), ImmutableMap.of("a", "g", "rows", 8), - DateTimes.of("2011-01-09"), ImmutableMap.of("a", "h", "rows", 9), - DateTimes.of("2011-01-09T01"), ImmutableMap.of("a", "h", "rows", 9) - ), - runner.run(QueryPlus.wrap(builder.intervals("2011-01-01/2011-01-10").build())) - ); - } - - @Test - public void testSelectCachingRenamedOutputName() - { - final Set dimensions = Sets.newHashSet("a"); - final Set metrics = Sets.newHashSet("rows"); - - Druids.SelectQueryBuilder builder = Druids.newSelectQueryBuilder() - .dataSource(DATA_SOURCE) - .intervals(SEG_SPEC) - .filters(DIM_FILTER) - .granularity(GRANULARITY) - .dimensions(Collections.singletonList("a")) - .metrics(Collections.singletonList("rows")) - .pagingSpec(new PagingSpec(null, 3)) - .context(CONTEXT); - - testQueryCaching( - getDefaultQueryRunner(), - builder.build(), - Intervals.of("2011-01-01/2011-01-02"), - makeSelectResults(dimensions, metrics, DateTimes.of("2011-01-01"), ImmutableMap.of("a", "b", "rows", 1)), - - Intervals.of("2011-01-02/2011-01-03"), - makeSelectResults(dimensions, metrics, DateTimes.of("2011-01-02"), ImmutableMap.of("a", "c", "rows", 5)), - - Intervals.of("2011-01-05/2011-01-10"), - makeSelectResults( - dimensions, - metrics, - DateTimes.of("2011-01-05"), ImmutableMap.of("a", "d", "rows", 5), - DateTimes.of("2011-01-06"), ImmutableMap.of("a", "e", "rows", 6), - DateTimes.of("2011-01-07"), ImmutableMap.of("a", "f", "rows", 7), - DateTimes.of("2011-01-08"), ImmutableMap.of("a", "g", "rows", 8), - DateTimes.of("2011-01-09"), ImmutableMap.of("a", "h", "rows", 9) - ), - - Intervals.of("2011-01-05/2011-01-10"), - makeSelectResults( - dimensions, - metrics, - DateTimes.of("2011-01-05T01"), ImmutableMap.of("a", "d", "rows", 5), - DateTimes.of("2011-01-06T01"), ImmutableMap.of("a", "e", "rows", 6), - DateTimes.of("2011-01-07T01"), ImmutableMap.of("a", "f", "rows", 7), - DateTimes.of("2011-01-08T01"), ImmutableMap.of("a", "g", "rows", 8), - DateTimes.of("2011-01-09T01"), ImmutableMap.of("a", "h", "rows", 9) - ) - ); - - QueryRunner runner = new FinalizeResultsQueryRunner( - getDefaultQueryRunner(), - new SelectQueryQueryToolChest( - JSON_MAPPER, - QueryRunnerTestHelper.noopIntervalChunkingQueryRunnerDecorator() - ) - ); - ResponseContext context = ResponseContext.createEmpty(); - TestHelper.assertExpectedResults( - makeSelectResults( - dimensions, - metrics, - DateTimes.of("2011-01-01"), ImmutableMap.of("a", "b", "rows", 1), - DateTimes.of("2011-01-02"), ImmutableMap.of("a", "c", "rows", 5), - DateTimes.of("2011-01-05"), ImmutableMap.of("a", "d", "rows", 5), - DateTimes.of("2011-01-05T01"), ImmutableMap.of("a", "d", "rows", 5), - DateTimes.of("2011-01-06"), ImmutableMap.of("a", "e", "rows", 6), - DateTimes.of("2011-01-06T01"), ImmutableMap.of("a", "e", "rows", 6), - DateTimes.of("2011-01-07"), ImmutableMap.of("a", "f", "rows", 7), - DateTimes.of("2011-01-07T01"), ImmutableMap.of("a", "f", "rows", 7), - DateTimes.of("2011-01-08"), ImmutableMap.of("a", "g", "rows", 8), - DateTimes.of("2011-01-08T01"), ImmutableMap.of("a", "g", "rows", 8), - DateTimes.of("2011-01-09"), ImmutableMap.of("a", "h", "rows", 9), - DateTimes.of("2011-01-09T01"), ImmutableMap.of("a", "h", "rows", 9) - ), - runner.run(QueryPlus.wrap(builder.intervals("2011-01-01/2011-01-10").build()), context) - ); - - SelectQuery query = builder - .intervals("2011-01-01/2011-01-10") - .dimensionSpecs(Collections.singletonList(new DefaultDimensionSpec("a", "a2"))) - .build(); - TestHelper.assertExpectedResults( - makeSelectResults( - dimensions, - metrics, - DateTimes.of("2011-01-01"), ImmutableMap.of("a2", "b", "rows", 1), - DateTimes.of("2011-01-02"), ImmutableMap.of("a2", "c", "rows", 5), - DateTimes.of("2011-01-05"), ImmutableMap.of("a2", "d", "rows", 5), - DateTimes.of("2011-01-05T01"), ImmutableMap.of("a2", "d", "rows", 5), - DateTimes.of("2011-01-06"), ImmutableMap.of("a2", "e", "rows", 6), - DateTimes.of("2011-01-06T01"), ImmutableMap.of("a2", "e", "rows", 6), - DateTimes.of("2011-01-07"), ImmutableMap.of("a2", "f", "rows", 7), - DateTimes.of("2011-01-07T01"), ImmutableMap.of("a2", "f", "rows", 7), - DateTimes.of("2011-01-08"), ImmutableMap.of("a2", "g", "rows", 8), - DateTimes.of("2011-01-08T01"), ImmutableMap.of("a2", "g", "rows", 8), - DateTimes.of("2011-01-09"), ImmutableMap.of("a2", "h", "rows", 9), - DateTimes.of("2011-01-09T01"), ImmutableMap.of("a2", "h", "rows", 9) - ), - runner.run(QueryPlus.wrap(query), context) - ); - } - @Test public void testGroupByCaching() { @@ -2060,18 +1870,6 @@ public void testQueryCaching( EasyMock.expect(queryable.run(EasyMock.capture(capture), EasyMock.capture(context))) .andReturn(toQueryableSearchResults(segmentIds, intervals, results)) .once(); - } else if (query instanceof SelectQuery) { - List segmentIds = new ArrayList<>(); - List intervals = new ArrayList<>(); - List>> results = new ArrayList<>(); - for (ServerExpectation expectation : expectations) { - segmentIds.add(expectation.getSegmentId()); - intervals.add(expectation.getInterval()); - results.add(expectation.getResults()); - } - EasyMock.expect(queryable.run(EasyMock.capture(capture), EasyMock.capture(context))) - .andReturn(toQueryableSelectResults(segmentIds, intervals, results)) - .once(); } else if (query instanceof GroupByQuery) { List segmentIds = new ArrayList<>(); List intervals = new ArrayList<>(); @@ -2351,42 +2149,6 @@ public Result apply( ); } - private Sequence> toQueryableSelectResults( - Iterable segmentIds, - Iterable intervals, - Iterable>> results - ) - { - return Sequences.simple( - FunctionalIterable - .create(segmentIds) - .trinaryTransform( - intervals, - results, - new TrinaryFn>, Result>() - { - @Override - @SuppressWarnings("unchecked") - public Result apply( - final SegmentId segmentId, - final Interval interval, - final Iterable> results - ) - { - return new Result( - results.iterator().next().getTimestamp(), - new BySegmentResultValueClass( - Lists.newArrayList(results), - segmentId.toString(), - interval - ) - ); - } - } - ) - ); - } - private Sequence toQueryableGroupByResults( GroupByQuery query, Iterable segmentIds, @@ -2605,32 +2367,6 @@ private Iterable> makeSearchResults(String dim, Object return retVal; } - private Iterable> makeSelectResults( - Set dimensions, - Set metrics, - Object... objects - ) - { - List> retVal = new ArrayList<>(); - int index = 0; - while (index < objects.length) { - DateTime timestamp = (DateTime) objects[index++]; - - List values = new ArrayList<>(); - - while (index < objects.length && !(objects[index] instanceof DateTime)) { - values.add(new EventHolder(null, 0, (Map) objects[index++])); - } - - Result result = new Result<>( - timestamp, - new SelectResultValue(ImmutableMap.of(timestamp.toString(), 0), dimensions, metrics, values) - ); - retVal.add(result); - } - return retVal; - } - private Iterable makeGroupByResults(GroupByQuery query, Object... objects) { List retVal = new ArrayList<>(); diff --git a/server/src/test/java/org/apache/druid/client/CachingClusteredClientTestUtils.java b/server/src/test/java/org/apache/druid/client/CachingClusteredClientTestUtils.java index 646031db7f9a..16f75be51976 100644 --- a/server/src/test/java/org/apache/druid/client/CachingClusteredClientTestUtils.java +++ b/server/src/test/java/org/apache/druid/client/CachingClusteredClientTestUtils.java @@ -21,7 +21,6 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.dataformat.smile.SmileFactory; -import com.google.common.base.Supplier; import com.google.common.collect.ImmutableMap; import org.apache.druid.jackson.DefaultObjectMapper; import org.apache.druid.java.util.common.Pair; @@ -38,9 +37,6 @@ import org.apache.druid.query.search.SearchQuery; import org.apache.druid.query.search.SearchQueryConfig; import org.apache.druid.query.search.SearchQueryQueryToolChest; -import org.apache.druid.query.select.SelectQuery; -import org.apache.druid.query.select.SelectQueryConfig; -import org.apache.druid.query.select.SelectQueryQueryToolChest; import org.apache.druid.query.timeboundary.TimeBoundaryQuery; import org.apache.druid.query.timeboundary.TimeBoundaryQueryQueryToolChest; import org.apache.druid.query.timeseries.TimeseriesQuery; @@ -56,8 +52,7 @@ public final class CachingClusteredClientTestUtils * of the test. */ public static Pair createWarehouse( - ObjectMapper objectMapper, - Supplier selectConfigSupplier + ObjectMapper objectMapper ) { final Pair factoryCloserPair = GroupByQueryRunnerTest.makeQueryRunnerFactory( @@ -88,13 +83,6 @@ public static Pair createWarehouse( QueryRunnerTestHelper.noopIntervalChunkingQueryRunnerDecorator() ) ) - .put( - SelectQuery.class, - new SelectQueryQueryToolChest( - objectMapper, - QueryRunnerTestHelper.noopIntervalChunkingQueryRunnerDecorator() - ) - ) .put( GroupByQuery.class, factory.getToolchest() diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/rel/DruidQuery.java b/sql/src/main/java/org/apache/druid/sql/calcite/rel/DruidQuery.java index 1c72b4217095..b9769606dbb6 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/rel/DruidQuery.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/rel/DruidQuery.java @@ -59,7 +59,6 @@ import org.apache.druid.query.ordering.StringComparator; import org.apache.druid.query.ordering.StringComparators; import org.apache.druid.query.scan.ScanQuery; -import org.apache.druid.query.select.SelectQuery; import org.apache.druid.query.timeseries.TimeseriesQuery; import org.apache.druid.query.topn.DimensionTopNMetricSpec; import org.apache.druid.query.topn.InvertedTopNMetricSpec; @@ -614,7 +613,7 @@ private RowSignature computeOutputRowSignature() /** * Return this query as some kind of Druid query. The returned query will either be {@link TopNQuery}, - * {@link TimeseriesQuery}, {@link GroupByQuery}, {@link ScanQuery}, or {@link SelectQuery}. + * {@link TimeseriesQuery}, {@link GroupByQuery}, {@link ScanQuery} * * @return Druid query */ diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/BaseCalciteQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/BaseCalciteQueryTest.java index 8bffa486210e..4d22779491c2 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/BaseCalciteQueryTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/BaseCalciteQueryTest.java @@ -54,7 +54,6 @@ import org.apache.druid.query.ordering.StringComparator; import org.apache.druid.query.ordering.StringComparators; import org.apache.druid.query.scan.ScanQuery; -import org.apache.druid.query.select.PagingSpec; import org.apache.druid.query.spec.MultipleIntervalSegmentSpec; import org.apache.druid.query.spec.QuerySegmentSpec; import org.apache.druid.segment.column.ColumnHolder; @@ -214,7 +213,6 @@ public int getMaxSemiJoinRowsInMemory() // Matches QUERY_CONTEXT_LOS_ANGELES public static final Map TIMESERIES_CONTEXT_LOS_ANGELES = new HashMap<>(); - public static final PagingSpec FIRST_PAGING_SPEC = new PagingSpec(null, 1000, true); public static QueryRunnerFactoryConglomerate conglomerate; public static Closer resourceCloser; diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/util/CalciteTests.java b/sql/src/test/java/org/apache/druid/sql/calcite/util/CalciteTests.java index 0cda7526ab4a..44ad2ca39b39 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/util/CalciteTests.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/util/CalciteTests.java @@ -84,10 +84,6 @@ import org.apache.druid.query.scan.ScanQueryEngine; import org.apache.druid.query.scan.ScanQueryQueryToolChest; import org.apache.druid.query.scan.ScanQueryRunnerFactory; -import org.apache.druid.query.select.SelectQuery; -import org.apache.druid.query.select.SelectQueryEngine; -import org.apache.druid.query.select.SelectQueryQueryToolChest; -import org.apache.druid.query.select.SelectQueryRunnerFactory; import org.apache.druid.query.timeseries.TimeseriesQuery; import org.apache.druid.query.timeseries.TimeseriesQueryEngine; import org.apache.druid.query.timeseries.TimeseriesQueryQueryToolChest; @@ -537,17 +533,6 @@ public int getNumMergeBuffers() new ScanQueryConfig() ) ) - .put( - SelectQuery.class, - new SelectQueryRunnerFactory( - new SelectQueryQueryToolChest( - TestHelper.makeJsonMapper(), - QueryRunnerTestHelper.noopIntervalChunkingQueryRunnerDecorator() - ), - new SelectQueryEngine(), - QueryRunnerTestHelper.NOOP_QUERYWATCHER - ) - ) .put( TimeseriesQuery.class, new TimeseriesQueryRunnerFactory(