From cbbbcb7a78cc79b9853ca5b08a7fc6393389455d Mon Sep 17 00:00:00 2001 From: Soumyava Das Date: Mon, 3 Oct 2022 09:43:35 -0700 Subject: [PATCH 01/10] Updating rules to handle queries correctly --- .../sql/calcite/rel/DruidOuterQueryRel.java | 23 ++++++--- .../druid/sql/calcite/rel/DruidQuery.java | 9 ---- .../druid/sql/calcite/rule/DruidRules.java | 11 ++++- .../druid/sql/calcite/CalciteQueryTest.java | 49 ++++++++++++++++--- 4 files changed, 68 insertions(+), 24 deletions(-) diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/rel/DruidOuterQueryRel.java b/sql/src/main/java/org/apache/druid/sql/calcite/rel/DruidOuterQueryRel.java index d9bd16343ef1..e5a4c3e845be 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/rel/DruidOuterQueryRel.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/rel/DruidOuterQueryRel.java @@ -32,8 +32,8 @@ import org.apache.calcite.rel.metadata.RelMetadataQuery; import org.apache.calcite.rel.type.RelDataType; import org.apache.druid.java.util.common.StringUtils; -import org.apache.druid.query.Druids; import org.apache.druid.query.QueryDataSource; +import org.apache.druid.query.TableDataSource; import org.apache.druid.segment.column.RowSignature; import org.apache.druid.sql.calcite.planner.PlannerContext; import org.apache.druid.sql.calcite.table.RowSignatures; @@ -46,9 +46,10 @@ */ public class DruidOuterQueryRel extends DruidRel { - private static final QueryDataSource DUMMY_DATA_SOURCE = new QueryDataSource( + /*private static final QueryDataSource DUMMY_DATA_SOURCE = new QueryDataSource( Druids.newScanQueryBuilder().dataSource("__subquery__").eternityInterval().build() - ); + );*/ + private static final TableDataSource DUMMY_DATA_SOURCE = new TableDataSource("__subquery__"); private final PartialDruidQuery partialQuery; private RelNode sourceRel; @@ -116,12 +117,20 @@ public DruidQuery toDruidQuery(final boolean finalizeAggregations) @Override public DruidQuery toDruidQueryForExplaining() { + final RowSignature sourceRowSignature; + if (sourceRel instanceof DruidRel) { + final DruidQuery subQuery = ((DruidRel) sourceRel).toDruidQueryForExplaining(); + sourceRowSignature = subQuery.getOutputRowSignature(); + } else { + // fallback for cases other than group by inner query with order by with alias on __time column + sourceRowSignature = RowSignatures.fromRelDataType( + sourceRel.getRowType().getFieldNames(), + sourceRel.getRowType() + ); + } return partialQuery.build( DUMMY_DATA_SOURCE, - RowSignatures.fromRelDataType( - sourceRel.getRowType().getFieldNames(), - sourceRel.getRowType() - ), + sourceRowSignature, getPlannerContext(), getCluster().getRexBuilder(), false diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/rel/DruidQuery.java b/sql/src/main/java/org/apache/druid/sql/calcite/rel/DruidQuery.java index fcf9fb754d8a..3bc402046451 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/rel/DruidQuery.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/rel/DruidQuery.java @@ -1298,15 +1298,6 @@ private ScanQuery toScanQuery() ); return null; } - if (!dataSource.isConcrete()) { - // Cannot handle this ordering. - // Scan cannot ORDER BY non-time columns. - plannerContext.setPlanningError( - "SQL query is a scan and requires order by on a datasource[%s], which is not supported.", - dataSource - ); - return null; - } } // Compute the list of columns to select, sorted and deduped. diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/rule/DruidRules.java b/sql/src/main/java/org/apache/druid/sql/calcite/rule/DruidRules.java index 113f37d896fb..a95106b5dcf7 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/rule/DruidRules.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/rule/DruidRules.java @@ -30,9 +30,11 @@ import org.apache.calcite.rel.core.Sort; import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.sql.calcite.planner.PlannerContext; +import org.apache.druid.sql.calcite.rel.CannotBuildQueryException; import org.apache.druid.sql.calcite.rel.DruidOuterQueryRel; import org.apache.druid.sql.calcite.rel.DruidRel; import org.apache.druid.sql.calcite.rel.PartialDruidQuery; +import org.apache.druid.utils.Throwables; import java.util.List; import java.util.function.BiFunction; @@ -223,7 +225,14 @@ public void onMatch(final RelOptRuleCall call) .withSort(sort) ); if (outerQueryRel.isValidDruidQuery()) { - call.transformTo(outerQueryRel); + try { + call.transformTo(outerQueryRel); + } + catch (Exception ex) { + if (Throwables.getCauseOfType(ex, CannotBuildQueryException.class) == null) { + throw ex; + } + } } } }; diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java index bd21a5798843..783a4634e695 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java @@ -6983,12 +6983,6 @@ public void testMinMaxAvgDailyCountWithLimit() ) ) .setInterval(querySegmentSpec(Filtration.eternity())) - .setLimitSpec( - new DefaultLimitSpec( - ImmutableList.of(), - 1 - ) - ) .setGranularity(Granularities.ALL) .setAggregatorSpecs( useDefault @@ -7022,7 +7016,7 @@ public void testMinMaxAvgDailyCountWithLimit() new FieldAccessPostAggregator(null, "_a2:count") ) ), - expressionPostAgg("s0", "timestamp_extract(\"_a3\",'EPOCH','UTC')") + expressionPostAgg("p0", "timestamp_extract(\"_a3\",'EPOCH','UTC')") ) ) .setContext(QUERY_CONTEXT_DEFAULT) @@ -11222,6 +11216,47 @@ public void testOrderByAlongWithAliasOrderByTimeGroupByMulti() ); } + @Test + public void testOrderByAlongWithInternalScanQuery() + { + testQuery( + "select __time as t, m1 from druid.foo where (m1 in (select distinct m1 from druid.foo)) order by 1 limit 1", + ImmutableList.of( + newScanQueryBuilder() + .intervals(querySegmentSpec(Filtration.eternity())) + .dataSource( + JoinDataSource.create( + new TableDataSource(CalciteTests.DATASOURCE1), + new QueryDataSource( + GroupByQuery.builder() + .setGranularity(Granularities.ALL) + .setDataSource(CalciteTests.DATASOURCE1) + .setInterval(querySegmentSpec(Intervals.of( + "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z"))) + .setDimensions(new DefaultDimensionSpec("m1", "d0", ColumnType.FLOAT)) + .build() + ), + "j0.", + "(\"m1\" == \"j0.d0\")", + JoinType.INNER, + null, + ExprMacroTable.nil() + ) + ) + .context(QUERY_CONTEXT_DEFAULT) + .intervals(querySegmentSpec(Intervals.of( + "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z"))) + .limit(1) + .columns(ImmutableList.of("__time", "m1")) + .order(ScanQuery.Order.ASCENDING) + .build() + ), + ImmutableList.of( + new Object[]{946684800000L, 1.0f} + ) + ); + } + @Test public void testOrderByAlongWithAliasOrderByTimeGroupByOneCol() { From 86a435d5883384211adf84688177a1f5dcb474dc Mon Sep 17 00:00:00 2001 From: Soumyava Das Date: Mon, 3 Oct 2022 10:02:14 -0700 Subject: [PATCH 02/10] Removing unused comment --- .../org/apache/druid/sql/calcite/rel/DruidOuterQueryRel.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/rel/DruidOuterQueryRel.java b/sql/src/main/java/org/apache/druid/sql/calcite/rel/DruidOuterQueryRel.java index e5a4c3e845be..cd8a8fba23c0 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/rel/DruidOuterQueryRel.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/rel/DruidOuterQueryRel.java @@ -46,9 +46,6 @@ */ public class DruidOuterQueryRel extends DruidRel { - /*private static final QueryDataSource DUMMY_DATA_SOURCE = new QueryDataSource( - Druids.newScanQueryBuilder().dataSource("__subquery__").eternityInterval().build() - );*/ private static final TableDataSource DUMMY_DATA_SOURCE = new TableDataSource("__subquery__"); private final PartialDruidQuery partialQuery; From 216733758ef5fc7c090e1e465c750ccaeda87c17 Mon Sep 17 00:00:00 2001 From: Soumyava Das Date: Mon, 3 Oct 2022 13:11:39 -0700 Subject: [PATCH 03/10] Updating a test as inner subquery is now a table data source instead of query data source --- .../org/apache/druid/sql/calcite/CalciteExplainQueryTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteExplainQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteExplainQueryTest.java index 70e0320e3a15..480fca6aeb88 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteExplainQueryTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteExplainQueryTest.java @@ -109,7 +109,7 @@ public void testExplainExactCountDistinctOfSemiJoinResult() + " )\n" + ")"; final String legacyExplanation = - "DruidOuterQueryRel(query=[{\"queryType\":\"groupBy\",\"dataSource\":{\"type\":\"query\",\"query\":{\"queryType\":\"scan\",\"dataSource\":{\"type\":\"table\",\"name\":\"__subquery__\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"resultFormat\":\"list\",\"granularity\":{\"type\":\"all\"}}},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"granularity\":{\"type\":\"all\"},\"dimensions\":[],\"aggregations\":[{\"type\":\"count\",\"name\":\"a0\"}],\"limitSpec\":{\"type\":\"NoopLimitSpec\"},\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\",\"sqlQueryId\":\"dummy\",\"vectorize\":\"false\",\"vectorizeVirtualColumns\":\"false\"}}], signature=[{a0:LONG}])\n" + "DruidOuterQueryRel(query=[{\"queryType\":\"timeseries\",\"dataSource\":{\"type\":\"table\",\"name\":\"__subquery__\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"granularity\":{\"type\":\"all\"},\"aggregations\":[{\"type\":\"count\",\"name\":\"a0\"}],\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\",\"sqlQueryId\":\"dummy\",\"vectorize\":\"false\",\"vectorizeVirtualColumns\":\"false\"}}], signature=[{a0:LONG}])\n" + " DruidJoinQueryRel(condition=[=(SUBSTRING($2, 1, 1), $8)], joinType=[inner], query=[{\"queryType\":\"groupBy\",\"dataSource\":{\"type\":\"table\",\"name\":\"__join__\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"granularity\":{\"type\":\"all\"},\"dimensions\":[{\"type\":\"default\",\"dimension\":\"dim2\",\"outputName\":\"d0\",\"outputType\":\"STRING\"}],\"limitSpec\":{\"type\":\"NoopLimitSpec\"},\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\",\"sqlQueryId\":\"dummy\",\"vectorize\":\"false\",\"vectorizeVirtualColumns\":\"false\"}}], signature=[{d0:STRING}])\n" + " DruidQueryRel(query=[{\"queryType\":\"scan\",\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"resultFormat\":\"compactedList\",\"columns\":[\"__time\",\"cnt\",\"dim1\",\"dim2\",\"dim3\",\"m1\",\"m2\",\"unique_dim1\"],\"legacy\":false,\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\",\"sqlQueryId\":\"dummy\",\"vectorize\":\"false\",\"vectorizeVirtualColumns\":\"false\"},\"granularity\":{\"type\":\"all\"}}], signature=[{__time:LONG, dim1:STRING, dim2:STRING, dim3:STRING, cnt:LONG, m1:FLOAT, m2:DOUBLE, unique_dim1:COMPLEX}])\n" + " DruidQueryRel(query=[{\"queryType\":\"groupBy\",\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"filter\":{\"type\":\"not\",\"field\":{\"type\":\"selector\",\"dimension\":\"dim1\",\"value\":null}},\"granularity\":{\"type\":\"all\"},\"dimensions\":[{\"type\":\"extraction\",\"dimension\":\"dim1\",\"outputName\":\"d0\",\"outputType\":\"STRING\",\"extractionFn\":{\"type\":\"substring\",\"index\":0,\"length\":1}}],\"limitSpec\":{\"type\":\"NoopLimitSpec\"},\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\",\"sqlQueryId\":\"dummy\",\"vectorize\":\"false\",\"vectorizeVirtualColumns\":\"false\"}}], signature=[{d0:STRING}])\n"; From 2ace39a27dd164cf8c310022e46bef37937b2ec9 Mon Sep 17 00:00:00 2001 From: Soumyava Das Date: Thu, 6 Oct 2022 15:22:45 -0700 Subject: [PATCH 04/10] Two additional test cases --- .../sql/calcite/CalciteJoinQueryTest.java | 48 +++++++++++++++++++ .../druid/sql/calcite/CalciteQueryTest.java | 41 ++++++++++++++++ 2 files changed, 89 insertions(+) diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteJoinQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteJoinQueryTest.java index d614c23d5d22..66bd5860ecf6 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteJoinQueryTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteJoinQueryTest.java @@ -161,6 +161,54 @@ public void testInnerJoinWithLimitAndAlias() ); } + @Test + public void testJoinWithAliasAndOrderByNoGroupBy() + { + minTopNThreshold = 1; + Map context = new HashMap<>(QUERY_CONTEXT_DEFAULT); + context.put(PlannerConfig.CTX_KEY_USE_APPROXIMATE_TOPN, false); + testQuery( + "select t1.__time from druid.foo as t1 join\n" + + " druid.numfoo as t2 on t1.dim2 = t2.dim2\n" + + " order by t1.__time ASC ", + context, // turn on exact topN + ImmutableList.of( + newScanQueryBuilder() + .intervals(querySegmentSpec(Filtration.eternity())) + .dataSource( + JoinDataSource.create( + new TableDataSource(CalciteTests.DATASOURCE1), + new QueryDataSource( + newScanQueryBuilder() + .dataSource(CalciteTests.DATASOURCE3) + .intervals(querySegmentSpec(Intervals.of( + "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z"))) + .columns("dim2") + .context(context) + .build() + ), + "j0.", + "(\"dim2\" == \"j0.dim2\")", + JoinType.INNER, + null, + ExprMacroTable.nil() + ) + ) + .columns("__time") + .order(ScanQuery.Order.ASCENDING) + .context(context) + .build() + ), + ImmutableList.of( + new Object[]{946684800000L}, + new Object[]{946684800000L}, + new Object[]{978307200000L}, + new Object[]{978307200000L}, + new Object[]{978393600000L} + ) + ); + } + @Test public void testExactTopNOnInnerJoinWithLimit() diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java index 783a4634e695..7970133e2a65 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java @@ -11257,6 +11257,47 @@ public void testOrderByAlongWithInternalScanQuery() ); } + @Test + public void testOrderByAlongWithInternalScanQueryNoDistinct() + { + testQuery( + "select __time, m1 from druid.foo where (m1 in (select m1 from druid.foo)) order by __time DESC limit 1", + ImmutableList.of( + newScanQueryBuilder() + .intervals(querySegmentSpec(Filtration.eternity())) + .dataSource( + JoinDataSource.create( + new TableDataSource(CalciteTests.DATASOURCE1), + new QueryDataSource( + GroupByQuery.builder() + .setGranularity(Granularities.ALL) + .setDataSource(CalciteTests.DATASOURCE1) + .setInterval(querySegmentSpec(Intervals.of( + "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z"))) + .setDimensions(new DefaultDimensionSpec("m1", "d0", ColumnType.FLOAT)) + .build() + ), + "j0.", + "(\"m1\" == \"j0.d0\")", + JoinType.INNER, + null, + ExprMacroTable.nil() + ) + ) + .context(QUERY_CONTEXT_DEFAULT) + .intervals(querySegmentSpec(Intervals.of( + "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z"))) + .limit(1) + .columns(ImmutableList.of("__time", "m1")) + .order(ScanQuery.Order.DESCENDING) + .build() + ), + ImmutableList.of( + new Object[]{978480000000L, 6.0f} + ) + ); + } + @Test public void testOrderByAlongWithAliasOrderByTimeGroupByOneCol() { From a4105647c654e5e54975a653c2f1e2e78a340eef Mon Sep 17 00:00:00 2001 From: Soumyava Das Date: Thu, 6 Oct 2022 16:27:37 -0700 Subject: [PATCH 05/10] Updating test case for sql compatibility --- .../druid/sql/calcite/CalciteJoinQueryTest.java | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteJoinQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteJoinQueryTest.java index 66bd5860ecf6..bd8344f7dc9e 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteJoinQueryTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteJoinQueryTest.java @@ -199,7 +199,16 @@ public void testJoinWithAliasAndOrderByNoGroupBy() .context(context) .build() ), - ImmutableList.of( + NullHandling.sqlCompatible() + ? ImmutableList.of( + new Object[]{946684800000L}, + new Object[]{946684800000L}, + new Object[]{946857600000L}, + new Object[]{978307200000L}, + new Object[]{978307200000L}, + new Object[]{978393600000L} + ) + : ImmutableList.of( new Object[]{946684800000L}, new Object[]{946684800000L}, new Object[]{978307200000L}, From 7a0a96e367af9dcc23a34ecd3fcdc87f3034b995 Mon Sep 17 00:00:00 2001 From: Soumyava Das Date: Tue, 11 Oct 2022 17:04:19 -0700 Subject: [PATCH 06/10] Updating benchmark tests with order by query with large iin filters --- .../apache/druid/benchmark/query/SqlBenchmark.java | 12 +++++++++++- .../apache/druid/sql/calcite/rule/DruidRules.java | 11 +---------- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlBenchmark.java b/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlBenchmark.java index 35974e4430df..d492db62ca50 100644 --- a/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlBenchmark.java +++ b/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlBenchmark.java @@ -392,7 +392,17 @@ public class SqlBenchmark // 20: GroupBy, doubles sketches "SELECT dimZipf, APPROX_QUANTILE_DS(sumFloatNormal, 0.5), DS_QUANTILES_SKETCH(maxLongUniform) " + "FROM foo " - + "GROUP BY 1" + + "GROUP BY 1", + + //21: Order by with alias with large in filter + "SELECT __time as t, dimSequential from foo " + + " where (dimSequential in (select DISTINCT dimSequential from foo)) " + + " order by 1 limit 1", + + //22: Order by without alias with large in filter + "SELECT __time, dimSequential from foo " + + " where (dimSequential in (select DISTINCT dimSequential from foo)) " + + " order by 1 limit 1" ); @Param({"5000000"}) diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/rule/DruidRules.java b/sql/src/main/java/org/apache/druid/sql/calcite/rule/DruidRules.java index a95106b5dcf7..113f37d896fb 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/rule/DruidRules.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/rule/DruidRules.java @@ -30,11 +30,9 @@ import org.apache.calcite.rel.core.Sort; import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.sql.calcite.planner.PlannerContext; -import org.apache.druid.sql.calcite.rel.CannotBuildQueryException; import org.apache.druid.sql.calcite.rel.DruidOuterQueryRel; import org.apache.druid.sql.calcite.rel.DruidRel; import org.apache.druid.sql.calcite.rel.PartialDruidQuery; -import org.apache.druid.utils.Throwables; import java.util.List; import java.util.function.BiFunction; @@ -225,14 +223,7 @@ public void onMatch(final RelOptRuleCall call) .withSort(sort) ); if (outerQueryRel.isValidDruidQuery()) { - try { - call.transformTo(outerQueryRel); - } - catch (Exception ex) { - if (Throwables.getCauseOfType(ex, CannotBuildQueryException.class) == null) { - throw ex; - } - } + call.transformTo(outerQueryRel); } } }; From b9743358dc9135a268d168d81f06abdb976b8b04 Mon Sep 17 00:00:00 2001 From: Soumyava Das Date: Tue, 11 Oct 2022 17:41:17 -0700 Subject: [PATCH 07/10] Another nested query --- .../org/apache/druid/benchmark/query/SqlBenchmark.java | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlBenchmark.java b/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlBenchmark.java index d492db62ca50..bc2290444660 100644 --- a/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlBenchmark.java +++ b/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlBenchmark.java @@ -402,7 +402,13 @@ public class SqlBenchmark //22: Order by without alias with large in filter "SELECT __time, dimSequential from foo " + " where (dimSequential in (select DISTINCT dimSequential from foo)) " - + " order by 1 limit 1" + + " order by 1 limit 1", + + //23: Group by and Order by with alias with large in filter nested query + "SELECT __time as t, dimSequential from foo " + + " where dimSequential in (select dimSequential from foo where " + + " dimSequential in (select dimSequential from foo)) " + + " group by 1,2 order by 1,2 limit 1" ); @Param({"5000000"}) From 1c25c15f3818e3d4113d316cbe528194356df71a Mon Sep 17 00:00:00 2001 From: Soumyava Das Date: Wed, 12 Oct 2022 09:45:04 -0700 Subject: [PATCH 08/10] Adding benchmark with ~1000 in filter --- .../druid/benchmark/query/SqlBenchmark.java | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlBenchmark.java b/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlBenchmark.java index bc2290444660..b995a4775ee1 100644 --- a/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlBenchmark.java +++ b/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlBenchmark.java @@ -408,7 +408,20 @@ public class SqlBenchmark "SELECT __time as t, dimSequential from foo " + " where dimSequential in (select dimSequential from foo where " + " dimSequential in (select dimSequential from foo)) " - + " group by 1,2 order by 1,2 limit 1" + + " group by 1,2 order by 1,2 limit 1", + + //24: Order by query with large in filters + "SELECT __time as t, dimZipf " + + " from foo " + + " where dimSequential IN ('0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '10', '11', '12', '13', '14', '15', '16', '17', '18', '19', '20', '21', '22', '23', '24', '25', '26', '27', '28', '29', '30', '31', '32', '33', '34', '35', '36', '37', '38', '39', '40', '41', '42', '43', '44', '45', '46', '47', '48', '49', '50', '51', '52', '53', '54', '55', '56', '57', '58', '59', '60', '61', '62', '63', '64', '65', '66', '67', '68', '69', '70', '71', '72', '73', '74', '75', '76', '77', '78', '79', '80', '81', '82', '83', '84', '85', '86', '87', '88', '89', '90', '91', '92', '93', '94', '95', '96', '97', '98', '99', '100', '101', '102', '103', '104', '105', '106', '107', '108', '109', '110', '111', '112', '113', '114', '115', '116', '117', '118', '119', '120', '121', '122', '123', '124', '125', '126', '127', '128', '129', '130', '131', '132', '133', '134', '135', '136', '137', '138', '139', '140', '141', '142', '143', '144', '145', '146', '147', '148', '149', '150', '151', '152', '153', '154', '155', '156', '157', '158', '159', '160', '161', '162', '163', '164', '165', '166', '167', '168', '169', '170', '171', '172', '173', '174', '175', '176', '177', '178', '179', '180', '181', '182', '183', '184', '185', '186', '187', '188', '189', '190', '191', '192', '193', '194', '195', '196', '197', '198', '199', '200', '201', '202', '203', '204', '205', '206', '207', '208', '209', '210', '211', '212', '213', '214', '215', '216', '217', '218', '219', '220', '221', '222', '223', '224', '225', '226', '227', '228', '229', '230', '231', '232', '233', '234', '235', '236', '237', '238', '239', '240', '241', '242', '243', '244', '245', '246', '247', '248', '249', '250', '251', '252', '253', '254', '255', '256', '257', '258', '259', '260', '261', '262', '263', '264', '265', '266', '267', '268', '269', '270', '271', '272', '273', '274', '275', '276', '277', '278', '279', '280', '281', '282', '283', '284', '285', '286', '287', '288', '289', '290', '291', '292', '293', '294', '295', '296', '297', '298', '299', '300', '301', '302', '303', '304', '305', '306', '307', '308', '309', '310', '311', '312', '313', '314', '315', '316', '317', '318', '319', '320', '321', '322', '323', '324', '325', '326', '327', '328', '329', '330', '331', '332', '333', '334', '335', '336', '337', '338', '339', '340', '341', '342', '343', '344', '345', '346', '347', '348', '349', '350', '351', '352', '353', '354', '355', '356', '357', '358', '359', '360', '361', '362', '363', '364', '365', '366', '367', '368', '369', '370', '371', '372', '373', '374', '375', '376', '377', '378', '379', '380', '381', '382', '383', '384', '385', '386', '387', '388', '389', '390', '391', '392', '393', '394', '395', '396', '397', '398', '399', '400', '401', '402', '403', '404', '405', '406', '407', '408', '409', '410', '411', '412', '413', '414', '415', '416', '417', '418', '419', '420', '421', '422', '423', '424', '425', '426', '427', '428', '429', '430', '431', '432', '433', '434', '435', '436', '437', '438', '439', '440', '441', '442', '443', '444', '445', '446', '447', '448', '449', '450', '451', '452', '453', '454', '455', '456', '457', '458', '459', '460', '461', '462', '463', '464', '465', '466', '467', '468', '469', '470', '471', '472', '473', '474', '475', '476', '477', '478', '479', '480', '481', '482', '483', '484', '485', '486', '487', '488', '489', '490', '491', '492', '493', '494', '495', '496', '497', '498', '499', '500', '501', '502', '503', '504', '505', '506', '507', '508', '509', '510', '511', '512', '513', '514', '515', '516', '517', '518', '519', '520', '521', '522', '523', '524', '525', '526', '527', '528', '529', '530', '531', '532', '533', '534', '535', '536', '537', '538', '539', '540', '541', '542', '543', '544', '545', '546', '547', '548', '549', '550', '551', '552', '553', '554', '555', '556', '557', '558', '559', '560', '561', '562', '563', '564', '565', '566', '567', '568', '569', '570', '571', '572', '573', '574', '575', '576', '577', '578', '579', '580', '581', '582', '583', '584', '585', '586', '587', '588', '589', '590', '591', '592', '593', '594', '595', '596', '597', '598', '599', '600', '601', '602', '603', '604', '605', '606', '607', '608', '609', '610', '611', '612', '613', '614', '615', '616', '617', '618', '619', '620', '621', '622', '623', '624', '625', '626', '627', '628', '629', '630', '631', '632', '633', '634', '635', '636', '637', '638', '639', '640', '641', '642', '643', '644', '645', '646', '647', '648', '649', '650', '651', '652', '653', '654', '655', '656', '657', '658', '659', '660', '661', '662', '663', '664', '665', '666', '667', '668', '669', '670', '671', '672', '673', '674', '675', '676', '677', '678', '679', '680', '681', '682', '683', '684', '685', '686', '687', '688', '689', '690', '691', '692', '693', '694', '695', '696', '697', '698', '699', '700', '701', '702', '703', '704', '705', '706', '707', '708', '709', '710', '711', '712', '713', '714', '715', '716', '717', '718', '719', '720', '721', '722', '723', '724', '725', '726', '727', '728', '729', '730', '731', '732', '733', '734', '735', '736', '737', '738', '739', '740', '741', '742', '743', '744', '745', '746', '747', '748', '749', '750', '751', '752', '753', '754', '755', '756', '757', '758', '759', '760', '761', '762', '763', '764', '765', '766', '767', '768', '769', '770', '771', '772', '773', '774', '775', '776', '777', '778', '779', '780', '781', '782', '783', '784', '785', '786', '787', '788', '789', '790', '791', '792', '793', '794', '795', '796', '797', '798', '799', '800', '801', '802', '803', '804', '805', '806', '807', '808', '809', '810', '811', '812', '813', '814', '815', '816', '817', '818', '819', '820', '821', '822', '823', '824', '825', '826', '827', '828', '829', '830', '831', '832', '833', '834', '835', '836', '837', '838', '839', '840', '841', '842', '843', '844', '845', '846', '847', '848', '849', '850', '851', '852', '853', '854', '855', '856', '857', '858', '859', '860', '861', '862', '863', '864', '865', '866', '867', '868', '869', '870', '871', '872', '873', '874', '875', '876', '877', '878', '879', '880', '881', '882', '883', '884', '885', '886', '887', '888', '889', '890', '891', '892', '893', '894', '895', '896', '897', '898', '899', '900', '901', '902', '903', '904', '905', '906', '907', '908', '909', '910', '911', '912', '913', '914', '915', '916', '917', '918', '919', '920', '921', '922', '923', '924', '925', '926', '927', '928', '929', '930', '931', '932', '933', '934', '935', '936', '937', '938', '939', '940', '941', '942', '943', '944', '945', '946', '947', '948', '949', '950', '951', '952', '953', '954', '955', '956', '957', '958', '959', '960', '961', '962', '963', '964', '965', '966', '967', '968', '969', '970', '971', '972', '973', '974', '975', '976', '977', '978', '979', '980', '981', '982', '983', '984', '985', '986', '987', '988', '989', '990', '991', '992', '993', '994', '995', '996', '997', '998') " + + " order by 1", + + //25. Nested subquery with large in filter + "SELECT __time as t, hyper from foo " + + " where dimUniform in ( " + + " select dimUniform from foo " + + " where dimSequential IN ('0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '10', '11', '12', '13', '14', '15', '16', '17', '18', '19', '20', '21', '22', '23', '24', '25', '26', '27', '28', '29', '30', '31', '32', '33', '34', '35', '36', '37', '38', '39', '40', '41', '42', '43', '44', '45', '46', '47', '48', '49', '50', '51', '52', '53', '54', '55', '56', '57', '58', '59', '60', '61', '62', '63', '64', '65', '66', '67', '68', '69', '70', '71', '72', '73', '74', '75', '76', '77', '78', '79', '80', '81', '82', '83', '84', '85', '86', '87', '88', '89', '90', '91', '92', '93', '94', '95', '96', '97', '98', '99', '100', '101', '102', '103', '104', '105', '106', '107', '108', '109', '110', '111', '112', '113', '114', '115', '116', '117', '118', '119', '120', '121', '122', '123', '124', '125', '126', '127', '128', '129', '130', '131', '132', '133', '134', '135', '136', '137', '138', '139', '140', '141', '142', '143', '144', '145', '146', '147', '148', '149', '150', '151', '152', '153', '154', '155', '156', '157', '158', '159', '160', '161', '162', '163', '164', '165', '166', '167', '168', '169', '170', '171', '172', '173', '174', '175', '176', '177', '178', '179', '180', '181', '182', '183', '184', '185', '186', '187', '188', '189', '190', '191', '192', '193', '194', '195', '196', '197', '198', '199', '200', '201', '202', '203', '204', '205', '206', '207', '208', '209', '210', '211', '212', '213', '214', '215', '216', '217', '218', '219', '220', '221', '222', '223', '224', '225', '226', '227', '228', '229', '230', '231', '232', '233', '234', '235', '236', '237', '238', '239', '240', '241', '242', '243', '244', '245', '246', '247', '248', '249', '250', '251', '252', '253', '254', '255', '256', '257', '258', '259', '260', '261', '262', '263', '264', '265', '266', '267', '268', '269', '270', '271', '272', '273', '274', '275', '276', '277', '278', '279', '280', '281', '282', '283', '284', '285', '286', '287', '288', '289', '290', '291', '292', '293', '294', '295', '296', '297', '298', '299', '300', '301', '302', '303', '304', '305', '306', '307', '308', '309', '310', '311', '312', '313', '314', '315', '316', '317', '318', '319', '320', '321', '322', '323', '324', '325', '326', '327', '328', '329', '330', '331', '332', '333', '334', '335', '336', '337', '338', '339', '340', '341', '342', '343', '344', '345', '346', '347', '348', '349', '350', '351', '352', '353', '354', '355', '356', '357', '358', '359', '360', '361', '362', '363', '364', '365', '366', '367', '368', '369', '370', '371', '372', '373', '374', '375', '376', '377', '378', '379', '380', '381', '382', '383', '384', '385', '386', '387', '388', '389', '390', '391', '392', '393', '394', '395', '396', '397', '398', '399', '400', '401', '402', '403', '404', '405', '406', '407', '408', '409', '410', '411', '412', '413', '414', '415', '416', '417', '418', '419', '420', '421', '422', '423', '424', '425', '426', '427', '428', '429', '430', '431', '432', '433', '434', '435', '436', '437', '438', '439', '440', '441', '442', '443', '444', '445', '446', '447', '448', '449', '450', '451', '452', '453', '454', '455', '456', '457', '458', '459', '460', '461', '462', '463', '464', '465', '466', '467', '468', '469', '470', '471', '472', '473', '474', '475', '476', '477', '478', '479', '480', '481', '482', '483', '484', '485', '486', '487', '488', '489', '490', '491', '492', '493', '494', '495', '496', '497', '498', '499', '500', '501', '502', '503', '504', '505', '506', '507', '508', '509', '510', '511', '512', '513', '514', '515', '516', '517', '518', '519', '520', '521', '522', '523', '524', '525', '526', '527', '528', '529', '530', '531', '532', '533', '534', '535', '536', '537', '538', '539', '540', '541', '542', '543', '544', '545', '546', '547', '548', '549', '550', '551', '552', '553', '554', '555', '556', '557', '558', '559', '560', '561', '562', '563', '564', '565', '566', '567', '568', '569', '570', '571', '572', '573', '574', '575', '576', '577', '578', '579', '580', '581', '582', '583', '584', '585', '586', '587', '588', '589', '590', '591', '592', '593', '594', '595', '596', '597', '598', '599', '600', '601', '602', '603', '604', '605', '606', '607', '608', '609', '610', '611', '612', '613', '614', '615', '616', '617', '618', '619', '620', '621', '622', '623', '624', '625', '626', '627', '628', '629', '630', '631', '632', '633', '634', '635', '636', '637', '638', '639', '640', '641', '642', '643', '644', '645', '646', '647', '648', '649', '650', '651', '652', '653', '654', '655', '656', '657', '658', '659', '660', '661', '662', '663', '664', '665', '666', '667', '668', '669', '670', '671', '672', '673', '674', '675', '676', '677', '678', '679', '680', '681', '682', '683', '684', '685', '686', '687', '688', '689', '690', '691', '692', '693', '694', '695', '696', '697', '698', '699', '700', '701', '702', '703', '704', '705', '706', '707', '708', '709', '710', '711', '712', '713', '714', '715', '716', '717', '718', '719', '720', '721', '722', '723', '724', '725', '726', '727', '728', '729', '730', '731', '732', '733', '734', '735', '736', '737', '738', '739', '740', '741', '742', '743', '744', '745', '746', '747', '748', '749', '750', '751', '752', '753', '754', '755', '756', '757', '758', '759', '760', '761', '762', '763', '764', '765', '766', '767', '768', '769', '770', '771', '772', '773', '774', '775', '776', '777', '778', '779', '780', '781', '782', '783', '784', '785', '786', '787', '788', '789', '790', '791', '792', '793', '794', '795', '796', '797', '798', '799', '800', '801', '802', '803', '804', '805', '806', '807', '808', '809', '810', '811', '812', '813', '814', '815', '816', '817', '818', '819', '820', '821', '822', '823', '824', '825', '826', '827', '828', '829', '830', '831', '832', '833', '834', '835', '836', '837', '838', '839', '840', '841', '842', '843', '844', '845', '846', '847', '848', '849', '850', '851', '852', '853', '854', '855', '856', '857', '858', '859', '860', '861', '862', '863', '864', '865', '866', '867', '868', '869', '870', '871', '872', '873', '874', '875', '876', '877', '878', '879', '880', '881', '882', '883', '884', '885', '886', '887', '888', '889', '890', '891', '892', '893', '894', '895', '896', '897', '898', '899', '900', '901', '902', '903', '904', '905', '906', '907', '908', '909', '910', '911', '912', '913', '914', '915', '916', '917', '918', '919', '920', '921', '922', '923', '924', '925', '926', '927', '928', '929', '930', '931', '932', '933', '934', '935', '936', '937', '938', '939', '940', '941', '942', '943', '944', '945', '946', '947', '948', '949', '950', '951', '952', '953', '954', '955', '956', '957', '958', '959', '960', '961', '962', '963', '964', '965', '966', '967', '968', '969', '970', '971', '972', '973', '974', '975', '976', '977', '978', '979', '980', '981', '982', '983', '984', '985', '986', '987', '988', '989', '990', '991', '992', '993', '994', '995', '996', '997', '998') " + + " ) order by 1" ); @Param({"5000000"}) From d5cb45174239267f4d64277bf56a00f6b9186d6e Mon Sep 17 00:00:00 2001 From: Soumyava Das Date: Wed, 12 Oct 2022 14:42:49 -0700 Subject: [PATCH 09/10] New test with alias and expression in subquery --- .../druid/sql/calcite/CalciteQueryTest.java | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java index 7970133e2a65..9f22dfc6bac1 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java @@ -100,6 +100,7 @@ import org.apache.druid.query.scan.ScanQuery; import org.apache.druid.query.scan.ScanQuery.ResultFormat; import org.apache.druid.query.spec.MultipleIntervalSegmentSpec; +import org.apache.druid.query.spec.QuerySegmentSpec; import org.apache.druid.query.topn.DimensionTopNMetricSpec; import org.apache.druid.query.topn.InvertedTopNMetricSpec; import org.apache.druid.query.topn.NumericTopNMetricSpec; @@ -11322,6 +11323,33 @@ public void testOrderByAlongWithAliasOrderByTimeGroupByOneCol() ); } + @Test + public void testOrderByAlongWithAliasInInnerSubQuery() + { + testQuery( + "select * from (select __time as t, dim2 as d, m1+m2 as sumval from druid.foo) order by 1 limit 1", + ImmutableList.of( + newScanQueryBuilder() + .dataSource(CalciteTests.DATASOURCE1) + .intervals(querySegmentSpec(Filtration.eternity())) + .virtualColumns(expressionVirtualColumn( + "v0", + "(\"m1\" + \"m2\")", + ColumnType.FLOAT + )) + .columns("__time", "dim2", "v0") + .resultFormat(ScanQuery.ResultFormat.RESULT_FORMAT_COMPACTED_LIST) + .context(QUERY_CONTEXT_DEFAULT) + .limit(1) + .order(ScanQuery.Order.ASCENDING) + .build() + ), + ImmutableList.of( + new Object[]{946684800000L, "a", 2.0f} + ) + ); + } + @Test public void testProjectAfterSort() { From 79277de214a6e6648e6222237ea5a77e65266304 Mon Sep 17 00:00:00 2001 From: Soumyava Das Date: Wed, 12 Oct 2022 15:09:21 -0700 Subject: [PATCH 10/10] checkstyle fix --- .../test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java | 1 - 1 file changed, 1 deletion(-) diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java index 9f22dfc6bac1..59b4103d7d41 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java @@ -100,7 +100,6 @@ import org.apache.druid.query.scan.ScanQuery; import org.apache.druid.query.scan.ScanQuery.ResultFormat; import org.apache.druid.query.spec.MultipleIntervalSegmentSpec; -import org.apache.druid.query.spec.QuerySegmentSpec; import org.apache.druid.query.topn.DimensionTopNMetricSpec; import org.apache.druid.query.topn.InvertedTopNMetricSpec; import org.apache.druid.query.topn.NumericTopNMetricSpec;