diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/rel/DruidOuterQueryRel.java b/sql/src/main/java/org/apache/druid/sql/calcite/rel/DruidOuterQueryRel.java index 3171eb58c02e..c49fc6bd04e6 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/rel/DruidOuterQueryRel.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/rel/DruidOuterQueryRel.java @@ -34,6 +34,7 @@ import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.query.Druids; import org.apache.druid.query.QueryDataSource; +import org.apache.druid.query.TableDataSource; import org.apache.druid.segment.column.RowSignature; import org.apache.druid.sql.calcite.planner.PlannerContext; import org.apache.druid.sql.calcite.table.RowSignatures; @@ -46,7 +47,9 @@ */ public class DruidOuterQueryRel extends DruidRel { - private static final QueryDataSource DUMMY_DATA_SOURCE = new QueryDataSource( + private static final TableDataSource DUMMY_DATA_SOURCE = new TableDataSource("__subquery__"); + + private static final QueryDataSource DUMMY_QUERY_DATA_SOURCE = new QueryDataSource( Druids.newScanQueryBuilder().dataSource("__subquery__").eternityInterval().build() ); @@ -117,7 +120,7 @@ public DruidQuery toDruidQuery(final boolean finalizeAggregations) public DruidQuery toDruidQueryForExplaining() { return partialQuery.build( - DUMMY_DATA_SOURCE, + partialQuery.getWindow() == null ? DUMMY_DATA_SOURCE : DUMMY_QUERY_DATA_SOURCE, RowSignatures.fromRelDataType( sourceRel.getRowType().getFieldNames(), sourceRel.getRowType() diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/rel/DruidQuery.java b/sql/src/main/java/org/apache/druid/sql/calcite/rel/DruidQuery.java index 2a093e6a3012..f24f0381451f 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/rel/DruidQuery.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/rel/DruidQuery.java @@ -1439,7 +1439,8 @@ private ScanQuery toScanQuery() } if (!plannerContext.featureAvailable(EngineFeature.SCAN_ORDER_BY_NON_TIME) && !orderByColumns.isEmpty()) { - if (orderByColumns.size() > 1 || !ColumnHolder.TIME_COLUMN_NAME.equals(orderByColumns.get(0).getColumnName())) { + if (orderByColumns.size() > 1 || orderByColumns.stream() + .anyMatch(orderBy -> !orderBy.getColumnName().equals(ColumnHolder.TIME_COLUMN_NAME))) { // Cannot handle this ordering. // Scan cannot ORDER BY non-time columns. plannerContext.setPlanningError( @@ -1448,15 +1449,6 @@ private ScanQuery toScanQuery() ); return null; } - if (!dataSource.isConcrete()) { - // Cannot handle this ordering. - // Scan cannot ORDER BY non-concrete datasources on _any_ column. - plannerContext.setPlanningError( - "SQL query requires order by on non-concrete datasource [%s], which is not supported.", - dataSource - ); - return null; - } } // Compute the list of columns to select, sorted and deduped. diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteExplainQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteExplainQueryTest.java index 70e0320e3a15..480fca6aeb88 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteExplainQueryTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteExplainQueryTest.java @@ -109,7 +109,7 @@ public void testExplainExactCountDistinctOfSemiJoinResult() + " )\n" + ")"; final String legacyExplanation = - "DruidOuterQueryRel(query=[{\"queryType\":\"groupBy\",\"dataSource\":{\"type\":\"query\",\"query\":{\"queryType\":\"scan\",\"dataSource\":{\"type\":\"table\",\"name\":\"__subquery__\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"resultFormat\":\"list\",\"granularity\":{\"type\":\"all\"}}},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"granularity\":{\"type\":\"all\"},\"dimensions\":[],\"aggregations\":[{\"type\":\"count\",\"name\":\"a0\"}],\"limitSpec\":{\"type\":\"NoopLimitSpec\"},\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\",\"sqlQueryId\":\"dummy\",\"vectorize\":\"false\",\"vectorizeVirtualColumns\":\"false\"}}], signature=[{a0:LONG}])\n" + "DruidOuterQueryRel(query=[{\"queryType\":\"timeseries\",\"dataSource\":{\"type\":\"table\",\"name\":\"__subquery__\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"granularity\":{\"type\":\"all\"},\"aggregations\":[{\"type\":\"count\",\"name\":\"a0\"}],\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\",\"sqlQueryId\":\"dummy\",\"vectorize\":\"false\",\"vectorizeVirtualColumns\":\"false\"}}], signature=[{a0:LONG}])\n" + " DruidJoinQueryRel(condition=[=(SUBSTRING($2, 1, 1), $8)], joinType=[inner], query=[{\"queryType\":\"groupBy\",\"dataSource\":{\"type\":\"table\",\"name\":\"__join__\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"granularity\":{\"type\":\"all\"},\"dimensions\":[{\"type\":\"default\",\"dimension\":\"dim2\",\"outputName\":\"d0\",\"outputType\":\"STRING\"}],\"limitSpec\":{\"type\":\"NoopLimitSpec\"},\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\",\"sqlQueryId\":\"dummy\",\"vectorize\":\"false\",\"vectorizeVirtualColumns\":\"false\"}}], signature=[{d0:STRING}])\n" + " DruidQueryRel(query=[{\"queryType\":\"scan\",\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"resultFormat\":\"compactedList\",\"columns\":[\"__time\",\"cnt\",\"dim1\",\"dim2\",\"dim3\",\"m1\",\"m2\",\"unique_dim1\"],\"legacy\":false,\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\",\"sqlQueryId\":\"dummy\",\"vectorize\":\"false\",\"vectorizeVirtualColumns\":\"false\"},\"granularity\":{\"type\":\"all\"}}], signature=[{__time:LONG, dim1:STRING, dim2:STRING, dim3:STRING, cnt:LONG, m1:FLOAT, m2:DOUBLE, unique_dim1:COMPLEX}])\n" + " DruidQueryRel(query=[{\"queryType\":\"groupBy\",\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"filter\":{\"type\":\"not\",\"field\":{\"type\":\"selector\",\"dimension\":\"dim1\",\"value\":null}},\"granularity\":{\"type\":\"all\"},\"dimensions\":[{\"type\":\"extraction\",\"dimension\":\"dim1\",\"outputName\":\"d0\",\"outputType\":\"STRING\",\"extractionFn\":{\"type\":\"substring\",\"index\":0,\"length\":1}}],\"limitSpec\":{\"type\":\"NoopLimitSpec\"},\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\",\"sqlQueryId\":\"dummy\",\"vectorize\":\"false\",\"vectorizeVirtualColumns\":\"false\"}}], signature=[{d0:STRING}])\n"; diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteJoinQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteJoinQueryTest.java index 32007ed7abb3..be65a64e80a1 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteJoinQueryTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteJoinQueryTest.java @@ -5264,4 +5264,62 @@ public void testRegressionFilteredAggregatorsSubqueryJoins(Map q ) ); } + + @Test + public void testJoinWithAliasAndOrderByNoGroupBy() + { + minTopNThreshold = 1; + Map context = new HashMap<>(QUERY_CONTEXT_DEFAULT); + context.put(PlannerConfig.CTX_KEY_USE_APPROXIMATE_TOPN, false); + testQuery( + "select t1.__time from druid.foo as t1 join\n" + + " druid.numfoo as t2 on t1.dim2 = t2.dim2\n" + + " order by t1.__time ASC ", + context, // turn on exact topN + ImmutableList.of( + newScanQueryBuilder() + .intervals(querySegmentSpec(Filtration.eternity())) + .dataSource( + JoinDataSource.create( + new TableDataSource(CalciteTests.DATASOURCE1), + new QueryDataSource( + newScanQueryBuilder() + .dataSource(CalciteTests.DATASOURCE3) + .intervals(querySegmentSpec(Intervals.of( + "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z"))) + .columns("dim2") + .context(context) + .build() + ), + "j0.", + "(\"dim2\" == \"j0.dim2\")", + JoinType.INNER, + null, + ExprMacroTable.nil(), + CalciteTests.createJoinableFactoryWrapper() + ) + ) + .columns("__time") + .order(ScanQuery.Order.ASCENDING) + .context(context) + .build() + ), + NullHandling.sqlCompatible() + ? ImmutableList.of( + new Object[]{946684800000L}, + new Object[]{946684800000L}, + new Object[]{946857600000L}, + new Object[]{978307200000L}, + new Object[]{978307200000L}, + new Object[]{978393600000L} + ) + : ImmutableList.of( + new Object[]{946684800000L}, + new Object[]{946684800000L}, + new Object[]{978307200000L}, + new Object[]{978307200000L}, + new Object[]{978393600000L} + ) + ); + } } diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java index 78e2e50f3b7a..f086aa8fe02a 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java @@ -7306,14 +7306,9 @@ public void testMinMaxAvgDailyCountWithLimit() ) ) .setInterval(querySegmentSpec(Filtration.eternity())) - .setLimitSpec( - new DefaultLimitSpec( - ImmutableList.of(), - 1 - ) - ) .setGranularity(Granularities.ALL) .setAggregatorSpecs( + useDefault ? aggregators( new LongMaxAggregatorFactory("_a0", "a0"), new LongMinAggregatorFactory("_a1", "a0"), @@ -7321,6 +7316,16 @@ public void testMinMaxAvgDailyCountWithLimit() new CountAggregatorFactory("_a2:count"), new LongMaxAggregatorFactory("_a3", "d0"), new CountAggregatorFactory("_a4") + ) : aggregators( + new LongMaxAggregatorFactory("_a0", "a0"), + new LongMinAggregatorFactory("_a1", "a0"), + new LongSumAggregatorFactory("_a2:sum", "a0"), + new FilteredAggregatorFactory( + new CountAggregatorFactory("_a2:count"), + not(selector("a0", null, null)) + ), + new LongMaxAggregatorFactory("_a3", "d0"), + new CountAggregatorFactory("_a4") ) ) .setPostAggregatorSpecs( @@ -7333,7 +7338,7 @@ public void testMinMaxAvgDailyCountWithLimit() new FieldAccessPostAggregator(null, "_a2:count") ) ), - expressionPostAgg("s0", "timestamp_extract(\"_a3\",'EPOCH','UTC')") + expressionPostAgg("p0", "timestamp_extract(\"_a3\",'EPOCH','UTC')") ) ) .setContext(QUERY_CONTEXT_DEFAULT) @@ -14604,4 +14609,88 @@ public void testComplexDecodeAgg() ) ); } + + @Test + public void testOrderByAlongWithInternalScanQuery() + { + testQuery( + "select __time as t, m1 from druid.foo where (m1 in (select distinct m1 from druid.foo)) order by 1 limit 1", + ImmutableList.of( + newScanQueryBuilder() + .intervals(querySegmentSpec(Filtration.eternity())) + .dataSource( + JoinDataSource.create( + new TableDataSource(CalciteTests.DATASOURCE1), + new QueryDataSource( + GroupByQuery.builder() + .setGranularity(Granularities.ALL) + .setDataSource(CalciteTests.DATASOURCE1) + .setInterval(querySegmentSpec(Intervals.of( + "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z"))) + .setDimensions(new DefaultDimensionSpec("m1", "d0", ColumnType.FLOAT)) + .build() + ), + "j0.", + "(\"m1\" == \"j0.d0\")", + JoinType.INNER, + null, + ExprMacroTable.nil(), + CalciteTests.createJoinableFactoryWrapper() + ) + ) + .context(QUERY_CONTEXT_DEFAULT) + .intervals(querySegmentSpec(Intervals.of( + "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z"))) + .limit(1) + .columns(ImmutableList.of("__time", "m1")) + .order(ScanQuery.Order.ASCENDING) + .build() + ), + ImmutableList.of( + new Object[]{946684800000L, 1.0f} + ) + ); + } + + @Test + public void testOrderByAlongWithInternalScanQueryNoDistinct() + { + testQuery( + "select __time, m1 from druid.foo where (m1 in (select m1 from druid.foo)) order by __time DESC limit 1", + ImmutableList.of( + newScanQueryBuilder() + .intervals(querySegmentSpec(Filtration.eternity())) + .dataSource( + JoinDataSource.create( + new TableDataSource(CalciteTests.DATASOURCE1), + new QueryDataSource( + GroupByQuery.builder() + .setGranularity(Granularities.ALL) + .setDataSource(CalciteTests.DATASOURCE1) + .setInterval(querySegmentSpec(Intervals.of( + "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z"))) + .setDimensions(new DefaultDimensionSpec("m1", "d0", ColumnType.FLOAT)) + .build() + ), + "j0.", + "(\"m1\" == \"j0.d0\")", + JoinType.INNER, + null, + ExprMacroTable.nil(), + CalciteTests.createJoinableFactoryWrapper() + ) + ) + .context(QUERY_CONTEXT_DEFAULT) + .intervals(querySegmentSpec(Intervals.of( + "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z"))) + .limit(1) + .columns(ImmutableList.of("__time", "m1")) + .order(ScanQuery.Order.DESCENDING) + .build() + ), + ImmutableList.of( + new Object[]{978480000000L, 6.0f} + ) + ); + } }