From c18be9e7c404fb77a9bd7e2df3cff9722160411b Mon Sep 17 00:00:00 2001 From: Laksh Singla Date: Thu, 11 Nov 2021 12:43:12 +0530 Subject: [PATCH 01/18] Initial commit, first implementation --- .../sql/calcite/planner/DruidPlanner.java | 84 ++++++++++++++++++- .../druid/sql/calcite/CalciteQueryTest.java | 22 +++++ 2 files changed, 104 insertions(+), 2 deletions(-) diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/planner/DruidPlanner.java b/sql/src/main/java/org/apache/druid/sql/calcite/planner/DruidPlanner.java index a1b9149be6ac..2915959c2dbf 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/planner/DruidPlanner.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/planner/DruidPlanner.java @@ -25,6 +25,7 @@ import com.google.common.base.Supplier; import com.google.common.base.Suppliers; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import org.apache.calcite.DataContext; import org.apache.calcite.adapter.java.JavaTypeFactory; import org.apache.calcite.config.CalciteConnectionConfig; @@ -42,13 +43,19 @@ import org.apache.calcite.prepare.Prepare; import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.RelRoot; +import org.apache.calcite.rel.RelWriter; import org.apache.calcite.rel.core.Sort; +import org.apache.calcite.rel.externalize.RelJsonWriter; +import org.apache.calcite.rel.externalize.RelWriterImpl; +import org.apache.calcite.rel.externalize.RelXmlWriter; import org.apache.calcite.rel.logical.LogicalSort; import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rel.type.RelDataTypeFactory; import org.apache.calcite.rex.RexBuilder; import org.apache.calcite.rex.RexNode; import org.apache.calcite.sql.SqlExplain; +import org.apache.calcite.sql.SqlExplainFormat; +import org.apache.calcite.sql.SqlExplainLevel; import org.apache.calcite.sql.SqlKind; import org.apache.calcite.sql.SqlNode; import org.apache.calcite.sql.parser.SqlParseException; @@ -65,15 +72,23 @@ import org.apache.druid.java.util.common.guava.Sequence; import org.apache.druid.java.util.common.guava.Sequences; import org.apache.druid.java.util.emitter.EmittingLogger; +import org.apache.druid.query.Query; import org.apache.druid.segment.DimensionHandlerUtils; import org.apache.druid.sql.calcite.rel.DruidConvention; +import org.apache.druid.sql.calcite.rel.DruidOuterQueryRel; +import org.apache.druid.sql.calcite.rel.DruidQuery; import org.apache.druid.sql.calcite.rel.DruidRel; +import org.apache.druid.sql.calcite.rel.DruidUnionRel; import javax.annotation.Nullable; + import java.io.Closeable; +import java.io.PrintWriter; +import java.io.StringWriter; import java.util.ArrayList; import java.util.Iterator; import java.util.List; +import java.util.Map; import java.util.Properties; public class DruidPlanner implements Closeable @@ -249,7 +264,7 @@ private PlannerResult planWithDruidConvention( ); if (explain != null) { - return planExplanation(druidRel, explain); + return planDruidExplanation(druidRel); } else { final Supplier> resultsSupplier = () -> { // sanity check @@ -384,6 +399,72 @@ private PlannerResult planExplanation( return new PlannerResult(resultsSupplier, getExplainStructType(rel.getCluster().getTypeFactory())); } + /** + * Construct a {@link PlannerResult} for an 'explain' query from a {@link RelNode} + */ + private PlannerResult planDruidExplanation( + final DruidRel rel + ) + { + String resources = null; + String explanation = null; + try { + explanation = dumpDruidPlan(rel); + resources = jsonMapper.writeValueAsString(plannerContext.getResources()); + } + catch (JsonProcessingException jpe) { + // this should never happen, we create the Resources here, not a user + log.error(jpe, "Encountered exception while serializing Resources for explain output"); + } + final Supplier> resultsSupplier = Suppliers.ofInstance( + Sequences.simple(ImmutableList.of(new Object[]{explanation, resources}))); + return new PlannerResult(resultsSupplier, getExplainStructType(rel.getCluster().getTypeFactory())); + } + + /** + * This method doesn't utilize the Calcite's internal {@link RelOptUtil#dumpPlan} since that tends to be verbose + * and not indicative of the native Druid Queries which will get executed + * This method assumes that the Planner has converted the RelNodes to DruidRels, and thereby we can implictly cast it + * + * @param rel Instance of the root {@link DruidRel} which is formed by running the planner transformations on it + * @return A string representing an array of native queries that correspond to the given SQL query, in JSON format + * @throws JsonProcessingException + */ + public static String dumpDruidPlan(DruidRel rel) throws JsonProcessingException + { + List> nativeQueries; + + // Only if rel is an instance of DruidUnionRel, do we run multiple native queries corresponding to single SQL query + // Also, DruidUnionRel can only be a top level node, so we don't need to check for this condition in the subsequent + // child nodes + if (rel instanceof DruidUnionRel) { + nativeQueries = new ArrayList<>(); + for (RelNode childRel : rel.getInputs()) { + DruidRel druidChildRel = (DruidRel) childRel; + if (childRel instanceof DruidUnionRel) { // Failsafe block, shouldn't be encountered + log.error("DruidUnionRel can only be the outermost Rel. This error shouldn't be encountered"); + // TODO: Throw an error here + } + DruidQuery query1 = druidChildRel.toDruidQuery(false); // Finalize aggregations + nativeQueries.add(ImmutableMap.of( + "query", + query1.getQuery(), + "signature", + query1.getOutputRowSignature().toString() + )); + } + } else { + DruidQuery query = rel.toDruidQuery(false); + nativeQueries = ImmutableList.of(ImmutableMap.of( + "query", + query.getQuery(), + "signature", + query.getOutputRowSignature().toString() + )); + } + return rel.getQueryMaker().getJsonMapper().writeValueAsString(nativeQueries); + } + /** * This method wraps the root with a {@link LogicalSort} that applies a limit (no ordering change). If the outer rel * is already a {@link Sort}, we can merge our outerLimit into it, similar to what is going on in @@ -393,7 +474,6 @@ private PlannerResult planExplanation( * the web console, allowing it to apply a limit to queries without rewriting the original SQL. * * @param root root node - * * @return root node wrapped with a limiting logical sort if a limit is specified in the query context. */ @Nullable diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java index a632f0eab385..7498bc6daf1b 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java @@ -8955,6 +8955,28 @@ public void testMaxSubqueryRows() throws Exception ); } + @Test + public void testExplainUnionAllOfQueries() throws Exception + { + skipVectorize(); // Required otherwise results of the EXPLAIN change + String query = "EXPLAIN PLAN FOR (SELECT dim1 FROM druid.foo UNION ALL SELECT dim1 from druid.foo2)"; + String resources = "[{\"name\":\"foo2\",\"type\":\"DATASOURCE\"},{\"name\":\"foo\",\"type\":\"DATASOURCE\"}]"; + String explanation = "[{\"query\":{\"dataSource\":{\"type\":\"union\",\"dataSources\":[{\"type\":\"table\",\"name\":\"foo\"},{\"type\":\"table\",\"name\":\"foo2\"}]},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"virtualColumns\":[],\"resultFormat\":\"compactedList\",\"batchSize\":20480,\"order\":\"none\",\"filter\":null,\"columns\":[\"dim1\"],\"legacy\":false,\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\",\"sqlQueryId\":\"dummy\",\"vectorize\":\"false\",\"vectorizeVirtualColumns\":\"false\"},\"descending\":false,\"granularity\":{\"type\":\"all\"}},\"signature\":\"{dim1:STRING}\"}]"; + testQuery(query, ImmutableList.of(), ImmutableList.of(new Object[]{explanation, resources})); + } + + @Test + public void testExplainUnionOfMultipleNativeQueries() throws Exception + { + skipVectorize(); + String query = "EXPLAIN PLAN FOR ((SELECT dim1 FROM druid.foo) UNION ALL (SELECT dim1 FROM druid.foo4 where dim1 in (SELECT dim1 FROM druid.foo)))"; + String explanation = "[" + + "{\"query\":{\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"virtualColumns\":[],\"resultFormat\":\"compactedList\",\"batchSize\":20480,\"order\":\"none\",\"filter\":null,\"columns\":[\"dim1\"],\"legacy\":false,\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\",\"sqlQueryId\":\"dummy\",\"vectorize\":\"false\",\"vectorizeVirtualColumns\":\"false\"},\"descending\":false,\"granularity\":{\"type\":\"all\"}},\"signature\":\"{dim1:STRING}\"}" + + ",{\"query\":{\"dataSource\":{\"type\":\"join\",\"left\":{\"type\":\"table\",\"name\":\"foo4\"},\"right\":{\"type\":\"query\",\"query\":{\"queryType\":\"groupBy\",\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"virtualColumns\":[],\"filter\":null,\"granularity\":{\"type\":\"all\"},\"dimensions\":[{\"type\":\"default\",\"dimension\":\"dim1\",\"outputName\":\"d0\",\"outputType\":\"STRING\"}],\"aggregations\":[],\"postAggregations\":[],\"having\":null,\"limitSpec\":{\"type\":\"NoopLimitSpec\"},\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\",\"sqlQueryId\":\"dummy\",\"vectorize\":\"false\",\"vectorizeVirtualColumns\":\"false\"},\"descending\":false}},\"rightPrefix\":\"j0.\",\"condition\":\"(\\\"dim1\\\" == \\\"j0.d0\\\")\",\"joinType\":\"INNER\",\"leftFilter\":null},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"virtualColumns\":[],\"resultFormat\":\"compactedList\",\"batchSize\":20480,\"order\":\"none\",\"filter\":null,\"columns\":[\"dim1\"],\"legacy\":false,\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\",\"sqlQueryId\":\"dummy\",\"vectorize\":\"false\",\"vectorizeVirtualColumns\":\"false\"},\"descending\":false,\"granularity\":{\"type\":\"all\"}},\"signature\":\"{dim1:STRING}\"}]"; + String resources = "[{\"name\":\"foo4\",\"type\":\"DATASOURCE\"},{\"name\":\"foo\",\"type\":\"DATASOURCE\"}]"; + testQuery(query, ImmutableList.of(), ImmutableList.of(new Object[]{explanation, resources})); + } + @Test public void testExplainExactCountDistinctOfSemiJoinResult() throws Exception { From 336e0ba7dd85eccc9843cc7f8f6dbadc48b0f472 Mon Sep 17 00:00:00 2001 From: Laksh Singla Date: Thu, 11 Nov 2021 19:48:11 +0530 Subject: [PATCH 02/18] Clean up the code, reuse existing implementation of explain planner --- .../sql/calcite/planner/DruidPlanner.java | 98 +++++++------------ 1 file changed, 37 insertions(+), 61 deletions(-) diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/planner/DruidPlanner.java b/sql/src/main/java/org/apache/druid/sql/calcite/planner/DruidPlanner.java index 2915959c2dbf..73b82f3e7845 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/planner/DruidPlanner.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/planner/DruidPlanner.java @@ -21,11 +21,12 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.base.Preconditions; import com.google.common.base.Supplier; import com.google.common.base.Suppliers; import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; import org.apache.calcite.DataContext; import org.apache.calcite.adapter.java.JavaTypeFactory; import org.apache.calcite.config.CalciteConnectionConfig; @@ -43,19 +44,13 @@ import org.apache.calcite.prepare.Prepare; import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.RelRoot; -import org.apache.calcite.rel.RelWriter; import org.apache.calcite.rel.core.Sort; -import org.apache.calcite.rel.externalize.RelJsonWriter; -import org.apache.calcite.rel.externalize.RelWriterImpl; -import org.apache.calcite.rel.externalize.RelXmlWriter; import org.apache.calcite.rel.logical.LogicalSort; import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rel.type.RelDataTypeFactory; import org.apache.calcite.rex.RexBuilder; import org.apache.calcite.rex.RexNode; import org.apache.calcite.sql.SqlExplain; -import org.apache.calcite.sql.SqlExplainFormat; -import org.apache.calcite.sql.SqlExplainLevel; import org.apache.calcite.sql.SqlKind; import org.apache.calcite.sql.SqlNode; import org.apache.calcite.sql.parser.SqlParseException; @@ -75,7 +70,6 @@ import org.apache.druid.query.Query; import org.apache.druid.segment.DimensionHandlerUtils; import org.apache.druid.sql.calcite.rel.DruidConvention; -import org.apache.druid.sql.calcite.rel.DruidOuterQueryRel; import org.apache.druid.sql.calcite.rel.DruidQuery; import org.apache.druid.sql.calcite.rel.DruidRel; import org.apache.druid.sql.calcite.rel.DruidUnionRel; @@ -83,13 +77,11 @@ import javax.annotation.Nullable; import java.io.Closeable; -import java.io.PrintWriter; -import java.io.StringWriter; import java.util.ArrayList; import java.util.Iterator; import java.util.List; -import java.util.Map; import java.util.Properties; +import java.util.stream.Collectors; public class DruidPlanner implements Closeable { @@ -264,7 +256,7 @@ private PlannerResult planWithDruidConvention( ); if (explain != null) { - return planDruidExplanation(druidRel); + return planExplanation(druidRel, explain, true); } else { final Supplier> resultsSupplier = () -> { // sanity check @@ -332,7 +324,7 @@ private PlannerResult planWithBindableConvention( } if (explain != null) { - return planExplanation(bindableRel, explain); + return planExplanation(bindableRel, explain, false); } else { final BindableRel theRel = bindableRel; final DataContext dataContext = plannerContext.createDataContext( @@ -381,12 +373,21 @@ public void cleanup(EnumeratorIterator iterFromMake) */ private PlannerResult planExplanation( final RelNode rel, - final SqlExplain explain + final SqlExplain explain, + final boolean isDruidConventionExplanation ) { - final String explanation = RelOptUtil.dumpPlan("", rel, explain.getFormat(), explain.getDetailLevel()); + String explanation = RelOptUtil.dumpPlan("", rel, explain.getFormat(), explain.getDetailLevel()); String resources; try { + if (isDruidConventionExplanation && rel instanceof DruidRel) { + DruidRel druidRel = (DruidRel) rel; + if (true) { + explanation = explainSqlPlanAsNativeQueries(druidRel); + } else { + explanation = RelOptUtil.dumpPlan("", rel, explain.getFormat(), explain.getDetailLevel()); + } + } resources = jsonMapper.writeValueAsString(plannerContext.getResources()); } catch (JsonProcessingException jpe) { @@ -399,28 +400,6 @@ private PlannerResult planExplanation( return new PlannerResult(resultsSupplier, getExplainStructType(rel.getCluster().getTypeFactory())); } - /** - * Construct a {@link PlannerResult} for an 'explain' query from a {@link RelNode} - */ - private PlannerResult planDruidExplanation( - final DruidRel rel - ) - { - String resources = null; - String explanation = null; - try { - explanation = dumpDruidPlan(rel); - resources = jsonMapper.writeValueAsString(plannerContext.getResources()); - } - catch (JsonProcessingException jpe) { - // this should never happen, we create the Resources here, not a user - log.error(jpe, "Encountered exception while serializing Resources for explain output"); - } - final Supplier> resultsSupplier = Suppliers.ofInstance( - Sequences.simple(ImmutableList.of(new Object[]{explanation, resources}))); - return new PlannerResult(resultsSupplier, getExplainStructType(rel.getCluster().getTypeFactory())); - } - /** * This method doesn't utilize the Calcite's internal {@link RelOptUtil#dumpPlan} since that tends to be verbose * and not indicative of the native Druid Queries which will get executed @@ -430,39 +409,36 @@ private PlannerResult planDruidExplanation( * @return A string representing an array of native queries that correspond to the given SQL query, in JSON format * @throws JsonProcessingException */ - public static String dumpDruidPlan(DruidRel rel) throws JsonProcessingException + public static String explainSqlPlanAsNativeQueries(DruidRel rel) throws JsonProcessingException { - List> nativeQueries; - // Only if rel is an instance of DruidUnionRel, do we run multiple native queries corresponding to single SQL query // Also, DruidUnionRel can only be a top level node, so we don't need to check for this condition in the subsequent // child nodes + List druidQueryList; if (rel instanceof DruidUnionRel) { - nativeQueries = new ArrayList<>(); - for (RelNode childRel : rel.getInputs()) { - DruidRel druidChildRel = (DruidRel) childRel; - if (childRel instanceof DruidUnionRel) { // Failsafe block, shouldn't be encountered - log.error("DruidUnionRel can only be the outermost Rel. This error shouldn't be encountered"); + druidQueryList = rel.getInputs().stream().map(childRel -> (DruidRel) childRel).map(childRel -> { + if (childRel instanceof DruidUnionRel) { + log.error("DruidUnionRel can only be the outermost RelNode. This error shouldn't be encountered"); // TODO: Throw an error here } - DruidQuery query1 = druidChildRel.toDruidQuery(false); // Finalize aggregations - nativeQueries.add(ImmutableMap.of( - "query", - query1.getQuery(), - "signature", - query1.getOutputRowSignature().toString() - )); - } + return childRel.toDruidQuery(false); + }).collect(Collectors.toList()); } else { - DruidQuery query = rel.toDruidQuery(false); - nativeQueries = ImmutableList.of(ImmutableMap.of( - "query", - query.getQuery(), - "signature", - query.getOutputRowSignature().toString() - )); + druidQueryList = ImmutableList.of(rel.toDruidQuery(false)); + } + + ObjectMapper objectMapper = rel.getQueryMaker().getJsonMapper(); + ArrayNode nativeQueriesArrayNode = objectMapper.createArrayNode(); + + for (DruidQuery druidQuery : druidQueryList) { + Query nativeQuery = druidQuery.getQuery(); + ObjectNode objectNode = objectMapper.createObjectNode(); + objectNode.put("query", objectMapper.convertValue(nativeQuery, ObjectNode.class)); + objectNode.put("signature", druidQuery.getOutputRowSignature().toString()); + nativeQueriesArrayNode.add(objectNode); } - return rel.getQueryMaker().getJsonMapper().writeValueAsString(nativeQueries); + + return objectMapper.writeValueAsString(nativeQueriesArrayNode); } /** From ed057ba77475f42f5951066833ae1560b2ee630b Mon Sep 17 00:00:00 2001 From: Laksh Singla Date: Fri, 12 Nov 2021 12:06:46 +0530 Subject: [PATCH 03/18] Update the existing EXPLAIN test cases, add one for UNION ALL --- .../sql/calcite/planner/DruidPlanner.java | 11 +- .../sql/avatica/DruidAvaticaHandlerTest.java | 11 +- .../druid/sql/calcite/CalciteQueryTest.java | 120 +++++++++++++++--- .../druid/sql/http/SqlResourceTest.java | 19 ++- 4 files changed, 126 insertions(+), 35 deletions(-) diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/planner/DruidPlanner.java b/sql/src/main/java/org/apache/druid/sql/calcite/planner/DruidPlanner.java index 73b82f3e7845..de14d45e168c 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/planner/DruidPlanner.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/planner/DruidPlanner.java @@ -409,7 +409,7 @@ private PlannerResult planExplanation( * @return A string representing an array of native queries that correspond to the given SQL query, in JSON format * @throws JsonProcessingException */ - public static String explainSqlPlanAsNativeQueries(DruidRel rel) throws JsonProcessingException + private String explainSqlPlanAsNativeQueries(DruidRel rel) throws JsonProcessingException { // Only if rel is an instance of DruidUnionRel, do we run multiple native queries corresponding to single SQL query // Also, DruidUnionRel can only be a top level node, so we don't need to check for this condition in the subsequent @@ -427,18 +427,17 @@ public static String explainSqlPlanAsNativeQueries(DruidRel rel) throws JsonP druidQueryList = ImmutableList.of(rel.toDruidQuery(false)); } - ObjectMapper objectMapper = rel.getQueryMaker().getJsonMapper(); - ArrayNode nativeQueriesArrayNode = objectMapper.createArrayNode(); + ArrayNode nativeQueriesArrayNode = jsonMapper.createArrayNode(); for (DruidQuery druidQuery : druidQueryList) { Query nativeQuery = druidQuery.getQuery(); - ObjectNode objectNode = objectMapper.createObjectNode(); - objectNode.put("query", objectMapper.convertValue(nativeQuery, ObjectNode.class)); + ObjectNode objectNode = jsonMapper.createObjectNode(); + objectNode.put("query", jsonMapper.convertValue(nativeQuery, ObjectNode.class)); objectNode.put("signature", druidQuery.getOutputRowSignature().toString()); nativeQueriesArrayNode.add(objectNode); } - return objectMapper.writeValueAsString(nativeQueriesArrayNode); + return jsonMapper.writeValueAsString(nativeQueriesArrayNode); } /** diff --git a/sql/src/test/java/org/apache/druid/sql/avatica/DruidAvaticaHandlerTest.java b/sql/src/test/java/org/apache/druid/sql/avatica/DruidAvaticaHandlerTest.java index a51fe0a53d0a..88d672ea5d8d 100644 --- a/sql/src/test/java/org/apache/druid/sql/avatica/DruidAvaticaHandlerTest.java +++ b/sql/src/test/java/org/apache/druid/sql/avatica/DruidAvaticaHandlerTest.java @@ -363,13 +363,18 @@ public void testExplainSelectCount() throws Exception "EXPLAIN PLAN FOR SELECT COUNT(*) AS cnt FROM druid.foo" ); + /*final String explanation = StringUtils.format( + "DruidQueryRel(query=[{\"queryType\":\"timeseries\",\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"descending\":false,\"virtualColumns\":[],\"filter\":null,\"granularity\":{\"type\":\"all\"},\"aggregations\":[{\"type\":\"count\",\"name\":\"a0\"}],\"postAggregations\":[],\"limit\":2147483647,\"context\":{\"sqlQueryId\":\"%s\",\"sqlStringifyArrays\":false,\"sqlTimeZone\":\"America/Los_Angeles\"}}], signature=[{a0:LONG}])\n", + DUMMY_SQL_QUERY_ID + )*/ + + final String explanation = "[{\"query\":{\"queryType\":\"timeseries\",\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"descending\":false,\"virtualColumns\":[],\"filter\":null,\"granularity\":{\"type\":\"all\"},\"aggregations\":[{\"type\":\"count\",\"name\":\"a0\"}],\"postAggregations\":[],\"limit\":2147483647,\"context\":{\"sqlQueryId\":\"dummy\",\"sqlStringifyArrays\":false,\"sqlTimeZone\":\"America/Los_Angeles\"}},\"signature\":\"{a0:LONG}\"}]"; + Assert.assertEquals( ImmutableList.of( ImmutableMap.of( "PLAN", - StringUtils.format("DruidQueryRel(query=[{\"queryType\":\"timeseries\",\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"descending\":false,\"virtualColumns\":[],\"filter\":null,\"granularity\":{\"type\":\"all\"},\"aggregations\":[{\"type\":\"count\",\"name\":\"a0\"}],\"postAggregations\":[],\"limit\":2147483647,\"context\":{\"sqlQueryId\":\"%s\",\"sqlStringifyArrays\":false,\"sqlTimeZone\":\"America/Los_Angeles\"}}], signature=[{a0:LONG}])\n", - DUMMY_SQL_QUERY_ID - ), + explanation, "RESOURCES", "[{\"name\":\"foo\",\"type\":\"DATASOURCE\"}]" ) diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java index 7498bc6daf1b..5c86fb2b11b1 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java @@ -1014,12 +1014,29 @@ public void testExplainSelectConstantExpression() throws Exception { // Skip vectorization since otherwise the "context" will change for each subtest. skipVectorize(); +// String explanation = "DruidQueryRel(query=[{\"queryType\":\"scan\",\"dataSource\":{\"type\":\"inline\",\"columnNames\":[\"EXPR$0\"],\"columnTypes\":[\"LONG\"],\"rows\":[[2]]},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"virtualColumns\":[],\"resultFormat\":\"compactedList\",\"batchSize\":20480,\"order\":\"none\",\"filter\":null,\"columns\":[\"EXPR$0\"],\"legacy\":false,\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\",\"sqlQueryId\":\"dummy\",\"vectorize\":\"false\",\"vectorizeVirtualColumns\":\"false\"},\"descending\":false,\"granularity\":{\"type\":\"all\"}}], signature=[{EXPR$0:LONG}])\n"; + String explanation = "[{" + + "\"query\":{\"queryType\":\"scan\"," + + "\"dataSource\":{\"type\":\"inline\",\"columnNames\":[\"EXPR$0\"],\"columnTypes\":[\"LONG\"],\"rows\":[[2]]}," + + "\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]}," + + "\"virtualColumns\":[]," + + "\"resultFormat\":\"compactedList\"," + + "\"batchSize\":20480," + + "\"order\":\"none\"," + + "\"filter\":null," + + "\"columns\":[\"EXPR$0\"]," + + "\"legacy\":false," + + "\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\",\"sqlQueryId\":\"dummy\",\"vectorize\":\"false\",\"vectorizeVirtualColumns\":\"false\"}," + + "\"descending\":false," + + "\"granularity\":{\"type\":\"all\"}}," + + "\"signature\":\"{EXPR$0:LONG}\"" + + "}]"; testQuery( "EXPLAIN PLAN FOR SELECT 1 + 1", ImmutableList.of(), ImmutableList.of( - new Object[]{"DruidQueryRel(query=[{\"queryType\":\"scan\",\"dataSource\":{\"type\":\"inline\",\"columnNames\":[\"EXPR$0\"],\"columnTypes\":[\"LONG\"],\"rows\":[[2]]},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"virtualColumns\":[],\"resultFormat\":\"compactedList\",\"batchSize\":20480,\"order\":\"none\",\"filter\":null,\"columns\":[\"EXPR$0\"],\"legacy\":false,\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\",\"sqlQueryId\":\"dummy\",\"vectorize\":\"false\",\"vectorizeVirtualColumns\":\"false\"},\"descending\":false,\"granularity\":{\"type\":\"all\"}}], signature=[{EXPR$0:LONG}])\n", "[]"} + new Object[]{explanation, "[]"} ) ); } @@ -1427,13 +1444,29 @@ public void testExplainSelectStar() throws Exception { // Skip vectorization since otherwise the "context" will change for each subtest. skipVectorize(); +// String explanation = "DruidQueryRel(query=[{\"queryType\":\"scan\",\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"virtualColumns\":[],\"resultFormat\":\"compactedList\",\"batchSize\":20480,\"order\":\"none\",\"filter\":null,\"columns\":[\"__time\",\"cnt\",\"dim1\",\"dim2\",\"dim3\",\"m1\",\"m2\",\"unique_dim1\"],\"legacy\":false,\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\",\"sqlQueryId\":\"dummy\",\"vectorize\":\"false\",\"vectorizeVirtualColumns\":\"false\"},\"descending\":false,\"granularity\":{\"type\":\"all\"}}], signature=[{__time:LONG, cnt:LONG, dim1:STRING, dim2:STRING, dim3:STRING, m1:FLOAT, m2:DOUBLE, unique_dim1:COMPLEX}])\n"; + String explanation = "[{" + + "\"query\":{\"queryType\":\"scan\"," + + "\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"}," + + "\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]}," + + "\"virtualColumns\":[]," + + "\"resultFormat\":\"compactedList\"," + + "\"batchSize\":20480," + + "\"order\":\"none\"," + + "\"filter\":null," + + "\"columns\":[\"__time\",\"cnt\",\"dim1\",\"dim2\",\"dim3\",\"m1\",\"m2\",\"unique_dim1\"]," + + "\"legacy\":false," + + "\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\",\"sqlQueryId\":\"dummy\",\"vectorize\":\"false\",\"vectorizeVirtualColumns\":\"false\"}," + + "\"descending\":false," + + "\"granularity\":{\"type\":\"all\"}}," + + "\"signature\":\"{__time:LONG, cnt:LONG, dim1:STRING, dim2:STRING, dim3:STRING, m1:FLOAT, m2:DOUBLE, unique_dim1:COMPLEX}\"}]"; testQuery( "EXPLAIN PLAN FOR SELECT * FROM druid.foo", ImmutableList.of(), ImmutableList.of( new Object[]{ - "DruidQueryRel(query=[{\"queryType\":\"scan\",\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"virtualColumns\":[],\"resultFormat\":\"compactedList\",\"batchSize\":20480,\"order\":\"none\",\"filter\":null,\"columns\":[\"__time\",\"cnt\",\"dim1\",\"dim2\",\"dim3\",\"m1\",\"m2\",\"unique_dim1\"],\"legacy\":false,\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\",\"sqlQueryId\":\"dummy\",\"vectorize\":\"false\",\"vectorizeVirtualColumns\":\"false\"},\"descending\":false,\"granularity\":{\"type\":\"all\"}}], signature=[{__time:LONG, cnt:LONG, dim1:STRING, dim2:STRING, dim3:STRING, m1:FLOAT, m2:DOUBLE, unique_dim1:COMPLEX}])\n", + explanation, "[{\"name\":\"foo\",\"type\":\"DATASOURCE\"}]" } ) @@ -5931,7 +5964,7 @@ public void testExplainCountStarOnView() throws Exception // Skip vectorization since otherwise the "context" will change for each subtest. skipVectorize(); - final String explanation = + /*final String explanation = "DruidQueryRel(query=[{" + "\"queryType\":\"timeseries\"," + "\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"}," @@ -5944,8 +5977,20 @@ public void testExplainCountStarOnView() throws Exception + "\"postAggregations\":[]," + "\"limit\":2147483647," + "\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\",\"sqlQueryId\":\"dummy\",\"vectorize\":\"false\",\"vectorizeVirtualColumns\":\"false\"}}]" - + ", signature=[{a0:LONG}])\n"; - + + ", signature=[{a0:LONG}])\n";*/ + final String explanation = "[{" + + "\"query\":{\"queryType\":\"timeseries\"," + + "\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"}," + + "\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]}," + + "\"descending\":false," + + "\"virtualColumns\":[]," + + "\"filter\":{\"type\":\"and\",\"fields\":[{\"type\":\"selector\",\"dimension\":\"dim2\",\"value\":\"a\",\"extractionFn\":null},{\"type\":\"not\",\"field\":{\"type\":\"selector\",\"dimension\":\"dim1\",\"value\":\"z\",\"extractionFn\":{\"type\":\"substring\",\"index\":0,\"length\":1}}}]}," + + "\"granularity\":{\"type\":\"all\"}," + + "\"aggregations\":[{\"type\":\"count\",\"name\":\"a0\"}]," + + "\"postAggregations\":[]," + + "\"limit\":2147483647," + + "\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\",\"sqlQueryId\":\"dummy\",\"vectorize\":\"false\",\"vectorizeVirtualColumns\":\"false\"}}," + + "\"signature\":\"{a0:LONG}\"}]"; final String resources = "[{\"name\":\"aview\",\"type\":\"VIEW\"}]"; testQuery( @@ -8956,24 +9001,42 @@ public void testMaxSubqueryRows() throws Exception } @Test - public void testExplainUnionAllOfQueries() throws Exception - { - skipVectorize(); // Required otherwise results of the EXPLAIN change - String query = "EXPLAIN PLAN FOR (SELECT dim1 FROM druid.foo UNION ALL SELECT dim1 from druid.foo2)"; - String resources = "[{\"name\":\"foo2\",\"type\":\"DATASOURCE\"},{\"name\":\"foo\",\"type\":\"DATASOURCE\"}]"; - String explanation = "[{\"query\":{\"dataSource\":{\"type\":\"union\",\"dataSources\":[{\"type\":\"table\",\"name\":\"foo\"},{\"type\":\"table\",\"name\":\"foo2\"}]},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"virtualColumns\":[],\"resultFormat\":\"compactedList\",\"batchSize\":20480,\"order\":\"none\",\"filter\":null,\"columns\":[\"dim1\"],\"legacy\":false,\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\",\"sqlQueryId\":\"dummy\",\"vectorize\":\"false\",\"vectorizeVirtualColumns\":\"false\"},\"descending\":false,\"granularity\":{\"type\":\"all\"}},\"signature\":\"{dim1:STRING}\"}]"; - testQuery(query, ImmutableList.of(), ImmutableList.of(new Object[]{explanation, resources})); - } - - @Test - public void testExplainUnionOfMultipleNativeQueries() throws Exception + public void testExplainUnionAllOfMultipleNativeQueries() throws Exception { skipVectorize(); String query = "EXPLAIN PLAN FOR ((SELECT dim1 FROM druid.foo) UNION ALL (SELECT dim1 FROM druid.foo4 where dim1 in (SELECT dim1 FROM druid.foo)))"; - String explanation = "[" - + "{\"query\":{\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"virtualColumns\":[],\"resultFormat\":\"compactedList\",\"batchSize\":20480,\"order\":\"none\",\"filter\":null,\"columns\":[\"dim1\"],\"legacy\":false,\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\",\"sqlQueryId\":\"dummy\",\"vectorize\":\"false\",\"vectorizeVirtualColumns\":\"false\"},\"descending\":false,\"granularity\":{\"type\":\"all\"}},\"signature\":\"{dim1:STRING}\"}" - + ",{\"query\":{\"dataSource\":{\"type\":\"join\",\"left\":{\"type\":\"table\",\"name\":\"foo4\"},\"right\":{\"type\":\"query\",\"query\":{\"queryType\":\"groupBy\",\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"virtualColumns\":[],\"filter\":null,\"granularity\":{\"type\":\"all\"},\"dimensions\":[{\"type\":\"default\",\"dimension\":\"dim1\",\"outputName\":\"d0\",\"outputType\":\"STRING\"}],\"aggregations\":[],\"postAggregations\":[],\"having\":null,\"limitSpec\":{\"type\":\"NoopLimitSpec\"},\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\",\"sqlQueryId\":\"dummy\",\"vectorize\":\"false\",\"vectorizeVirtualColumns\":\"false\"},\"descending\":false}},\"rightPrefix\":\"j0.\",\"condition\":\"(\\\"dim1\\\" == \\\"j0.d0\\\")\",\"joinType\":\"INNER\",\"leftFilter\":null},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"virtualColumns\":[],\"resultFormat\":\"compactedList\",\"batchSize\":20480,\"order\":\"none\",\"filter\":null,\"columns\":[\"dim1\"],\"legacy\":false,\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\",\"sqlQueryId\":\"dummy\",\"vectorize\":\"false\",\"vectorizeVirtualColumns\":\"false\"},\"descending\":false,\"granularity\":{\"type\":\"all\"}},\"signature\":\"{dim1:STRING}\"}]"; String resources = "[{\"name\":\"foo4\",\"type\":\"DATASOURCE\"},{\"name\":\"foo\",\"type\":\"DATASOURCE\"}]"; + String explanation = "[" + + "{\"query\":{\"queryType\":\"scan\"," + + "\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"}," + + "\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]}," + + "\"virtualColumns\":[]," + + "\"resultFormat\":\"compactedList\"," + + "\"batchSize\":20480," + + "\"order\":\"none\"," + + "\"filter\":null," + + "\"columns\":[\"dim1\"]," + + "\"legacy\":false," + + "\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\",\"sqlQueryId\":\"dummy\",\"vectorize\":\"false\",\"vectorizeVirtualColumns\":\"false\"}," + + "\"descending\":false," + + "\"granularity\":{\"type\":\"all\"}}," + + "\"signature\":\"{dim1:STRING}\"}," + + "{\"query\":{\"queryType\":\"scan\"," + + "\"dataSource\":{\"type\":\"join\",\"left\":{\"type\":\"table\",\"name\":\"foo4\"},\"right\":{\"type\":\"query\",\"query\":{\"queryType\":\"groupBy\",\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"virtualColumns\":[],\"filter\":null,\"granularity\":{\"type\":\"all\"},\"dimensions\":[{\"type\":\"default\",\"dimension\":\"dim1\",\"outputName\":\"d0\",\"outputType\":\"STRING\"}],\"aggregations\":[],\"postAggregations\":[],\"having\":null,\"limitSpec\":{\"type\":\"NoopLimitSpec\"},\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\",\"sqlQueryId\":\"dummy\",\"vectorize\":\"false\",\"vectorizeVirtualColumns\":\"false\"},\"descending\":false}},\"rightPrefix\":\"j0.\",\"condition\":\"(\\\"dim1\\\" == \\\"j0.d0\\\")\",\"joinType\":\"INNER\",\"leftFilter\":null}," + + "\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]}," + + "\"virtualColumns\":[]," + + "\"resultFormat\":\"compactedList\"," + + "\"batchSize\":20480," + + "\"order\":\"none\"," + + "\"filter\":null," + + "\"columns\":[\"dim1\"]," + + "\"legacy\":false," + + "\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\",\"sqlQueryId\":\"dummy\",\"vectorize\":\"false\",\"vectorizeVirtualColumns\":\"false\"}" + + ",\"descending\":false," + + "\"granularity\":{\"type\":\"all\"}}," + + "\"signature\":\"{dim1:STRING}\"}" + + "]"; + testQuery(query, ImmutableList.of(), ImmutableList.of(new Object[]{explanation, resources})); } @@ -8983,12 +9046,27 @@ public void testExplainExactCountDistinctOfSemiJoinResult() throws Exception // Skip vectorization since otherwise the "context" will change for each subtest. skipVectorize(); - final String explanation = + /*final String explanation = "DruidOuterQueryRel(query=[{\"queryType\":\"timeseries\",\"dataSource\":{\"type\":\"table\",\"name\":\"__subquery__\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"descending\":false,\"virtualColumns\":[],\"filter\":null,\"granularity\":{\"type\":\"all\"},\"aggregations\":[{\"type\":\"count\",\"name\":\"a0\"}],\"postAggregations\":[],\"limit\":2147483647,\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\",\"sqlQueryId\":\"dummy\",\"vectorize\":\"false\",\"vectorizeVirtualColumns\":\"false\"}}], signature=[{a0:LONG}])\n" + " DruidJoinQueryRel(condition=[=(SUBSTRING($3, 1, 1), $8)], joinType=[inner], query=[{\"queryType\":\"groupBy\",\"dataSource\":{\"type\":\"table\",\"name\":\"__join__\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"virtualColumns\":[],\"filter\":null,\"granularity\":{\"type\":\"all\"},\"dimensions\":[{\"type\":\"default\",\"dimension\":\"dim2\",\"outputName\":\"d0\",\"outputType\":\"STRING\"}],\"aggregations\":[],\"postAggregations\":[],\"having\":null,\"limitSpec\":{\"type\":\"NoopLimitSpec\"},\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\",\"sqlQueryId\":\"dummy\",\"vectorize\":\"false\",\"vectorizeVirtualColumns\":\"false\"},\"descending\":false}], signature=[{d0:STRING}])\n" + " DruidQueryRel(query=[{\"queryType\":\"scan\",\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"virtualColumns\":[],\"resultFormat\":\"compactedList\",\"batchSize\":20480,\"order\":\"none\",\"filter\":null,\"columns\":[\"__time\",\"cnt\",\"dim1\",\"dim2\",\"dim3\",\"m1\",\"m2\",\"unique_dim1\"],\"legacy\":false,\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\",\"sqlQueryId\":\"dummy\",\"vectorize\":\"false\",\"vectorizeVirtualColumns\":\"false\"},\"descending\":false,\"granularity\":{\"type\":\"all\"}}], signature=[{__time:LONG, cnt:LONG, dim1:STRING, dim2:STRING, dim3:STRING, m1:FLOAT, m2:DOUBLE, unique_dim1:COMPLEX}])\n" + " DruidQueryRel(query=[{\"queryType\":\"groupBy\",\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"virtualColumns\":[],\"filter\":{\"type\":\"not\",\"field\":{\"type\":\"selector\",\"dimension\":\"dim1\",\"value\":null,\"extractionFn\":null}},\"granularity\":{\"type\":\"all\"},\"dimensions\":[{\"type\":\"extraction\",\"dimension\":\"dim1\",\"outputName\":\"d0\",\"outputType\":\"STRING\",\"extractionFn\":{\"type\":\"substring\",\"index\":0,\"length\":1}}],\"aggregations\":[],\"postAggregations\":[],\"having\":null,\"limitSpec\":{\"type\":\"NoopLimitSpec\"},\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\",\"sqlQueryId\":\"dummy\",\"vectorize\":\"false\",\"vectorizeVirtualColumns\":\"false\"},\"descending\":false}], signature=[{d0:STRING}])\n"; - +*/ + final String explanation = "[" + + "{\"query\":{\"queryType\":\"groupBy\"," + + "\"dataSource\":{\"type\":\"query\",\"query\":{\"queryType\":\"groupBy\",\"dataSource\":{\"type\":\"join\",\"left\":{\"type\":\"table\",\"name\":\"foo\"},\"right\":{\"type\":\"query\",\"query\":{\"queryType\":\"groupBy\",\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"virtualColumns\":[],\"filter\":{\"type\":\"not\",\"field\":{\"type\":\"selector\",\"dimension\":\"dim1\",\"value\":null,\"extractionFn\":null}},\"granularity\":{\"type\":\"all\"},\"dimensions\":[{\"type\":\"extraction\",\"dimension\":\"dim1\",\"outputName\":\"d0\",\"outputType\":\"STRING\",\"extractionFn\":{\"type\":\"substring\",\"index\":0,\"length\":1}}],\"aggregations\":[],\"postAggregations\":[],\"having\":null,\"limitSpec\":{\"type\":\"NoopLimitSpec\"},\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\",\"sqlQueryId\":\"dummy\",\"vectorize\":\"false\",\"vectorizeVirtualColumns\":\"false\"},\"descending\":false}},\"rightPrefix\":\"j0.\",\"condition\":\"(substring(\\\"dim2\\\", 0, 1) == \\\"j0.d0\\\")\",\"joinType\":\"INNER\",\"leftFilter\":null},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"virtualColumns\":[],\"filter\":null,\"granularity\":{\"type\":\"all\"},\"dimensions\":[{\"type\":\"default\",\"dimension\":\"dim2\",\"outputName\":\"d0\",\"outputType\":\"STRING\"}],\"aggregations\":[],\"postAggregations\":[],\"having\":null,\"limitSpec\":{\"type\":\"NoopLimitSpec\"},\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\",\"sqlQueryId\":\"dummy\",\"vectorize\":\"false\",\"vectorizeVirtualColumns\":\"false\"},\"descending\":false}}," + + "\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]}," + + "\"virtualColumns\":[]," + + "\"filter\":null," + + "\"granularity\":{\"type\":\"all\"}," + + "\"dimensions\":[]," + + "\"aggregations\":[{\"type\":\"count\",\"name\":\"a0\"}]," + + "\"postAggregations\":[]," + + "\"having\":null," + + "\"limitSpec\":{\"type\":\"NoopLimitSpec\"}," + + "\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\",\"sqlQueryId\":\"dummy\",\"vectorize\":\"false\",\"vectorizeVirtualColumns\":\"false\"}," + + "\"descending\":false}," + + "\"signature\":\"{a0:LONG}\"}]"; final String resources = "[{\"name\":\"foo\",\"type\":\"DATASOURCE\"}]"; testQuery( diff --git a/sql/src/test/java/org/apache/druid/sql/http/SqlResourceTest.java b/sql/src/test/java/org/apache/druid/sql/http/SqlResourceTest.java index 6da9ccbfec54..80494466fc9c 100644 --- a/sql/src/test/java/org/apache/druid/sql/http/SqlResourceTest.java +++ b/sql/src/test/java/org/apache/druid/sql/http/SqlResourceTest.java @@ -870,17 +870,26 @@ public void testExplainCountStar() throws Exception { Map queryContext = ImmutableMap.of(PlannerContext.CTX_SQL_QUERY_ID, DUMMY_SQL_QUERY_ID); final List> rows = doPost( - new SqlQuery("EXPLAIN PLAN FOR SELECT COUNT(*) AS cnt FROM druid.foo", ResultFormat.OBJECT, false, queryContext, null) + new SqlQuery( + "EXPLAIN PLAN FOR SELECT COUNT(*) AS cnt FROM druid.foo", + ResultFormat.OBJECT, + false, + queryContext, + null + ) ).rhs; + /*final String explanation = StringUtils.format( + "DruidQueryRel(query=[{\"queryType\":\"timeseries\",\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"descending\":false,\"virtualColumns\":[],\"filter\":null,\"granularity\":{\"type\":\"all\"},\"aggregations\":[{\"type\":\"count\",\"name\":\"a0\"}],\"postAggregations\":[],\"limit\":2147483647,\"context\":{\"sqlQueryId\":\"%s\"}}], signature=[{a0:LONG}])\n", + DUMMY_SQL_QUERY_ID + ); + */ + final String explanation = "[{\"query\":{\"queryType\":\"timeseries\",\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"descending\":false,\"virtualColumns\":[],\"filter\":null,\"granularity\":{\"type\":\"all\"},\"aggregations\":[{\"type\":\"count\",\"name\":\"a0\"}],\"postAggregations\":[],\"limit\":2147483647,\"context\":{\"sqlQueryId\":\"dummy\"}},\"signature\":\"{a0:LONG}\"}]"; Assert.assertEquals( ImmutableList.of( ImmutableMap.of( "PLAN", - StringUtils.format( - "DruidQueryRel(query=[{\"queryType\":\"timeseries\",\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"descending\":false,\"virtualColumns\":[],\"filter\":null,\"granularity\":{\"type\":\"all\"},\"aggregations\":[{\"type\":\"count\",\"name\":\"a0\"}],\"postAggregations\":[],\"limit\":2147483647,\"context\":{\"sqlQueryId\":\"%s\"}}], signature=[{a0:LONG}])\n", - DUMMY_SQL_QUERY_ID - ), + explanation, "RESOURCES", "[{\"name\":\"foo\",\"type\":\"DATASOURCE\"}]" ) From 9c8d85b4fc784d407b8f5dd93e36edcfcd5ccd18 Mon Sep 17 00:00:00 2001 From: Laksh Singla Date: Mon, 15 Nov 2021 14:11:40 +0530 Subject: [PATCH 04/18] Fix checkstyle, remove unused import --- .../java/org/apache/druid/sql/calcite/CalciteQueryTest.java | 4 ++-- .../test/java/org/apache/druid/sql/http/SqlResourceTest.java | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java index 5c86fb2b11b1..b430f7f6bbc0 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java @@ -1014,7 +1014,7 @@ public void testExplainSelectConstantExpression() throws Exception { // Skip vectorization since otherwise the "context" will change for each subtest. skipVectorize(); -// String explanation = "DruidQueryRel(query=[{\"queryType\":\"scan\",\"dataSource\":{\"type\":\"inline\",\"columnNames\":[\"EXPR$0\"],\"columnTypes\":[\"LONG\"],\"rows\":[[2]]},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"virtualColumns\":[],\"resultFormat\":\"compactedList\",\"batchSize\":20480,\"order\":\"none\",\"filter\":null,\"columns\":[\"EXPR$0\"],\"legacy\":false,\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\",\"sqlQueryId\":\"dummy\",\"vectorize\":\"false\",\"vectorizeVirtualColumns\":\"false\"},\"descending\":false,\"granularity\":{\"type\":\"all\"}}], signature=[{EXPR$0:LONG}])\n"; + // String explanation = "DruidQueryRel(query=[{\"queryType\":\"scan\",\"dataSource\":{\"type\":\"inline\",\"columnNames\":[\"EXPR$0\"],\"columnTypes\":[\"LONG\"],\"rows\":[[2]]},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"virtualColumns\":[],\"resultFormat\":\"compactedList\",\"batchSize\":20480,\"order\":\"none\",\"filter\":null,\"columns\":[\"EXPR$0\"],\"legacy\":false,\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\",\"sqlQueryId\":\"dummy\",\"vectorize\":\"false\",\"vectorizeVirtualColumns\":\"false\"},\"descending\":false,\"granularity\":{\"type\":\"all\"}}], signature=[{EXPR$0:LONG}])\n"; String explanation = "[{" + "\"query\":{\"queryType\":\"scan\"," + "\"dataSource\":{\"type\":\"inline\",\"columnNames\":[\"EXPR$0\"],\"columnTypes\":[\"LONG\"],\"rows\":[[2]]}," @@ -1444,7 +1444,7 @@ public void testExplainSelectStar() throws Exception { // Skip vectorization since otherwise the "context" will change for each subtest. skipVectorize(); -// String explanation = "DruidQueryRel(query=[{\"queryType\":\"scan\",\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"virtualColumns\":[],\"resultFormat\":\"compactedList\",\"batchSize\":20480,\"order\":\"none\",\"filter\":null,\"columns\":[\"__time\",\"cnt\",\"dim1\",\"dim2\",\"dim3\",\"m1\",\"m2\",\"unique_dim1\"],\"legacy\":false,\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\",\"sqlQueryId\":\"dummy\",\"vectorize\":\"false\",\"vectorizeVirtualColumns\":\"false\"},\"descending\":false,\"granularity\":{\"type\":\"all\"}}], signature=[{__time:LONG, cnt:LONG, dim1:STRING, dim2:STRING, dim3:STRING, m1:FLOAT, m2:DOUBLE, unique_dim1:COMPLEX}])\n"; + // String explanation = "DruidQueryRel(query=[{\"queryType\":\"scan\",\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"virtualColumns\":[],\"resultFormat\":\"compactedList\",\"batchSize\":20480,\"order\":\"none\",\"filter\":null,\"columns\":[\"__time\",\"cnt\",\"dim1\",\"dim2\",\"dim3\",\"m1\",\"m2\",\"unique_dim1\"],\"legacy\":false,\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\",\"sqlQueryId\":\"dummy\",\"vectorize\":\"false\",\"vectorizeVirtualColumns\":\"false\"},\"descending\":false,\"granularity\":{\"type\":\"all\"}}], signature=[{__time:LONG, cnt:LONG, dim1:STRING, dim2:STRING, dim3:STRING, m1:FLOAT, m2:DOUBLE, unique_dim1:COMPLEX}])\n"; String explanation = "[{" + "\"query\":{\"queryType\":\"scan\"," + "\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"}," diff --git a/sql/src/test/java/org/apache/druid/sql/http/SqlResourceTest.java b/sql/src/test/java/org/apache/druid/sql/http/SqlResourceTest.java index 80494466fc9c..f1e0ce5fafcf 100644 --- a/sql/src/test/java/org/apache/druid/sql/http/SqlResourceTest.java +++ b/sql/src/test/java/org/apache/druid/sql/http/SqlResourceTest.java @@ -38,7 +38,6 @@ import org.apache.druid.java.util.common.ISE; import org.apache.druid.java.util.common.NonnullPair; import org.apache.druid.java.util.common.Pair; -import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.java.util.common.concurrent.Execs; import org.apache.druid.java.util.common.guava.LazySequence; import org.apache.druid.java.util.common.guava.Sequence; From 48af23f50ef4c3dd3b497f31d4710da3d7f7d0e6 Mon Sep 17 00:00:00 2001 From: Laksh Singla Date: Thu, 18 Nov 2021 16:03:10 +0530 Subject: [PATCH 05/18] Add flag to control the output being generated in SQL explain --- .../sql/calcite/planner/DruidPlanner.java | 7 +++---- .../sql/calcite/planner/PlannerConfig.java | 21 +++++++++++++++++-- 2 files changed, 22 insertions(+), 6 deletions(-) diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/planner/DruidPlanner.java b/sql/src/main/java/org/apache/druid/sql/calcite/planner/DruidPlanner.java index de14d45e168c..f7a13650fc0d 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/planner/DruidPlanner.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/planner/DruidPlanner.java @@ -381,11 +381,10 @@ private PlannerResult planExplanation( String resources; try { if (isDruidConventionExplanation && rel instanceof DruidRel) { - DruidRel druidRel = (DruidRel) rel; - if (true) { + // Show the native queries instead of Calcite's explain if the legacy flag is turned off + if (!plannerContext.getPlannerConfig().isUseLegacyDruidExplain()) { + DruidRel druidRel = (DruidRel) rel; explanation = explainSqlPlanAsNativeQueries(druidRel); - } else { - explanation = RelOptUtil.dumpPlan("", rel, explain.getFormat(), explain.getDetailLevel()); } } resources = jsonMapper.writeValueAsString(plannerContext.getResources()); diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/planner/PlannerConfig.java b/sql/src/main/java/org/apache/druid/sql/calcite/planner/PlannerConfig.java index 43dfa5a36c25..9c149ca54f52 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/planner/PlannerConfig.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/planner/PlannerConfig.java @@ -33,6 +33,7 @@ public class PlannerConfig public static final String CTX_KEY_USE_GROUPING_SET_FOR_EXACT_DISTINCT = "useGroupingSetForExactDistinct"; public static final String CTX_KEY_USE_APPROXIMATE_TOPN = "useApproximateTopN"; public static final String CTX_COMPUTE_INNER_JOIN_COST_AS_FILTER = "computeInnerJoinCostAsFilter"; + public static final String CTX_KEY_USE_LEGACY_DRUID_EXPLAIN = "useLegacyDruidExplain"; @JsonProperty private Period metadataRefreshPeriod = new Period("PT1M"); @@ -70,6 +71,9 @@ public class PlannerConfig @JsonProperty private boolean authorizeSystemTablesDirectly = false; + @JsonProperty + private boolean useLegacyDruidExplain = false; + public long getMetadataSegmentPollPeriod() { return metadataSegmentPollPeriod; @@ -137,6 +141,11 @@ public boolean isAuthorizeSystemTablesDirectly() return authorizeSystemTablesDirectly; } + public boolean isUseLegacyDruidExplain() + { + return useLegacyDruidExplain; + } + public PlannerConfig withOverrides(final Map context) { if (context == null) { @@ -166,6 +175,11 @@ public PlannerConfig withOverrides(final Map context) CTX_COMPUTE_INNER_JOIN_COST_AS_FILTER, computeInnerJoinCostAsFilter ); + newConfig.useLegacyDruidExplain = getContextBoolean( + context, + CTX_KEY_USE_LEGACY_DRUID_EXPLAIN, + useLegacyDruidExplain + ); newConfig.requireTimeCondition = isRequireTimeCondition(); newConfig.sqlTimeZone = getSqlTimeZone(); newConfig.awaitInitializationOnStart = isAwaitInitializationOnStart(); @@ -213,7 +227,8 @@ public boolean equals(final Object o) metadataSegmentPollPeriod == that.metadataSegmentPollPeriod && serializeComplexValues == that.serializeComplexValues && Objects.equals(metadataRefreshPeriod, that.metadataRefreshPeriod) && - Objects.equals(sqlTimeZone, that.sqlTimeZone); + Objects.equals(sqlTimeZone, that.sqlTimeZone) && + useLegacyDruidExplain == that.useLegacyDruidExplain; } @Override @@ -230,7 +245,8 @@ public int hashCode() sqlTimeZone, metadataSegmentCacheEnable, metadataSegmentPollPeriod, - serializeComplexValues + serializeComplexValues, + useLegacyDruidExplain ); } @@ -248,6 +264,7 @@ public String toString() ", metadataSegmentPollPeriod=" + metadataSegmentPollPeriod + ", sqlTimeZone=" + sqlTimeZone + ", serializeComplexValues=" + serializeComplexValues + + ", useLegacyDruidExplain=" + useLegacyDruidExplain + '}'; } } From 890dd4a1e587e0c5f325e830505c0afde0f81e90 Mon Sep 17 00:00:00 2001 From: Laksh Singla Date: Thu, 18 Nov 2021 17:57:41 +0530 Subject: [PATCH 06/18] Add tests for the overridding of the context --- .../sql/calcite/planner/PlannerConfig.java | 2 +- .../druid/sql/calcite/CalciteQueryTest.java | 93 ++++++++++++++++++- 2 files changed, 93 insertions(+), 2 deletions(-) diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/planner/PlannerConfig.java b/sql/src/main/java/org/apache/druid/sql/calcite/planner/PlannerConfig.java index 9c149ca54f52..047ca9aaf639 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/planner/PlannerConfig.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/planner/PlannerConfig.java @@ -178,7 +178,7 @@ public PlannerConfig withOverrides(final Map context) newConfig.useLegacyDruidExplain = getContextBoolean( context, CTX_KEY_USE_LEGACY_DRUID_EXPLAIN, - useLegacyDruidExplain + isUseLegacyDruidExplain() ); newConfig.requireTimeCondition = isRequireTimeCondition(); newConfig.sqlTimeZone = getSqlTimeZone(); diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java index cf314a2d9445..d50e9059d5d1 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java @@ -1088,8 +1088,11 @@ public void testExplainSelectConstantExpression() throws Exception + "\"signature\":\"{EXPR$0:LONG}\"" + "}]"; - testQuery( + Map context = new HashMap<>(QUERY_CONTEXT_DEFAULT); + context.put(PlannerConfig.CTX_KEY_USE_LEGACY_DRUID_EXPLAIN, true); + testQuery( "EXPLAIN PLAN FOR SELECT 1 + 1", + context, ImmutableList.of(), ImmutableList.of( new Object[]{explanation, "[]"} @@ -9188,6 +9191,94 @@ public void testExplainExactCountDistinctOfSemiJoinResult() throws Exception ); } + @Test + public void testExplainSelectStarWithOverrides() throws Exception + { + PlannerConfig useLegacyDruidExplainConfig = new PlannerConfig() + { + @Override + public boolean isUseLegacyDruidExplain() + { + return true; + } + }; + + Map useLegacyDruidExplainContext = new HashMap<>(QUERY_CONTEXT_DEFAULT); + useLegacyDruidExplainContext.put(PlannerConfig.CTX_KEY_USE_LEGACY_DRUID_EXPLAIN, true); + + Map useRegularDruidExplainContext = new HashMap<>(QUERY_CONTEXT_DEFAULT); + useRegularDruidExplainContext.put(PlannerConfig.CTX_KEY_USE_LEGACY_DRUID_EXPLAIN, false); + + + // Skip vectorization since otherwise the "context" will change for each subtest. + skipVectorize(); + String legacyExplanation = "DruidQueryRel(query=[{\"queryType\":\"scan\",\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"virtualColumns\":[],\"resultFormat\":\"compactedList\",\"batchSize\":20480,\"order\":\"none\",\"filter\":null,\"columns\":[\"__time\",\"cnt\",\"dim1\",\"dim2\",\"dim3\",\"m1\",\"m2\",\"unique_dim1\"],\"legacy\":false,\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\",\"sqlQueryId\":\"dummy\",\"vectorize\":\"false\",\"vectorizeVirtualColumns\":\"false\"},\"descending\":false,\"granularity\":{\"type\":\"all\"}}], signature=[{__time:LONG, cnt:LONG, dim1:STRING, dim2:STRING, dim3:STRING, m1:FLOAT, m2:DOUBLE, unique_dim1:COMPLEX}])\n"; + String legacyExplanationWithContext = "DruidQueryRel(query=[{\"queryType\":\"scan\",\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"virtualColumns\":[],\"resultFormat\":\"compactedList\",\"batchSize\":20480,\"order\":\"none\",\"filter\":null,\"columns\":[\"__time\",\"cnt\",\"dim1\",\"dim2\",\"dim3\",\"m1\",\"m2\",\"unique_dim1\"],\"legacy\":false,\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\",\"sqlQueryId\":\"dummy\",\"useLegacyDruidExplain\":true,\"vectorize\":\"false\",\"vectorizeVirtualColumns\":\"false\"},\"descending\":false,\"granularity\":{\"type\":\"all\"}}], signature=[{__time:LONG, cnt:LONG, dim1:STRING, dim2:STRING, dim3:STRING, m1:FLOAT, m2:DOUBLE, unique_dim1:COMPLEX}])\n"; + String explanation = "[{" + + "\"query\":{\"queryType\":\"scan\"," + + "\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"}," + + "\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]}," + + "\"virtualColumns\":[]," + + "\"resultFormat\":\"compactedList\"," + + "\"batchSize\":20480," + + "\"order\":\"none\"," + + "\"filter\":null," + + "\"columns\":[\"__time\",\"cnt\",\"dim1\",\"dim2\",\"dim3\",\"m1\",\"m2\",\"unique_dim1\"]," + + "\"legacy\":false," + + "\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\",\"sqlQueryId\":\"dummy\",\"vectorize\":\"false\",\"vectorizeVirtualColumns\":\"false\"}," + + "\"descending\":false," + + "\"granularity\":{\"type\":\"all\"}}," + + "\"signature\":\"{__time:LONG, cnt:LONG, dim1:STRING, dim2:STRING, dim3:STRING, m1:FLOAT, m2:DOUBLE, unique_dim1:COMPLEX}\"}]"; + + String explanationWithContext = "[{" + + "\"query\":{\"queryType\":\"scan\"," + + "\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"}," + + "\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]}," + + "\"virtualColumns\":[]," + + "\"resultFormat\":\"compactedList\"," + + "\"batchSize\":20480," + + "\"order\":\"none\"," + + "\"filter\":null," + + "\"columns\":[\"__time\",\"cnt\",\"dim1\",\"dim2\",\"dim3\",\"m1\",\"m2\",\"unique_dim1\"]," + + "\"legacy\":false," + + "\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\",\"sqlQueryId\":\"dummy\",\"useLegacyDruidExplain\":false}," + + "\"descending\":false," + + "\"granularity\":{\"type\":\"all\"}}," + + "\"signature\":\"{__time:LONG, cnt:LONG, dim1:STRING, dim2:STRING, dim3:STRING, m1:FLOAT, m2:DOUBLE, unique_dim1:COMPLEX}\"}]"; + String sql = "EXPLAIN PLAN FOR SELECT * FROM druid.foo"; + String resources = "[{\"name\":\"foo\",\"type\":\"DATASOURCE\"}]"; + + // Test when default config and no overrides + testQuery(sql, ImmutableList.of(), ImmutableList.of(new Object[]{explanation, resources})); + + // Test when default config and useLegacyDruidExplain is overridden in the context + testQuery( + sql, + useLegacyDruidExplainContext, + ImmutableList.of(), + ImmutableList.of(new Object[]{legacyExplanationWithContext, resources}) + ); + + // Test when useLegacyDruidExplain disabled by default and no overrides + testQuery( + useLegacyDruidExplainConfig, + sql, + CalciteTests.REGULAR_USER_AUTH_RESULT, + ImmutableList.of(), + ImmutableList.of(new Object[]{legacyExplanation, resources}) + ); + + // Test when useLegacyDruidExplain disabled by default but is overriden in the context + testQuery( + useLegacyDruidExplainConfig, + useRegularDruidExplainContext, + sql, + CalciteTests.REGULAR_USER_AUTH_RESULT, + ImmutableList.of(), + ImmutableList.of(new Object[]{explanationWithContext, resources}) + ); + } + @Test public void testExactCountDistinctUsingSubqueryWithWherePushDown() throws Exception { From 5023eb18247792a1692088fc7e731fae38d7b9be Mon Sep 17 00:00:00 2001 From: Laksh Singla Date: Thu, 18 Nov 2021 18:00:44 +0530 Subject: [PATCH 07/18] Remove commented out explanations --- .../sql/avatica/DruidAvaticaHandlerTest.java | 5 ----- .../druid/sql/calcite/CalciteQueryTest.java | 21 ------------------- .../druid/sql/http/SqlResourceTest.java | 5 ----- 3 files changed, 31 deletions(-) diff --git a/sql/src/test/java/org/apache/druid/sql/avatica/DruidAvaticaHandlerTest.java b/sql/src/test/java/org/apache/druid/sql/avatica/DruidAvaticaHandlerTest.java index 3a28beb03b49..9a1b66b8ed3f 100644 --- a/sql/src/test/java/org/apache/druid/sql/avatica/DruidAvaticaHandlerTest.java +++ b/sql/src/test/java/org/apache/druid/sql/avatica/DruidAvaticaHandlerTest.java @@ -364,11 +364,6 @@ public void testExplainSelectCount() throws Exception "EXPLAIN PLAN FOR SELECT COUNT(*) AS cnt FROM druid.foo" ); - /*final String explanation = StringUtils.format( - "DruidQueryRel(query=[{\"queryType\":\"timeseries\",\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"descending\":false,\"virtualColumns\":[],\"filter\":null,\"granularity\":{\"type\":\"all\"},\"aggregations\":[{\"type\":\"count\",\"name\":\"a0\"}],\"postAggregations\":[],\"limit\":2147483647,\"context\":{\"sqlQueryId\":\"%s\",\"sqlStringifyArrays\":false,\"sqlTimeZone\":\"America/Los_Angeles\"}}], signature=[{a0:LONG}])\n", - DUMMY_SQL_QUERY_ID - )*/ - final String explanation = "[{\"query\":{\"queryType\":\"timeseries\",\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"descending\":false,\"virtualColumns\":[],\"filter\":null,\"granularity\":{\"type\":\"all\"},\"aggregations\":[{\"type\":\"count\",\"name\":\"a0\"}],\"postAggregations\":[],\"limit\":2147483647,\"context\":{\"sqlQueryId\":\"dummy\",\"sqlStringifyArrays\":false,\"sqlTimeZone\":\"America/Los_Angeles\"}},\"signature\":\"{a0:LONG}\"}]"; Assert.assertEquals( diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java index d50e9059d5d1..00af4a7b4f59 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java @@ -6069,20 +6069,6 @@ public void testExplainCountStarOnView() throws Exception // Skip vectorization since otherwise the "context" will change for each subtest. skipVectorize(); - /*final String explanation = - "DruidQueryRel(query=[{" - + "\"queryType\":\"timeseries\"," - + "\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"}," - + "\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]}," - + "\"descending\":false," - + "\"virtualColumns\":[]," - + "\"filter\":{\"type\":\"and\",\"fields\":[{\"type\":\"selector\",\"dimension\":\"dim2\",\"value\":\"a\",\"extractionFn\":null},{\"type\":\"not\",\"field\":{\"type\":\"selector\",\"dimension\":\"dim1\",\"value\":\"z\",\"extractionFn\":{\"type\":\"substring\",\"index\":0,\"length\":1}}}]}," - + "\"granularity\":{\"type\":\"all\"}," - + "\"aggregations\":[{\"type\":\"count\",\"name\":\"a0\"}]," - + "\"postAggregations\":[]," - + "\"limit\":2147483647," - + "\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\",\"sqlQueryId\":\"dummy\",\"vectorize\":\"false\",\"vectorizeVirtualColumns\":\"false\"}}]" - + ", signature=[{a0:LONG}])\n";*/ final String explanation = "[{" + "\"query\":{\"queryType\":\"timeseries\"," + "\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"}," @@ -9153,13 +9139,6 @@ public void testExplainExactCountDistinctOfSemiJoinResult() throws Exception { // Skip vectorization since otherwise the "context" will change for each subtest. skipVectorize(); - - /*final String explanation = - "DruidOuterQueryRel(query=[{\"queryType\":\"timeseries\",\"dataSource\":{\"type\":\"table\",\"name\":\"__subquery__\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"descending\":false,\"virtualColumns\":[],\"filter\":null,\"granularity\":{\"type\":\"all\"},\"aggregations\":[{\"type\":\"count\",\"name\":\"a0\"}],\"postAggregations\":[],\"limit\":2147483647,\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\",\"sqlQueryId\":\"dummy\",\"vectorize\":\"false\",\"vectorizeVirtualColumns\":\"false\"}}], signature=[{a0:LONG}])\n" - + " DruidJoinQueryRel(condition=[=(SUBSTRING($3, 1, 1), $8)], joinType=[inner], query=[{\"queryType\":\"groupBy\",\"dataSource\":{\"type\":\"table\",\"name\":\"__join__\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"virtualColumns\":[],\"filter\":null,\"granularity\":{\"type\":\"all\"},\"dimensions\":[{\"type\":\"default\",\"dimension\":\"dim2\",\"outputName\":\"d0\",\"outputType\":\"STRING\"}],\"aggregations\":[],\"postAggregations\":[],\"having\":null,\"limitSpec\":{\"type\":\"NoopLimitSpec\"},\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\",\"sqlQueryId\":\"dummy\",\"vectorize\":\"false\",\"vectorizeVirtualColumns\":\"false\"},\"descending\":false}], signature=[{d0:STRING}])\n" - + " DruidQueryRel(query=[{\"queryType\":\"scan\",\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"virtualColumns\":[],\"resultFormat\":\"compactedList\",\"batchSize\":20480,\"order\":\"none\",\"filter\":null,\"columns\":[\"__time\",\"cnt\",\"dim1\",\"dim2\",\"dim3\",\"m1\",\"m2\",\"unique_dim1\"],\"legacy\":false,\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\",\"sqlQueryId\":\"dummy\",\"vectorize\":\"false\",\"vectorizeVirtualColumns\":\"false\"},\"descending\":false,\"granularity\":{\"type\":\"all\"}}], signature=[{__time:LONG, cnt:LONG, dim1:STRING, dim2:STRING, dim3:STRING, m1:FLOAT, m2:DOUBLE, unique_dim1:COMPLEX}])\n" - + " DruidQueryRel(query=[{\"queryType\":\"groupBy\",\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"virtualColumns\":[],\"filter\":{\"type\":\"not\",\"field\":{\"type\":\"selector\",\"dimension\":\"dim1\",\"value\":null,\"extractionFn\":null}},\"granularity\":{\"type\":\"all\"},\"dimensions\":[{\"type\":\"extraction\",\"dimension\":\"dim1\",\"outputName\":\"d0\",\"outputType\":\"STRING\",\"extractionFn\":{\"type\":\"substring\",\"index\":0,\"length\":1}}],\"aggregations\":[],\"postAggregations\":[],\"having\":null,\"limitSpec\":{\"type\":\"NoopLimitSpec\"},\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\",\"sqlQueryId\":\"dummy\",\"vectorize\":\"false\",\"vectorizeVirtualColumns\":\"false\"},\"descending\":false}], signature=[{d0:STRING}])\n"; -*/ final String explanation = "[" + "{\"query\":{\"queryType\":\"groupBy\"," + "\"dataSource\":{\"type\":\"query\",\"query\":{\"queryType\":\"groupBy\",\"dataSource\":{\"type\":\"join\",\"left\":{\"type\":\"table\",\"name\":\"foo\"},\"right\":{\"type\":\"query\",\"query\":{\"queryType\":\"groupBy\",\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"virtualColumns\":[],\"filter\":{\"type\":\"not\",\"field\":{\"type\":\"selector\",\"dimension\":\"dim1\",\"value\":null,\"extractionFn\":null}},\"granularity\":{\"type\":\"all\"},\"dimensions\":[{\"type\":\"extraction\",\"dimension\":\"dim1\",\"outputName\":\"d0\",\"outputType\":\"STRING\",\"extractionFn\":{\"type\":\"substring\",\"index\":0,\"length\":1}}],\"aggregations\":[],\"postAggregations\":[],\"having\":null,\"limitSpec\":{\"type\":\"NoopLimitSpec\"},\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\",\"sqlQueryId\":\"dummy\",\"vectorize\":\"false\",\"vectorizeVirtualColumns\":\"false\"},\"descending\":false}},\"rightPrefix\":\"j0.\",\"condition\":\"(substring(\\\"dim2\\\", 0, 1) == \\\"j0.d0\\\")\",\"joinType\":\"INNER\",\"leftFilter\":null},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"virtualColumns\":[],\"filter\":null,\"granularity\":{\"type\":\"all\"},\"dimensions\":[{\"type\":\"default\",\"dimension\":\"dim2\",\"outputName\":\"d0\",\"outputType\":\"STRING\"}],\"aggregations\":[],\"postAggregations\":[],\"having\":null,\"limitSpec\":{\"type\":\"NoopLimitSpec\"},\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\",\"sqlQueryId\":\"dummy\",\"vectorize\":\"false\",\"vectorizeVirtualColumns\":\"false\"},\"descending\":false}}," diff --git a/sql/src/test/java/org/apache/druid/sql/http/SqlResourceTest.java b/sql/src/test/java/org/apache/druid/sql/http/SqlResourceTest.java index 1342fcac1592..784b2bae1527 100644 --- a/sql/src/test/java/org/apache/druid/sql/http/SqlResourceTest.java +++ b/sql/src/test/java/org/apache/druid/sql/http/SqlResourceTest.java @@ -1052,11 +1052,6 @@ public void testExplainCountStar() throws Exception null ) ).rhs; - /*final String explanation = StringUtils.format( - "DruidQueryRel(query=[{\"queryType\":\"timeseries\",\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"descending\":false,\"virtualColumns\":[],\"filter\":null,\"granularity\":{\"type\":\"all\"},\"aggregations\":[{\"type\":\"count\",\"name\":\"a0\"}],\"postAggregations\":[],\"limit\":2147483647,\"context\":{\"sqlQueryId\":\"%s\"}}], signature=[{a0:LONG}])\n", - DUMMY_SQL_QUERY_ID - ); - */ final String explanation = "[{\"query\":{\"queryType\":\"timeseries\",\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"descending\":false,\"virtualColumns\":[],\"filter\":null,\"granularity\":{\"type\":\"all\"},\"aggregations\":[{\"type\":\"count\",\"name\":\"a0\"}],\"postAggregations\":[],\"limit\":2147483647,\"context\":{\"sqlQueryId\":\"dummy\"}},\"signature\":\"{a0:LONG}\"}]"; Assert.assertEquals( From dcb58d9183217d8e8bb504e2a29a1c6dd7f4b7e8 Mon Sep 17 00:00:00 2001 From: Laksh Singla Date: Thu, 18 Nov 2021 18:03:51 +0530 Subject: [PATCH 08/18] Checkstyle fix --- .../java/org/apache/druid/sql/calcite/CalciteQueryTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java index 00af4a7b4f59..9c914e3b1d5e 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java @@ -1090,7 +1090,7 @@ public void testExplainSelectConstantExpression() throws Exception Map context = new HashMap<>(QUERY_CONTEXT_DEFAULT); context.put(PlannerConfig.CTX_KEY_USE_LEGACY_DRUID_EXPLAIN, true); - testQuery( + testQuery( "EXPLAIN PLAN FOR SELECT 1 + 1", context, ImmutableList.of(), From 0fe78ac7b2a62c1fc41daa72895b37b5b56632a2 Mon Sep 17 00:00:00 2001 From: Laksh Singla Date: Thu, 18 Nov 2021 18:13:25 +0530 Subject: [PATCH 09/18] Fix testcases, cleanup the remaining comments --- .../java/org/apache/druid/sql/calcite/CalciteQueryTest.java | 6 ------ 1 file changed, 6 deletions(-) diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java index 9c914e3b1d5e..7a63f378e0ef 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java @@ -1070,7 +1070,6 @@ public void testExplainSelectConstantExpression() throws Exception { // Skip vectorization since otherwise the "context" will change for each subtest. skipVectorize(); - // String explanation = "DruidQueryRel(query=[{\"queryType\":\"scan\",\"dataSource\":{\"type\":\"inline\",\"columnNames\":[\"EXPR$0\"],\"columnTypes\":[\"LONG\"],\"rows\":[[2]]},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"virtualColumns\":[],\"resultFormat\":\"compactedList\",\"batchSize\":20480,\"order\":\"none\",\"filter\":null,\"columns\":[\"EXPR$0\"],\"legacy\":false,\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\",\"sqlQueryId\":\"dummy\",\"vectorize\":\"false\",\"vectorizeVirtualColumns\":\"false\"},\"descending\":false,\"granularity\":{\"type\":\"all\"}}], signature=[{EXPR$0:LONG}])\n"; String explanation = "[{" + "\"query\":{\"queryType\":\"scan\"," + "\"dataSource\":{\"type\":\"inline\",\"columnNames\":[\"EXPR$0\"],\"columnTypes\":[\"LONG\"],\"rows\":[[2]]}," @@ -1088,11 +1087,8 @@ public void testExplainSelectConstantExpression() throws Exception + "\"signature\":\"{EXPR$0:LONG}\"" + "}]"; - Map context = new HashMap<>(QUERY_CONTEXT_DEFAULT); - context.put(PlannerConfig.CTX_KEY_USE_LEGACY_DRUID_EXPLAIN, true); testQuery( "EXPLAIN PLAN FOR SELECT 1 + 1", - context, ImmutableList.of(), ImmutableList.of( new Object[]{explanation, "[]"} @@ -1503,7 +1499,6 @@ public void testExplainSelectStar() throws Exception { // Skip vectorization since otherwise the "context" will change for each subtest. skipVectorize(); - // String explanation = "DruidQueryRel(query=[{\"queryType\":\"scan\",\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"virtualColumns\":[],\"resultFormat\":\"compactedList\",\"batchSize\":20480,\"order\":\"none\",\"filter\":null,\"columns\":[\"__time\",\"cnt\",\"dim1\",\"dim2\",\"dim3\",\"m1\",\"m2\",\"unique_dim1\"],\"legacy\":false,\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\",\"sqlQueryId\":\"dummy\",\"vectorize\":\"false\",\"vectorizeVirtualColumns\":\"false\"},\"descending\":false,\"granularity\":{\"type\":\"all\"}}], signature=[{__time:LONG, cnt:LONG, dim1:STRING, dim2:STRING, dim3:STRING, m1:FLOAT, m2:DOUBLE, unique_dim1:COMPLEX}])\n"; String explanation = "[{" + "\"query\":{\"queryType\":\"scan\"," + "\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"}," @@ -6068,7 +6063,6 @@ public void testExplainCountStarOnView() throws Exception { // Skip vectorization since otherwise the "context" will change for each subtest. skipVectorize(); - final String explanation = "[{" + "\"query\":{\"queryType\":\"timeseries\"," + "\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"}," From b728bd8bf97e7760a6aa060cd576ff3820a57dfd Mon Sep 17 00:00:00 2001 From: Laksh Singla Date: Fri, 19 Nov 2021 01:31:32 +0530 Subject: [PATCH 10/18] Add docs, throw ISE in a conditional --- docs/configuration/index.md | 1 + .../org/apache/druid/sql/calcite/planner/DruidPlanner.java | 7 ++++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/docs/configuration/index.md b/docs/configuration/index.md index 2afe3f9dab3d..45f2526c45b9 100644 --- a/docs/configuration/index.md +++ b/docs/configuration/index.md @@ -1827,6 +1827,7 @@ The Druid SQL server is configured through the following properties on the Broke |`druid.sql.planner.metadataSegmentCacheEnable`|Whether to keep a cache of published segments in broker. If true, broker polls coordinator in background to get segments from metadata store and maintains a local cache. If false, coordinator's REST API will be invoked when broker needs published segments info.|false| |`druid.sql.planner.metadataSegmentPollPeriod`|How often to poll coordinator for published segments list if `druid.sql.planner.metadataSegmentCacheEnable` is set to true. Poll period is in milliseconds. |60000| |`druid.sql.planner.authorizeSystemTablesDirectly`|If true, Druid authorizes queries against any of the system schema tables (`sys` in SQL) as `SYSTEM_TABLE` resources which require `READ` access, in addition to permissions based content filtering.|false| +|`druid.sql.planner.useLegacyDruidExplain`|If true, `EXPLAIN PLAN FOR` will return the explain plan in legacy format, else it will return a JSON representation of the native queries the given SQL statement translates to. It can be overridden per query with `useLegacyDruidExplain` context key.|false| |`druid.sql.approxCountDistinct.function`|Implementation to use for the [`APPROX_COUNT_DISTINCT` function](../querying/sql.md#aggregation-functions). Without extensions loaded, the only valid value is `APPROX_COUNT_DISTINCT_BUILTIN` (a HyperLogLog, or HLL, based implementation). If the [DataSketches extension](../development/extensions-core/datasketches-extension.md) is loaded, this can also be `APPROX_COUNT_DISTINCT_DS_HLL` (alternative HLL implementation) or `APPROX_COUNT_DISTINCT_DS_THETA`.

Theta sketches use significantly more memory than HLL sketches, so you should prefer one of the two HLL implementations.|APPROX_COUNT_DISTINCT_BUILTIN| > Previous versions of Druid had properties named `druid.sql.planner.maxQueryCount` and `druid.sql.planner.maxSemiJoinRowsInMemory`. diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/planner/DruidPlanner.java b/sql/src/main/java/org/apache/druid/sql/calcite/planner/DruidPlanner.java index f7a13650fc0d..e7ce52ee0a2e 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/planner/DruidPlanner.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/planner/DruidPlanner.java @@ -63,6 +63,7 @@ import org.apache.calcite.tools.RelConversionException; import org.apache.calcite.tools.ValidationException; import org.apache.calcite.util.Pair; +import org.apache.druid.java.util.common.ISE; import org.apache.druid.java.util.common.guava.BaseSequence; import org.apache.druid.java.util.common.guava.Sequence; import org.apache.druid.java.util.common.guava.Sequences; @@ -394,6 +395,10 @@ private PlannerResult planExplanation( log.error(jpe, "Encountered exception while serializing Resources for explain output"); resources = null; } + catch (ISE ise) { + log.error("Unable to translate to a native Druid query. Resorting to legacy Druid explain plan"); + resources = null; + } final Supplier> resultsSupplier = Suppliers.ofInstance( Sequences.simple(ImmutableList.of(new Object[]{explanation, resources}))); return new PlannerResult(resultsSupplier, getExplainStructType(rel.getCluster().getTypeFactory())); @@ -418,7 +423,7 @@ private String explainSqlPlanAsNativeQueries(DruidRel rel) throws JsonProcess druidQueryList = rel.getInputs().stream().map(childRel -> (DruidRel) childRel).map(childRel -> { if (childRel instanceof DruidUnionRel) { log.error("DruidUnionRel can only be the outermost RelNode. This error shouldn't be encountered"); - // TODO: Throw an error here + throw new ISE("DruidUnionRel is only supported at the outermost RelNode."); } return childRel.toDruidQuery(false); }).collect(Collectors.toList()); From c2b4690e2edf0628a5b8e98142529e0883127537 Mon Sep 17 00:00:00 2001 From: Laksh Singla Date: Tue, 23 Nov 2021 16:43:18 +0530 Subject: [PATCH 11/18] Add exception in the logging --- .../java/org/apache/druid/sql/calcite/planner/DruidPlanner.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/planner/DruidPlanner.java b/sql/src/main/java/org/apache/druid/sql/calcite/planner/DruidPlanner.java index e7ce52ee0a2e..b9b954b9332b 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/planner/DruidPlanner.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/planner/DruidPlanner.java @@ -396,7 +396,7 @@ private PlannerResult planExplanation( resources = null; } catch (ISE ise) { - log.error("Unable to translate to a native Druid query. Resorting to legacy Druid explain plan"); + log.error(ise, "Unable to translate to a native Druid query. Resorting to legacy Druid explain plan"); resources = null; } final Supplier> resultsSupplier = Suppliers.ofInstance( From 6d19f48576b2902a76eeee7721bb32746d598756 Mon Sep 17 00:00:00 2001 From: Laksh Singla Date: Wed, 24 Nov 2021 15:38:18 +0530 Subject: [PATCH 12/18] Change default format to legacy, update testcases to check for both the outputs --- .../sql/calcite/planner/PlannerConfig.java | 2 +- .../sql/calcite/BaseCalciteQueryTest.java | 9 +++ .../druid/sql/calcite/CalciteQueryTest.java | 57 +++++++++++---- .../sql/calcite/CalciteSelectQueryTest.java | 72 +++++++++++++++++-- .../druid/sql/http/SqlResourceTest.java | 7 +- 5 files changed, 126 insertions(+), 21 deletions(-) diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/planner/PlannerConfig.java b/sql/src/main/java/org/apache/druid/sql/calcite/planner/PlannerConfig.java index 047ca9aaf639..88d3ab5a7f42 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/planner/PlannerConfig.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/planner/PlannerConfig.java @@ -72,7 +72,7 @@ public class PlannerConfig private boolean authorizeSystemTablesDirectly = false; @JsonProperty - private boolean useLegacyDruidExplain = false; + private boolean useLegacyDruidExplain = true; public long getMetadataSegmentPollPeriod() { diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/BaseCalciteQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/BaseCalciteQueryTest.java index c536faa87e19..0f025f07949c 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/BaseCalciteQueryTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/BaseCalciteQueryTest.java @@ -189,6 +189,15 @@ public boolean isAuthorizeSystemTablesDirectly() } }; + public static final PlannerConfig PLANNER_CONFIG_LEGACY_DRUID_EXPLAIN_FALSE = new PlannerConfig() + { + @Override + public boolean isUseLegacyDruidExplain() + { + return false; + } + }; + public static final String DUMMY_SQL_ID = "dummy"; public static final String LOS_ANGELES = "America/Los_Angeles"; diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java index 58a120012558..de8731799369 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java @@ -4167,6 +4167,8 @@ public void testExplainCountStarOnView() throws Exception // Skip vectorization since otherwise the "context" will change for each subtest. skipVectorize(); + final String query = "EXPLAIN PLAN FOR SELECT COUNT(*) FROM view.aview WHERE dim1_firstchar <> 'z'"; + final String legacyExplanation = "DruidQueryRel(query=[{\"queryType\":\"timeseries\",\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"descending\":false,\"virtualColumns\":[],\"filter\":{\"type\":\"and\",\"fields\":[{\"type\":\"selector\",\"dimension\":\"dim2\",\"value\":\"a\",\"extractionFn\":null},{\"type\":\"not\",\"field\":{\"type\":\"selector\",\"dimension\":\"dim1\",\"value\":\"z\",\"extractionFn\":{\"type\":\"substring\",\"index\":0,\"length\":1}}}]},\"granularity\":{\"type\":\"all\"},\"aggregations\":[{\"type\":\"count\",\"name\":\"a0\"}],\"postAggregations\":[],\"limit\":2147483647,\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\",\"sqlQueryId\":\"dummy\",\"vectorize\":\"false\",\"vectorizeVirtualColumns\":\"false\"}}], signature=[{a0:LONG}])\n"; final String explanation = "DruidQueryRel(query=[{" + "\"queryType\":\"timeseries\"," @@ -4181,11 +4183,19 @@ public void testExplainCountStarOnView() throws Exception + "\"limit\":2147483647," + "\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\",\"sqlQueryId\":\"dummy\",\"vectorize\":\"false\",\"vectorizeVirtualColumns\":\"false\"}}]" + ", signature=[{a0:LONG}])\n"; - final String resources = "[{\"name\":\"aview\",\"type\":\"VIEW\"}]"; testQuery( - "EXPLAIN PLAN FOR SELECT COUNT(*) FROM view.aview WHERE dim1_firstchar <> 'z'", + query, + ImmutableList.of(), + ImmutableList.of( + new Object[]{legacyExplanation, resources} + ) + ); + testQuery( + PLANNER_CONFIG_LEGACY_DRUID_EXPLAIN_FALSE, + query, + CalciteTests.REGULAR_USER_AUTH_RESULT, ImmutableList.of(), ImmutableList.of( new Object[]{explanation, resources} @@ -6721,23 +6731,46 @@ public void testExplainExactCountDistinctOfSemiJoinResult() throws Exception // Skip vectorization since otherwise the "context" will change for each subtest. skipVectorize(); - final String explanation = + final String query = "EXPLAIN PLAN FOR SELECT COUNT(*)\n" + + "FROM (\n" + + " SELECT DISTINCT dim2\n" + + " FROM druid.foo\n" + + " WHERE SUBSTRING(dim2, 1, 1) IN (\n" + + " SELECT SUBSTRING(dim1, 1, 1) FROM druid.foo WHERE dim1 IS NOT NULL\n" + + " )\n" + + ")"; + final String legacyExplanation = "DruidOuterQueryRel(query=[{\"queryType\":\"timeseries\",\"dataSource\":{\"type\":\"table\",\"name\":\"__subquery__\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"descending\":false,\"virtualColumns\":[],\"filter\":null,\"granularity\":{\"type\":\"all\"},\"aggregations\":[{\"type\":\"count\",\"name\":\"a0\"}],\"postAggregations\":[],\"limit\":2147483647,\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\",\"sqlQueryId\":\"dummy\",\"vectorize\":\"false\",\"vectorizeVirtualColumns\":\"false\"}}], signature=[{a0:LONG}])\n" + " DruidJoinQueryRel(condition=[=(SUBSTRING($3, 1, 1), $8)], joinType=[inner], query=[{\"queryType\":\"groupBy\",\"dataSource\":{\"type\":\"table\",\"name\":\"__join__\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"virtualColumns\":[],\"filter\":null,\"granularity\":{\"type\":\"all\"},\"dimensions\":[{\"type\":\"default\",\"dimension\":\"dim2\",\"outputName\":\"d0\",\"outputType\":\"STRING\"}],\"aggregations\":[],\"postAggregations\":[],\"having\":null,\"limitSpec\":{\"type\":\"NoopLimitSpec\"},\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\",\"sqlQueryId\":\"dummy\",\"vectorize\":\"false\",\"vectorizeVirtualColumns\":\"false\"},\"descending\":false}], signature=[{d0:STRING}])\n" + " DruidQueryRel(query=[{\"queryType\":\"scan\",\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"virtualColumns\":[],\"resultFormat\":\"compactedList\",\"batchSize\":20480,\"filter\":null,\"columns\":[\"__time\",\"cnt\",\"dim1\",\"dim2\",\"dim3\",\"m1\",\"m2\",\"unique_dim1\"],\"legacy\":false,\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\",\"sqlQueryId\":\"dummy\",\"vectorize\":\"false\",\"vectorizeVirtualColumns\":\"false\"},\"descending\":false,\"granularity\":{\"type\":\"all\"}}], signature=[{__time:LONG, cnt:LONG, dim1:STRING, dim2:STRING, dim3:STRING, m1:FLOAT, m2:DOUBLE, unique_dim1:COMPLEX}])\n" + " DruidQueryRel(query=[{\"queryType\":\"groupBy\",\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"virtualColumns\":[],\"filter\":{\"type\":\"not\",\"field\":{\"type\":\"selector\",\"dimension\":\"dim1\",\"value\":null,\"extractionFn\":null}},\"granularity\":{\"type\":\"all\"},\"dimensions\":[{\"type\":\"extraction\",\"dimension\":\"dim1\",\"outputName\":\"d0\",\"outputType\":\"STRING\",\"extractionFn\":{\"type\":\"substring\",\"index\":0,\"length\":1}}],\"aggregations\":[],\"postAggregations\":[],\"having\":null,\"limitSpec\":{\"type\":\"NoopLimitSpec\"},\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\",\"sqlQueryId\":\"dummy\",\"vectorize\":\"false\",\"vectorizeVirtualColumns\":\"false\"},\"descending\":false}], signature=[{d0:STRING}])\n"; - + final String explanation = "[" + + "{\"query\":{\"queryType\":\"groupBy\"," + + "\"dataSource\":{\"type\":\"query\",\"query\":{\"queryType\":\"groupBy\",\"dataSource\":{\"type\":\"join\",\"left\":{\"type\":\"table\",\"name\":\"foo\"},\"right\":{\"type\":\"query\",\"query\":{\"queryType\":\"groupBy\",\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"virtualColumns\":[],\"filter\":{\"type\":\"not\",\"field\":{\"type\":\"selector\",\"dimension\":\"dim1\",\"value\":null,\"extractionFn\":null}},\"granularity\":{\"type\":\"all\"},\"dimensions\":[{\"type\":\"extraction\",\"dimension\":\"dim1\",\"outputName\":\"d0\",\"outputType\":\"STRING\",\"extractionFn\":{\"type\":\"substring\",\"index\":0,\"length\":1}}],\"aggregations\":[],\"postAggregations\":[],\"having\":null,\"limitSpec\":{\"type\":\"NoopLimitSpec\"},\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\",\"sqlQueryId\":\"dummy\",\"vectorize\":\"false\",\"vectorizeVirtualColumns\":\"false\"},\"descending\":false}},\"rightPrefix\":\"j0.\",\"condition\":\"(substring(\\\"dim2\\\", 0, 1) == \\\"j0.d0\\\")\",\"joinType\":\"INNER\",\"leftFilter\":null},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"virtualColumns\":[],\"filter\":null,\"granularity\":{\"type\":\"all\"},\"dimensions\":[{\"type\":\"default\",\"dimension\":\"dim2\",\"outputName\":\"d0\",\"outputType\":\"STRING\"}],\"aggregations\":[],\"postAggregations\":[],\"having\":null,\"limitSpec\":{\"type\":\"NoopLimitSpec\"},\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\",\"sqlQueryId\":\"dummy\",\"vectorize\":\"false\",\"vectorizeVirtualColumns\":\"false\"},\"descending\":false}}," + + "\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]}," + + "\"virtualColumns\":[]," + + "\"filter\":null," + + "\"granularity\":{\"type\":\"all\"}," + + "\"dimensions\":[]," + + "\"aggregations\":[{\"type\":\"count\",\"name\":\"a0\"}]," + + "\"postAggregations\":[]," + + "\"having\":null," + + "\"limitSpec\":{\"type\":\"NoopLimitSpec\"}," + + "\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\",\"sqlQueryId\":\"dummy\",\"vectorize\":\"false\",\"vectorizeVirtualColumns\":\"false\"}," + + "\"descending\":false}," + + "\"signature\":\"{a0:LONG}\"}]"; final String resources = "[{\"name\":\"foo\",\"type\":\"DATASOURCE\"}]"; testQuery( - "EXPLAIN PLAN FOR SELECT COUNT(*)\n" - + "FROM (\n" - + " SELECT DISTINCT dim2\n" - + " FROM druid.foo\n" - + " WHERE SUBSTRING(dim2, 1, 1) IN (\n" - + " SELECT SUBSTRING(dim1, 1, 1) FROM druid.foo WHERE dim1 IS NOT NULL\n" - + " )\n" - + ")", + query, + ImmutableList.of(), + ImmutableList.of(new Object[]{legacyExplanation, resources}) + ); + + testQuery( + PLANNER_CONFIG_LEGACY_DRUID_EXPLAIN_FALSE, + query, + CalciteTests.REGULAR_USER_AUTH_RESULT, ImmutableList.of(), ImmutableList.of(new Object[]{explanation, resources}) ); diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteSelectQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteSelectQueryTest.java index c6f9808a9901..1e1b6f455f8c 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteSelectQueryTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteSelectQueryTest.java @@ -409,14 +409,44 @@ public void testExplainSelectConstantExpression() throws Exception { // Skip vectorization since otherwise the "context" will change for each subtest. skipVectorize(); + final String query = "EXPLAIN PLAN FOR SELECT 1 + 1"; + final String explanation = "[{" + + "\"query\":{\"queryType\":\"scan\"," + + "\"dataSource\":{\"type\":\"inline\",\"columnNames\":[\"EXPR$0\"],\"columnTypes\":[\"LONG\"],\"rows\":[[2]]}," + + "\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]}," + + "\"virtualColumns\":[]," + + "\"resultFormat\":\"compactedList\"," + + "\"batchSize\":20480," + + "\"filter\":null," + + "\"columns\":[\"EXPR$0\"]," + + "\"legacy\":false," + + "\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\",\"sqlQueryId\":\"dummy\",\"vectorize\":\"false\",\"vectorizeVirtualColumns\":\"false\"}," + + "\"descending\":false," + + "\"granularity\":{\"type\":\"all\"}}," + + "\"signature\":\"{EXPR$0:LONG}\"" + + "}]"; + final String legacyExplanation = "DruidQueryRel(query=[{\"queryType\":\"scan\",\"dataSource\":{\"type\":\"inline\",\"columnNames\":[\"EXPR$0\"],\"columnTypes\":[\"LONG\"],\"rows\":[[2]]},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"virtualColumns\":[],\"resultFormat\":\"compactedList\",\"batchSize\":20480,\"filter\":null,\"columns\":[\"EXPR$0\"],\"legacy\":false,\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\",\"sqlQueryId\":\"dummy\",\"vectorize\":\"false\",\"vectorizeVirtualColumns\":\"false\"},\"descending\":false,\"granularity\":{\"type\":\"all\"}}], signature=[{EXPR$0:LONG}])\n"; + final String resources = "[]"; testQuery( - "EXPLAIN PLAN FOR SELECT 1 + 1", + query, ImmutableList.of(), ImmutableList.of( new Object[]{ - "DruidQueryRel(query=[{\"queryType\":\"scan\",\"dataSource\":{\"type\":\"inline\",\"columnNames\":[\"EXPR$0\"],\"columnTypes\":[\"LONG\"],\"rows\":[[2]]},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"virtualColumns\":[],\"resultFormat\":\"compactedList\",\"batchSize\":20480,\"filter\":null,\"columns\":[\"EXPR$0\"],\"legacy\":false,\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\",\"sqlQueryId\":\"dummy\",\"vectorize\":\"false\",\"vectorizeVirtualColumns\":\"false\"},\"descending\":false,\"granularity\":{\"type\":\"all\"}}], signature=[{EXPR$0:LONG}])\n", - "[]" + legacyExplanation, + resources + } + ) + ); + testQuery( + PLANNER_CONFIG_LEGACY_DRUID_EXPLAIN_FALSE, + query, + CalciteTests.REGULAR_USER_AUTH_RESULT, + ImmutableList.of(), + ImmutableList.of( + new Object[]{ + explanation, + resources } ) ); @@ -1078,13 +1108,43 @@ public void testExplainSelectStar() throws Exception // Skip vectorization since otherwise the "context" will change for each subtest. skipVectorize(); + final String query = "EXPLAIN PLAN FOR SELECT * FROM druid.foo"; + final String legacyExplanation = "DruidQueryRel(query=[{\"queryType\":\"scan\",\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"virtualColumns\":[],\"resultFormat\":\"compactedList\",\"batchSize\":20480,\"filter\":null,\"columns\":[\"__time\",\"cnt\",\"dim1\",\"dim2\",\"dim3\",\"m1\",\"m2\",\"unique_dim1\"],\"legacy\":false,\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\",\"sqlQueryId\":\"dummy\",\"vectorize\":\"false\",\"vectorizeVirtualColumns\":\"false\"},\"descending\":false,\"granularity\":{\"type\":\"all\"}}], signature=[{__time:LONG, cnt:LONG, dim1:STRING, dim2:STRING, dim3:STRING, m1:FLOAT, m2:DOUBLE, unique_dim1:COMPLEX}])\n"; + final String explanation ="[{" + + "\"query\":{\"queryType\":\"scan\"," + + "\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"}," + + "\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]}," + + "\"virtualColumns\":[]," + + "\"resultFormat\":\"compactedList\"," + + "\"batchSize\":20480," + + "\"filter\":null," + + "\"columns\":[\"__time\",\"cnt\",\"dim1\",\"dim2\",\"dim3\",\"m1\",\"m2\",\"unique_dim1\"]," + + "\"legacy\":false," + + "\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\",\"sqlQueryId\":\"dummy\",\"vectorize\":\"false\",\"vectorizeVirtualColumns\":\"false\"}," + + "\"descending\":false," + + "\"granularity\":{\"type\":\"all\"}}," + + "\"signature\":\"{__time:LONG, cnt:LONG, dim1:STRING, dim2:STRING, dim3:STRING, m1:FLOAT, m2:DOUBLE, unique_dim1:COMPLEX}\"}]"; + final String resources = "[{\"name\":\"foo\",\"type\":\"DATASOURCE\"}]"; + testQuery( - "EXPLAIN PLAN FOR SELECT * FROM druid.foo", + query, + ImmutableList.of(), + ImmutableList.of( + new Object[]{ + legacyExplanation, + resources + } + ) + ); + testQuery( + PLANNER_CONFIG_LEGACY_DRUID_EXPLAIN_FALSE, + query, + CalciteTests.REGULAR_USER_AUTH_RESULT, ImmutableList.of(), ImmutableList.of( new Object[]{ - "DruidQueryRel(query=[{\"queryType\":\"scan\",\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"virtualColumns\":[],\"resultFormat\":\"compactedList\",\"batchSize\":20480,\"filter\":null,\"columns\":[\"__time\",\"cnt\",\"dim1\",\"dim2\",\"dim3\",\"m1\",\"m2\",\"unique_dim1\"],\"legacy\":false,\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\",\"sqlQueryId\":\"dummy\",\"vectorize\":\"false\",\"vectorizeVirtualColumns\":\"false\"},\"descending\":false,\"granularity\":{\"type\":\"all\"}}], signature=[{__time:LONG, cnt:LONG, dim1:STRING, dim2:STRING, dim3:STRING, m1:FLOAT, m2:DOUBLE, unique_dim1:COMPLEX}])\n", - "[{\"name\":\"foo\",\"type\":\"DATASOURCE\"}]" + explanation, + resources } ) ); diff --git a/sql/src/test/java/org/apache/druid/sql/http/SqlResourceTest.java b/sql/src/test/java/org/apache/druid/sql/http/SqlResourceTest.java index 784b2bae1527..c09a6114870d 100644 --- a/sql/src/test/java/org/apache/druid/sql/http/SqlResourceTest.java +++ b/sql/src/test/java/org/apache/druid/sql/http/SqlResourceTest.java @@ -38,6 +38,7 @@ import org.apache.druid.java.util.common.ISE; import org.apache.druid.java.util.common.NonnullPair; import org.apache.druid.java.util.common.Pair; +import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.java.util.common.concurrent.Execs; import org.apache.druid.java.util.common.guava.LazySequence; import org.apache.druid.java.util.common.guava.Sequence; @@ -1052,13 +1053,15 @@ public void testExplainCountStar() throws Exception null ) ).rhs; - final String explanation = "[{\"query\":{\"queryType\":\"timeseries\",\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"descending\":false,\"virtualColumns\":[],\"filter\":null,\"granularity\":{\"type\":\"all\"},\"aggregations\":[{\"type\":\"count\",\"name\":\"a0\"}],\"postAggregations\":[],\"limit\":2147483647,\"context\":{\"sqlQueryId\":\"dummy\"}},\"signature\":\"{a0:LONG}\"}]"; Assert.assertEquals( ImmutableList.of( ImmutableMap.of( "PLAN", - explanation, + StringUtils.format( + "DruidQueryRel(query=[{\"queryType\":\"timeseries\",\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"descending\":false,\"virtualColumns\":[],\"filter\":null,\"granularity\":{\"type\":\"all\"},\"aggregations\":[{\"type\":\"count\",\"name\":\"a0\"}],\"postAggregations\":[],\"limit\":2147483647,\"context\":{\"sqlQueryId\":\"%s\"}}], signature=[{a0:LONG}])\n", + DUMMY_SQL_QUERY_ID + ), "RESOURCES", "[{\"name\":\"foo\",\"type\":\"DATASOURCE\"}]" ) From a7b880ea1b7e37c0626eadc22d1ef779c81319fb Mon Sep 17 00:00:00 2001 From: Laksh Singla Date: Wed, 24 Nov 2021 16:02:22 +0530 Subject: [PATCH 13/18] Add tests for overrides --- .../sql/avatica/DruidAvaticaHandlerTest.java | 6 +- .../druid/sql/calcite/CalciteQueryTest.java | 79 +++++++++++++++++++ 2 files changed, 82 insertions(+), 3 deletions(-) diff --git a/sql/src/test/java/org/apache/druid/sql/avatica/DruidAvaticaHandlerTest.java b/sql/src/test/java/org/apache/druid/sql/avatica/DruidAvaticaHandlerTest.java index 9a1b66b8ed3f..d767eff8665d 100644 --- a/sql/src/test/java/org/apache/druid/sql/avatica/DruidAvaticaHandlerTest.java +++ b/sql/src/test/java/org/apache/druid/sql/avatica/DruidAvaticaHandlerTest.java @@ -364,13 +364,13 @@ public void testExplainSelectCount() throws Exception "EXPLAIN PLAN FOR SELECT COUNT(*) AS cnt FROM druid.foo" ); - final String explanation = "[{\"query\":{\"queryType\":\"timeseries\",\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"descending\":false,\"virtualColumns\":[],\"filter\":null,\"granularity\":{\"type\":\"all\"},\"aggregations\":[{\"type\":\"count\",\"name\":\"a0\"}],\"postAggregations\":[],\"limit\":2147483647,\"context\":{\"sqlQueryId\":\"dummy\",\"sqlStringifyArrays\":false,\"sqlTimeZone\":\"America/Los_Angeles\"}},\"signature\":\"{a0:LONG}\"}]"; - Assert.assertEquals( ImmutableList.of( ImmutableMap.of( "PLAN", - explanation, + StringUtils.format("DruidQueryRel(query=[{\"queryType\":\"timeseries\",\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"descending\":false,\"virtualColumns\":[],\"filter\":null,\"granularity\":{\"type\":\"all\"},\"aggregations\":[{\"type\":\"count\",\"name\":\"a0\"}],\"postAggregations\":[],\"limit\":2147483647,\"context\":{\"sqlQueryId\":\"%s\",\"sqlStringifyArrays\":false,\"sqlTimeZone\":\"America/Los_Angeles\"}}], signature=[{a0:LONG}])\n", + DUMMY_SQL_QUERY_ID + ), "RESOURCES", "[{\"name\":\"foo\",\"type\":\"DATASOURCE\"}]" ) diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java index de8731799369..c43774d8cc55 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java @@ -6776,6 +6776,85 @@ public void testExplainExactCountDistinctOfSemiJoinResult() throws Exception ); } + // This testcase has been added here and not in CalciteSelectQueryTests since this checks if the overrides are working + // properly when displaying the output of "EXPLAIN PLAN FOR ..." queries + @Test + public void testExplainSelectStarWithOverrides() throws Exception + { + Map useLegacyDruidExplainContext = new HashMap<>(QUERY_CONTEXT_DEFAULT); + useLegacyDruidExplainContext.put(PlannerConfig.CTX_KEY_USE_LEGACY_DRUID_EXPLAIN, true); + + Map useRegularDruidExplainContext = new HashMap<>(QUERY_CONTEXT_DEFAULT); + useRegularDruidExplainContext.put(PlannerConfig.CTX_KEY_USE_LEGACY_DRUID_EXPLAIN, false); + + + // Skip vectorization since otherwise the "context" will change for each subtest. + skipVectorize(); + String legacyExplanation = "DruidQueryRel(query=[{\"queryType\":\"scan\",\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"virtualColumns\":[],\"resultFormat\":\"compactedList\",\"batchSize\":20480,\"filter\":null,\"columns\":[\"__time\",\"cnt\",\"dim1\",\"dim2\",\"dim3\",\"m1\",\"m2\",\"unique_dim1\"],\"legacy\":false,\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\",\"sqlQueryId\":\"dummy\",\"vectorize\":\"false\",\"vectorizeVirtualColumns\":\"false\"},\"descending\":false,\"granularity\":{\"type\":\"all\"}}], signature=[{__time:LONG, cnt:LONG, dim1:STRING, dim2:STRING, dim3:STRING, m1:FLOAT, m2:DOUBLE, unique_dim1:COMPLEX}])\n"; + String legacyExplanationWithContext = "DruidQueryRel(query=[{\"queryType\":\"scan\",\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"virtualColumns\":[],\"resultFormat\":\"compactedList\",\"batchSize\":20480,\"filter\":null,\"columns\":[\"__time\",\"cnt\",\"dim1\",\"dim2\",\"dim3\",\"m1\",\"m2\",\"unique_dim1\"],\"legacy\":false,\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\",\"sqlQueryId\":\"dummy\",\"useLegacyDruidExplain\":true},\"descending\":false,\"granularity\":{\"type\":\"all\"}}], signature=[{__time:LONG, cnt:LONG, dim1:STRING, dim2:STRING, dim3:STRING, m1:FLOAT, m2:DOUBLE, unique_dim1:COMPLEX}])\n"; + String explanation = "[{" + + "\"query\":{\"queryType\":\"scan\"," + + "\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"}," + + "\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]}," + + "\"virtualColumns\":[]," + + "\"resultFormat\":\"compactedList\"," + + "\"batchSize\":20480," + + "\"filter\":null," + + "\"columns\":[\"__time\",\"cnt\",\"dim1\",\"dim2\",\"dim3\",\"m1\",\"m2\",\"unique_dim1\"]," + + "\"legacy\":false," + + "\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\",\"sqlQueryId\":\"dummy\",\"vectorize\":\"false\",\"vectorizeVirtualColumns\":\"false\"}," + + "\"descending\":false," + + "\"granularity\":{\"type\":\"all\"}}," + + "\"signature\":\"{__time:LONG, cnt:LONG, dim1:STRING, dim2:STRING, dim3:STRING, m1:FLOAT, m2:DOUBLE, unique_dim1:COMPLEX}\"}]"; + + String explanationWithContext = "[{" + + "\"query\":{\"queryType\":\"scan\"," + + "\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"}," + + "\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]}," + + "\"virtualColumns\":[]," + + "\"resultFormat\":\"compactedList\"," + + "\"batchSize\":20480," + + "\"filter\":null," + + "\"columns\":[\"__time\",\"cnt\",\"dim1\",\"dim2\",\"dim3\",\"m1\",\"m2\",\"unique_dim1\"]," + + "\"legacy\":false," + + "\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\",\"sqlQueryId\":\"dummy\",\"useLegacyDruidExplain\":false,\"vectorize\":\"false\",\"vectorizeVirtualColumns\":\"false\"}," + + "\"descending\":false," + + "\"granularity\":{\"type\":\"all\"}}," + + "\"signature\":\"{__time:LONG, cnt:LONG, dim1:STRING, dim2:STRING, dim3:STRING, m1:FLOAT, m2:DOUBLE, unique_dim1:COMPLEX}\"}]"; + String sql = "EXPLAIN PLAN FOR SELECT * FROM druid.foo"; + String resources = "[{\"name\":\"foo\",\"type\":\"DATASOURCE\"}]"; + + // Test when default config and no overrides + testQuery(sql, ImmutableList.of(), ImmutableList.of(new Object[]{legacyExplanation, resources})); + + // Test when default config and useLegacyDruidExplain is overridden in the context + testQuery( + sql, + useRegularDruidExplainContext, + ImmutableList.of(), + ImmutableList.of(new Object[]{explanationWithContext, resources}) + ); + + // Test when useLegacyDruidExplain disabled by default and no overrides + testQuery( + PLANNER_CONFIG_LEGACY_DRUID_EXPLAIN_FALSE, + sql, + CalciteTests.REGULAR_USER_AUTH_RESULT, + ImmutableList.of(), + ImmutableList.of(new Object[]{explanation, resources}) + ); + + // Test when useLegacyDruidExplain disabled by default but is overriden in the context + testQuery( + PLANNER_CONFIG_LEGACY_DRUID_EXPLAIN_FALSE, + useLegacyDruidExplainContext, + sql, + CalciteTests.REGULAR_USER_AUTH_RESULT, + ImmutableList.of(), + ImmutableList.of(new Object[]{legacyExplanationWithContext, resources}) + ); + } + @Test public void testExactCountDistinctUsingSubqueryWithWherePushDown() throws Exception { From 11274af1c4443df464253456496e54f3c4bd4552 Mon Sep 17 00:00:00 2001 From: Laksh Singla Date: Wed, 24 Nov 2021 16:11:57 +0530 Subject: [PATCH 14/18] Checkstyle, testcase fix --- .../druid/sql/calcite/CalciteQueryTest.java | 27 +++++++++---------- .../sql/calcite/CalciteSelectQueryTest.java | 2 +- 2 files changed, 14 insertions(+), 15 deletions(-) diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java index c43774d8cc55..50cf7f990a6a 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java @@ -4169,20 +4169,19 @@ public void testExplainCountStarOnView() throws Exception final String query = "EXPLAIN PLAN FOR SELECT COUNT(*) FROM view.aview WHERE dim1_firstchar <> 'z'"; final String legacyExplanation = "DruidQueryRel(query=[{\"queryType\":\"timeseries\",\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"descending\":false,\"virtualColumns\":[],\"filter\":{\"type\":\"and\",\"fields\":[{\"type\":\"selector\",\"dimension\":\"dim2\",\"value\":\"a\",\"extractionFn\":null},{\"type\":\"not\",\"field\":{\"type\":\"selector\",\"dimension\":\"dim1\",\"value\":\"z\",\"extractionFn\":{\"type\":\"substring\",\"index\":0,\"length\":1}}}]},\"granularity\":{\"type\":\"all\"},\"aggregations\":[{\"type\":\"count\",\"name\":\"a0\"}],\"postAggregations\":[],\"limit\":2147483647,\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\",\"sqlQueryId\":\"dummy\",\"vectorize\":\"false\",\"vectorizeVirtualColumns\":\"false\"}}], signature=[{a0:LONG}])\n"; - final String explanation = - "DruidQueryRel(query=[{" - + "\"queryType\":\"timeseries\"," - + "\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"}," - + "\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]}," - + "\"descending\":false," - + "\"virtualColumns\":[]," - + "\"filter\":{\"type\":\"and\",\"fields\":[{\"type\":\"selector\",\"dimension\":\"dim2\",\"value\":\"a\",\"extractionFn\":null},{\"type\":\"not\",\"field\":{\"type\":\"selector\",\"dimension\":\"dim1\",\"value\":\"z\",\"extractionFn\":{\"type\":\"substring\",\"index\":0,\"length\":1}}}]}," - + "\"granularity\":{\"type\":\"all\"}," - + "\"aggregations\":[{\"type\":\"count\",\"name\":\"a0\"}]," - + "\"postAggregations\":[]," - + "\"limit\":2147483647," - + "\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\",\"sqlQueryId\":\"dummy\",\"vectorize\":\"false\",\"vectorizeVirtualColumns\":\"false\"}}]" - + ", signature=[{a0:LONG}])\n"; + final String explanation = "[{" + + "\"query\":{\"queryType\":\"timeseries\"," + + "\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"}," + + "\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]}," + + "\"descending\":false," + + "\"virtualColumns\":[]," + + "\"filter\":{\"type\":\"and\",\"fields\":[{\"type\":\"selector\",\"dimension\":\"dim2\",\"value\":\"a\",\"extractionFn\":null},{\"type\":\"not\",\"field\":{\"type\":\"selector\",\"dimension\":\"dim1\",\"value\":\"z\",\"extractionFn\":{\"type\":\"substring\",\"index\":0,\"length\":1}}}]}," + + "\"granularity\":{\"type\":\"all\"}," + + "\"aggregations\":[{\"type\":\"count\",\"name\":\"a0\"}]," + + "\"postAggregations\":[]," + + "\"limit\":2147483647," + + "\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\",\"sqlQueryId\":\"dummy\",\"vectorize\":\"false\",\"vectorizeVirtualColumns\":\"false\"}}," + + "\"signature\":\"{a0:LONG}\"}]"; final String resources = "[{\"name\":\"aview\",\"type\":\"VIEW\"}]"; testQuery( diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteSelectQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteSelectQueryTest.java index 1e1b6f455f8c..08c2062906ea 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteSelectQueryTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteSelectQueryTest.java @@ -1110,7 +1110,7 @@ public void testExplainSelectStar() throws Exception final String query = "EXPLAIN PLAN FOR SELECT * FROM druid.foo"; final String legacyExplanation = "DruidQueryRel(query=[{\"queryType\":\"scan\",\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"virtualColumns\":[],\"resultFormat\":\"compactedList\",\"batchSize\":20480,\"filter\":null,\"columns\":[\"__time\",\"cnt\",\"dim1\",\"dim2\",\"dim3\",\"m1\",\"m2\",\"unique_dim1\"],\"legacy\":false,\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\",\"sqlQueryId\":\"dummy\",\"vectorize\":\"false\",\"vectorizeVirtualColumns\":\"false\"},\"descending\":false,\"granularity\":{\"type\":\"all\"}}], signature=[{__time:LONG, cnt:LONG, dim1:STRING, dim2:STRING, dim3:STRING, m1:FLOAT, m2:DOUBLE, unique_dim1:COMPLEX}])\n"; - final String explanation ="[{" + final String explanation = "[{" + "\"query\":{\"queryType\":\"scan\"," + "\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"}," + "\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]}," From 0dd494a5dd5d48d1cb40987f8f5f1997f611e1a0 Mon Sep 17 00:00:00 2001 From: Laksh Singla Date: Wed, 24 Nov 2021 16:13:55 +0530 Subject: [PATCH 15/18] Nit, doc default value fix --- docs/configuration/index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/configuration/index.md b/docs/configuration/index.md index 45f2526c45b9..fb5e7c67a6df 100644 --- a/docs/configuration/index.md +++ b/docs/configuration/index.md @@ -1827,7 +1827,7 @@ The Druid SQL server is configured through the following properties on the Broke |`druid.sql.planner.metadataSegmentCacheEnable`|Whether to keep a cache of published segments in broker. If true, broker polls coordinator in background to get segments from metadata store and maintains a local cache. If false, coordinator's REST API will be invoked when broker needs published segments info.|false| |`druid.sql.planner.metadataSegmentPollPeriod`|How often to poll coordinator for published segments list if `druid.sql.planner.metadataSegmentCacheEnable` is set to true. Poll period is in milliseconds. |60000| |`druid.sql.planner.authorizeSystemTablesDirectly`|If true, Druid authorizes queries against any of the system schema tables (`sys` in SQL) as `SYSTEM_TABLE` resources which require `READ` access, in addition to permissions based content filtering.|false| -|`druid.sql.planner.useLegacyDruidExplain`|If true, `EXPLAIN PLAN FOR` will return the explain plan in legacy format, else it will return a JSON representation of the native queries the given SQL statement translates to. It can be overridden per query with `useLegacyDruidExplain` context key.|false| +|`druid.sql.planner.useLegacyDruidExplain`|If true, `EXPLAIN PLAN FOR` will return the explain plan in legacy format, else it will return a JSON representation of the native queries the given SQL statement translates to. It can be overridden per query with `useLegacyDruidExplain` context key.|true| |`druid.sql.approxCountDistinct.function`|Implementation to use for the [`APPROX_COUNT_DISTINCT` function](../querying/sql.md#aggregation-functions). Without extensions loaded, the only valid value is `APPROX_COUNT_DISTINCT_BUILTIN` (a HyperLogLog, or HLL, based implementation). If the [DataSketches extension](../development/extensions-core/datasketches-extension.md) is loaded, this can also be `APPROX_COUNT_DISTINCT_DS_HLL` (alternative HLL implementation) or `APPROX_COUNT_DISTINCT_DS_THETA`.

Theta sketches use significantly more memory than HLL sketches, so you should prefer one of the two HLL implementations.|APPROX_COUNT_DISTINCT_BUILTIN| > Previous versions of Druid had properties named `druid.sql.planner.maxQueryCount` and `druid.sql.planner.maxSemiJoinRowsInMemory`. From 1a52f1692cffcf8d8e7d9cddd8d931623a2c3d36 Mon Sep 17 00:00:00 2001 From: Laksh Singla Date: Wed, 24 Nov 2021 17:40:14 +0530 Subject: [PATCH 16/18] Add temporary serialization for rowsignature --- .../druid/segment/column/RowSignature.java | 18 ++++++ .../segment/column/RowSignatureTest.java | 58 +++++++++++++++++++ 2 files changed, 76 insertions(+) create mode 100644 processing/src/test/java/org/apache/druid/segment/column/RowSignatureTest.java diff --git a/processing/src/main/java/org/apache/druid/segment/column/RowSignature.java b/processing/src/main/java/org/apache/druid/segment/column/RowSignature.java index 510ebe3043b9..d8ff16a6a3e1 100644 --- a/processing/src/main/java/org/apache/druid/segment/column/RowSignature.java +++ b/processing/src/main/java/org/apache/druid/segment/column/RowSignature.java @@ -19,8 +19,10 @@ package org.apache.druid.segment.column; +import com.fasterxml.jackson.annotation.JsonValue; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import it.unimi.dsi.fastutil.objects.Object2IntMap; import it.unimi.dsi.fastutil.objects.Object2IntOpenHashMap; import org.apache.druid.java.util.common.IAE; @@ -193,6 +195,22 @@ public String toString() return s.append("}").toString(); } + // Temporary method for displaying the output in SQL EXPLAIN PLAN + @JsonValue + public List> toJsonValue() + { + List> jsonMap = new ArrayList<>(); + for (String columnName : columnNames) { + jsonMap.add( + ImmutableMap.of( + "name", columnName, + "type", columnTypes.get(columnName).asTypeString() + ) + ); + } + return jsonMap; + } + @Nullable @Override public ColumnCapabilities getColumnCapabilities(String column) diff --git a/processing/src/test/java/org/apache/druid/segment/column/RowSignatureTest.java b/processing/src/test/java/org/apache/druid/segment/column/RowSignatureTest.java new file mode 100644 index 000000000000..2739aed02749 --- /dev/null +++ b/processing/src/test/java/org/apache/druid/segment/column/RowSignatureTest.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.segment.column; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.junit.Assert; +import org.junit.Test; + +public class RowSignatureTest +{ + + @Test + public void testRowSignatureSerializationToJson() throws JsonProcessingException + { + RowSignature rowSignature = RowSignature.builder() + .add("foo1", ColumnType.DOUBLE) + .add("foo2", ColumnType.STRING) + .add("foo3", ColumnType.FLOAT) + .add("foo4", ColumnType.LONG) + .add("foo5", ColumnType.UNKNOWN_COMPLEX) + .add("foo6", ColumnType.ofComplex("bar")) + .add("foo7", ColumnType.STRING_ARRAY) + .add("foo8", ColumnType.LONG_ARRAY) + .add("foo9", ColumnType.DOUBLE_ARRAY) + .build(); + String expectedSerialization = "[" + + "{\"name\":\"foo1\",\"type\":\"DOUBLE\"}," + + "{\"name\":\"foo2\",\"type\":\"STRING\"}," + + "{\"name\":\"foo3\",\"type\":\"FLOAT\"}," + + "{\"name\":\"foo4\",\"type\":\"LONG\"}," + + "{\"name\":\"foo5\",\"type\":\"COMPLEX\"}," + + "{\"name\":\"foo6\",\"type\":\"COMPLEX\"}," + + "{\"name\":\"foo7\",\"type\":\"ARRAY\"}," + + "{\"name\":\"foo8\",\"type\":\"ARRAY\"}," + + "{\"name\":\"foo9\",\"type\":\"ARRAY\"}" + + "]"; + String rowSignatureSerialized = new ObjectMapper().writeValueAsString(rowSignature); + Assert.assertEquals(expectedSerialization, rowSignatureSerialized); + } +} From 3e41ba369cf5ba53dc6d782c5f554a6258c39b72 Mon Sep 17 00:00:00 2001 From: Laksh Singla Date: Wed, 24 Nov 2021 18:21:39 +0530 Subject: [PATCH 17/18] Update testcases for the new signature implementation --- .../apache/druid/segment/column/RowSignature.java | 2 +- .../druid/sql/calcite/planner/DruidPlanner.java | 2 +- .../apache/druid/sql/calcite/CalciteQueryTest.java | 12 ++++++++---- .../druid/sql/calcite/CalciteSelectQueryTest.java | 11 ++++++----- 4 files changed, 16 insertions(+), 11 deletions(-) diff --git a/processing/src/main/java/org/apache/druid/segment/column/RowSignature.java b/processing/src/main/java/org/apache/druid/segment/column/RowSignature.java index d8ff16a6a3e1..71e9d3cbcc4f 100644 --- a/processing/src/main/java/org/apache/druid/segment/column/RowSignature.java +++ b/processing/src/main/java/org/apache/druid/segment/column/RowSignature.java @@ -197,7 +197,7 @@ public String toString() // Temporary method for displaying the output in SQL EXPLAIN PLAN @JsonValue - public List> toJsonValue() + private List> toJsonValue() { List> jsonMap = new ArrayList<>(); for (String columnName : columnNames) { diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/planner/DruidPlanner.java b/sql/src/main/java/org/apache/druid/sql/calcite/planner/DruidPlanner.java index b9b954b9332b..41e1dab2d841 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/planner/DruidPlanner.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/planner/DruidPlanner.java @@ -437,7 +437,7 @@ private String explainSqlPlanAsNativeQueries(DruidRel rel) throws JsonProcess Query nativeQuery = druidQuery.getQuery(); ObjectNode objectNode = jsonMapper.createObjectNode(); objectNode.put("query", jsonMapper.convertValue(nativeQuery, ObjectNode.class)); - objectNode.put("signature", druidQuery.getOutputRowSignature().toString()); + objectNode.put("signature", jsonMapper.convertValue(druidQuery.getOutputRowSignature(), ArrayNode.class)); nativeQueriesArrayNode.add(objectNode); } diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java index 50cf7f990a6a..47e6133e2f6d 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java @@ -4181,7 +4181,8 @@ public void testExplainCountStarOnView() throws Exception + "\"postAggregations\":[]," + "\"limit\":2147483647," + "\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\",\"sqlQueryId\":\"dummy\",\"vectorize\":\"false\",\"vectorizeVirtualColumns\":\"false\"}}," - + "\"signature\":\"{a0:LONG}\"}]"; + + "\"signature\":[{\"name\":\"a0\",\"type\":\"LONG\"}]" + + "}]"; final String resources = "[{\"name\":\"aview\",\"type\":\"VIEW\"}]"; testQuery( @@ -6757,7 +6758,8 @@ public void testExplainExactCountDistinctOfSemiJoinResult() throws Exception + "\"limitSpec\":{\"type\":\"NoopLimitSpec\"}," + "\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\",\"sqlQueryId\":\"dummy\",\"vectorize\":\"false\",\"vectorizeVirtualColumns\":\"false\"}," + "\"descending\":false}," - + "\"signature\":\"{a0:LONG}\"}]"; + + "\"signature\":[{\"name\":\"a0\",\"type\":\"LONG\"}]" + + "}]"; final String resources = "[{\"name\":\"foo\",\"type\":\"DATASOURCE\"}]"; testQuery( @@ -6804,7 +6806,8 @@ public void testExplainSelectStarWithOverrides() throws Exception + "\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\",\"sqlQueryId\":\"dummy\",\"vectorize\":\"false\",\"vectorizeVirtualColumns\":\"false\"}," + "\"descending\":false," + "\"granularity\":{\"type\":\"all\"}}," - + "\"signature\":\"{__time:LONG, cnt:LONG, dim1:STRING, dim2:STRING, dim3:STRING, m1:FLOAT, m2:DOUBLE, unique_dim1:COMPLEX}\"}]"; + + "\"signature\":[{\"name\":\"__time\",\"type\":\"LONG\"},{\"name\":\"cnt\",\"type\":\"LONG\"},{\"name\":\"dim1\",\"type\":\"STRING\"},{\"name\":\"dim2\",\"type\":\"STRING\"},{\"name\":\"dim3\",\"type\":\"STRING\"},{\"name\":\"m1\",\"type\":\"FLOAT\"},{\"name\":\"m2\",\"type\":\"DOUBLE\"},{\"name\":\"unique_dim1\",\"type\":\"COMPLEX\"}]" + + "}]"; String explanationWithContext = "[{" + "\"query\":{\"queryType\":\"scan\"," @@ -6819,7 +6822,8 @@ public void testExplainSelectStarWithOverrides() throws Exception + "\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\",\"sqlQueryId\":\"dummy\",\"useLegacyDruidExplain\":false,\"vectorize\":\"false\",\"vectorizeVirtualColumns\":\"false\"}," + "\"descending\":false," + "\"granularity\":{\"type\":\"all\"}}," - + "\"signature\":\"{__time:LONG, cnt:LONG, dim1:STRING, dim2:STRING, dim3:STRING, m1:FLOAT, m2:DOUBLE, unique_dim1:COMPLEX}\"}]"; + + "\"signature\":[{\"name\":\"__time\",\"type\":\"LONG\"},{\"name\":\"cnt\",\"type\":\"LONG\"},{\"name\":\"dim1\",\"type\":\"STRING\"},{\"name\":\"dim2\",\"type\":\"STRING\"},{\"name\":\"dim3\",\"type\":\"STRING\"},{\"name\":\"m1\",\"type\":\"FLOAT\"},{\"name\":\"m2\",\"type\":\"DOUBLE\"},{\"name\":\"unique_dim1\",\"type\":\"COMPLEX\"}]" + + "}]"; String sql = "EXPLAIN PLAN FOR SELECT * FROM druid.foo"; String resources = "[{\"name\":\"foo\",\"type\":\"DATASOURCE\"}]"; diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteSelectQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteSelectQueryTest.java index 08c2062906ea..905ed75ee07f 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteSelectQueryTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteSelectQueryTest.java @@ -423,7 +423,7 @@ public void testExplainSelectConstantExpression() throws Exception + "\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\",\"sqlQueryId\":\"dummy\",\"vectorize\":\"false\",\"vectorizeVirtualColumns\":\"false\"}," + "\"descending\":false," + "\"granularity\":{\"type\":\"all\"}}," - + "\"signature\":\"{EXPR$0:LONG}\"" + + "\"signature\":[{\"name\":\"EXPR$0\",\"type\":\"LONG\"}]" + "}]"; final String legacyExplanation = "DruidQueryRel(query=[{\"queryType\":\"scan\",\"dataSource\":{\"type\":\"inline\",\"columnNames\":[\"EXPR$0\"],\"columnTypes\":[\"LONG\"],\"rows\":[[2]]},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"virtualColumns\":[],\"resultFormat\":\"compactedList\",\"batchSize\":20480,\"filter\":null,\"columns\":[\"EXPR$0\"],\"legacy\":false,\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\",\"sqlQueryId\":\"dummy\",\"vectorize\":\"false\",\"vectorizeVirtualColumns\":\"false\"},\"descending\":false,\"granularity\":{\"type\":\"all\"}}], signature=[{EXPR$0:LONG}])\n"; final String resources = "[]"; @@ -1120,10 +1120,11 @@ public void testExplainSelectStar() throws Exception + "\"filter\":null," + "\"columns\":[\"__time\",\"cnt\",\"dim1\",\"dim2\",\"dim3\",\"m1\",\"m2\",\"unique_dim1\"]," + "\"legacy\":false," - + "\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\",\"sqlQueryId\":\"dummy\",\"vectorize\":\"false\",\"vectorizeVirtualColumns\":\"false\"}," - + "\"descending\":false," - + "\"granularity\":{\"type\":\"all\"}}," - + "\"signature\":\"{__time:LONG, cnt:LONG, dim1:STRING, dim2:STRING, dim3:STRING, m1:FLOAT, m2:DOUBLE, unique_dim1:COMPLEX}\"}]"; + + "\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\",\"sqlQueryId\":\"dummy\",\"vectorize\":\"false\",\"vectorizeVirtualColumns\":\"false\"}," + + "\"descending\":false," + + "\"granularity\":{\"type\":\"all\"}}," + + "\"signature\":[{\"name\":\"__time\",\"type\":\"LONG\"},{\"name\":\"cnt\",\"type\":\"LONG\"},{\"name\":\"dim1\",\"type\":\"STRING\"},{\"name\":\"dim2\",\"type\":\"STRING\"},{\"name\":\"dim3\",\"type\":\"STRING\"},{\"name\":\"m1\",\"type\":\"FLOAT\"},{\"name\":\"m2\",\"type\":\"DOUBLE\"},{\"name\":\"unique_dim1\",\"type\":\"COMPLEX\"}]" + + "}]"; final String resources = "[{\"name\":\"foo\",\"type\":\"DATASOURCE\"}]"; testQuery( From 28aac502824f8e7d5ac46e46c63cb2b9f9ab68ab Mon Sep 17 00:00:00 2001 From: Laksh Singla Date: Thu, 25 Nov 2021 11:45:55 +0530 Subject: [PATCH 18/18] Change the flag to useNativeQueryExplain --- docs/configuration/index.md | 2 +- .../sql/calcite/planner/DruidPlanner.java | 6 ++-- .../sql/calcite/planner/PlannerConfig.java | 20 ++++++------- .../sql/calcite/BaseCalciteQueryTest.java | 6 ++-- .../druid/sql/calcite/CalciteQueryTest.java | 30 +++++++++---------- .../sql/calcite/CalciteSelectQueryTest.java | 4 +-- 6 files changed, 35 insertions(+), 33 deletions(-) diff --git a/docs/configuration/index.md b/docs/configuration/index.md index ab8afb9ce688..0708c07e16db 100644 --- a/docs/configuration/index.md +++ b/docs/configuration/index.md @@ -1834,7 +1834,7 @@ The Druid SQL server is configured through the following properties on the Broke |`druid.sql.planner.metadataSegmentCacheEnable`|Whether to keep a cache of published segments in broker. If true, broker polls coordinator in background to get segments from metadata store and maintains a local cache. If false, coordinator's REST API will be invoked when broker needs published segments info.|false| |`druid.sql.planner.metadataSegmentPollPeriod`|How often to poll coordinator for published segments list if `druid.sql.planner.metadataSegmentCacheEnable` is set to true. Poll period is in milliseconds. |60000| |`druid.sql.planner.authorizeSystemTablesDirectly`|If true, Druid authorizes queries against any of the system schema tables (`sys` in SQL) as `SYSTEM_TABLE` resources which require `READ` access, in addition to permissions based content filtering.|false| -|`druid.sql.planner.useLegacyDruidExplain`|If true, `EXPLAIN PLAN FOR` will return the explain plan in legacy format, else it will return a JSON representation of the native queries the given SQL statement translates to. It can be overridden per query with `useLegacyDruidExplain` context key.|true| +|`druid.sql.planner.useNativeQueryExplain`|If true, `EXPLAIN PLAN FOR` will return the explain plan as a JSON representation of equivalent native query(s), else it will return the original version of explain plan generated by Calcite. It can be overridden per query with `useNativeQueryExplain` context key.|false| |`druid.sql.approxCountDistinct.function`|Implementation to use for the [`APPROX_COUNT_DISTINCT` function](../querying/sql.md#aggregation-functions). Without extensions loaded, the only valid value is `APPROX_COUNT_DISTINCT_BUILTIN` (a HyperLogLog, or HLL, based implementation). If the [DataSketches extension](../development/extensions-core/datasketches-extension.md) is loaded, this can also be `APPROX_COUNT_DISTINCT_DS_HLL` (alternative HLL implementation) or `APPROX_COUNT_DISTINCT_DS_THETA`.

Theta sketches use significantly more memory than HLL sketches, so you should prefer one of the two HLL implementations.|APPROX_COUNT_DISTINCT_BUILTIN| > Previous versions of Druid had properties named `druid.sql.planner.maxQueryCount` and `druid.sql.planner.maxSemiJoinRowsInMemory`. diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/planner/DruidPlanner.java b/sql/src/main/java/org/apache/druid/sql/calcite/planner/DruidPlanner.java index 9d40260de22b..3a5dbeb24fab 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/planner/DruidPlanner.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/planner/DruidPlanner.java @@ -93,8 +93,8 @@ import java.util.Iterator; import java.util.List; import java.util.Properties; -import java.util.stream.Collectors; import java.util.Set; +import java.util.stream.Collectors; public class DruidPlanner implements Closeable { @@ -397,7 +397,7 @@ private PlannerResult planExplanation( try { if (isDruidConventionExplanation && rel instanceof DruidRel) { // Show the native queries instead of Calcite's explain if the legacy flag is turned off - if (!plannerContext.getPlannerConfig().isUseLegacyDruidExplain()) { + if (plannerContext.getPlannerConfig().isUseNativeQueryExplain()) { DruidRel druidRel = (DruidRel) rel; explanation = explainSqlPlanAsNativeQueries(druidRel); } @@ -448,6 +448,8 @@ private String explainSqlPlanAsNativeQueries(DruidRel rel) throws JsonProcess druidQueryList = ImmutableList.of(rel.toDruidQuery(false)); } + // Putting the queries as object node in an ArrayNode, since directly returning a list causes issues when + // serializing the "queryType" ArrayNode nativeQueriesArrayNode = jsonMapper.createArrayNode(); for (DruidQuery druidQuery : druidQueryList) { diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/planner/PlannerConfig.java b/sql/src/main/java/org/apache/druid/sql/calcite/planner/PlannerConfig.java index 88d3ab5a7f42..6203fdce92de 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/planner/PlannerConfig.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/planner/PlannerConfig.java @@ -33,7 +33,7 @@ public class PlannerConfig public static final String CTX_KEY_USE_GROUPING_SET_FOR_EXACT_DISTINCT = "useGroupingSetForExactDistinct"; public static final String CTX_KEY_USE_APPROXIMATE_TOPN = "useApproximateTopN"; public static final String CTX_COMPUTE_INNER_JOIN_COST_AS_FILTER = "computeInnerJoinCostAsFilter"; - public static final String CTX_KEY_USE_LEGACY_DRUID_EXPLAIN = "useLegacyDruidExplain"; + public static final String CTX_KEY_USE_NATIVE_QUERY_EXPLAIN = "useNativeQueryExplain"; @JsonProperty private Period metadataRefreshPeriod = new Period("PT1M"); @@ -72,7 +72,7 @@ public class PlannerConfig private boolean authorizeSystemTablesDirectly = false; @JsonProperty - private boolean useLegacyDruidExplain = true; + private boolean useNativeQueryExplain = false; public long getMetadataSegmentPollPeriod() { @@ -141,9 +141,9 @@ public boolean isAuthorizeSystemTablesDirectly() return authorizeSystemTablesDirectly; } - public boolean isUseLegacyDruidExplain() + public boolean isUseNativeQueryExplain() { - return useLegacyDruidExplain; + return useNativeQueryExplain; } public PlannerConfig withOverrides(final Map context) @@ -175,10 +175,10 @@ public PlannerConfig withOverrides(final Map context) CTX_COMPUTE_INNER_JOIN_COST_AS_FILTER, computeInnerJoinCostAsFilter ); - newConfig.useLegacyDruidExplain = getContextBoolean( + newConfig.useNativeQueryExplain = getContextBoolean( context, - CTX_KEY_USE_LEGACY_DRUID_EXPLAIN, - isUseLegacyDruidExplain() + CTX_KEY_USE_NATIVE_QUERY_EXPLAIN, + isUseNativeQueryExplain() ); newConfig.requireTimeCondition = isRequireTimeCondition(); newConfig.sqlTimeZone = getSqlTimeZone(); @@ -228,7 +228,7 @@ public boolean equals(final Object o) serializeComplexValues == that.serializeComplexValues && Objects.equals(metadataRefreshPeriod, that.metadataRefreshPeriod) && Objects.equals(sqlTimeZone, that.sqlTimeZone) && - useLegacyDruidExplain == that.useLegacyDruidExplain; + useNativeQueryExplain == that.useNativeQueryExplain; } @Override @@ -246,7 +246,7 @@ public int hashCode() metadataSegmentCacheEnable, metadataSegmentPollPeriod, serializeComplexValues, - useLegacyDruidExplain + useNativeQueryExplain ); } @@ -264,7 +264,7 @@ public String toString() ", metadataSegmentPollPeriod=" + metadataSegmentPollPeriod + ", sqlTimeZone=" + sqlTimeZone + ", serializeComplexValues=" + serializeComplexValues + - ", useLegacyDruidExplain=" + useLegacyDruidExplain + + ", useNativeQueryExplain=" + useNativeQueryExplain + '}'; } } diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/BaseCalciteQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/BaseCalciteQueryTest.java index 2693f05582fa..ff85165a7794 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/BaseCalciteQueryTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/BaseCalciteQueryTest.java @@ -191,12 +191,12 @@ public boolean isAuthorizeSystemTablesDirectly() } }; - public static final PlannerConfig PLANNER_CONFIG_LEGACY_DRUID_EXPLAIN_FALSE = new PlannerConfig() + public static final PlannerConfig PLANNER_CONFIG_NATIVE_QUERY_EXPLAIN = new PlannerConfig() { @Override - public boolean isUseLegacyDruidExplain() + public boolean isUseNativeQueryExplain() { - return false; + return true; } }; diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java index 47e6133e2f6d..2d199bff541c 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java @@ -4193,7 +4193,7 @@ public void testExplainCountStarOnView() throws Exception ) ); testQuery( - PLANNER_CONFIG_LEGACY_DRUID_EXPLAIN_FALSE, + PLANNER_CONFIG_NATIVE_QUERY_EXPLAIN, query, CalciteTests.REGULAR_USER_AUTH_RESULT, ImmutableList.of(), @@ -6769,7 +6769,7 @@ public void testExplainExactCountDistinctOfSemiJoinResult() throws Exception ); testQuery( - PLANNER_CONFIG_LEGACY_DRUID_EXPLAIN_FALSE, + PLANNER_CONFIG_NATIVE_QUERY_EXPLAIN, query, CalciteTests.REGULAR_USER_AUTH_RESULT, ImmutableList.of(), @@ -6782,17 +6782,17 @@ public void testExplainExactCountDistinctOfSemiJoinResult() throws Exception @Test public void testExplainSelectStarWithOverrides() throws Exception { - Map useLegacyDruidExplainContext = new HashMap<>(QUERY_CONTEXT_DEFAULT); - useLegacyDruidExplainContext.put(PlannerConfig.CTX_KEY_USE_LEGACY_DRUID_EXPLAIN, true); + Map useRegularExplainContext = new HashMap<>(QUERY_CONTEXT_DEFAULT); + useRegularExplainContext.put(PlannerConfig.CTX_KEY_USE_NATIVE_QUERY_EXPLAIN, false); - Map useRegularDruidExplainContext = new HashMap<>(QUERY_CONTEXT_DEFAULT); - useRegularDruidExplainContext.put(PlannerConfig.CTX_KEY_USE_LEGACY_DRUID_EXPLAIN, false); + Map useNativeQueryExplain = new HashMap<>(QUERY_CONTEXT_DEFAULT); + useNativeQueryExplain.put(PlannerConfig.CTX_KEY_USE_NATIVE_QUERY_EXPLAIN, true); // Skip vectorization since otherwise the "context" will change for each subtest. skipVectorize(); String legacyExplanation = "DruidQueryRel(query=[{\"queryType\":\"scan\",\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"virtualColumns\":[],\"resultFormat\":\"compactedList\",\"batchSize\":20480,\"filter\":null,\"columns\":[\"__time\",\"cnt\",\"dim1\",\"dim2\",\"dim3\",\"m1\",\"m2\",\"unique_dim1\"],\"legacy\":false,\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\",\"sqlQueryId\":\"dummy\",\"vectorize\":\"false\",\"vectorizeVirtualColumns\":\"false\"},\"descending\":false,\"granularity\":{\"type\":\"all\"}}], signature=[{__time:LONG, cnt:LONG, dim1:STRING, dim2:STRING, dim3:STRING, m1:FLOAT, m2:DOUBLE, unique_dim1:COMPLEX}])\n"; - String legacyExplanationWithContext = "DruidQueryRel(query=[{\"queryType\":\"scan\",\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"virtualColumns\":[],\"resultFormat\":\"compactedList\",\"batchSize\":20480,\"filter\":null,\"columns\":[\"__time\",\"cnt\",\"dim1\",\"dim2\",\"dim3\",\"m1\",\"m2\",\"unique_dim1\"],\"legacy\":false,\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\",\"sqlQueryId\":\"dummy\",\"useLegacyDruidExplain\":true},\"descending\":false,\"granularity\":{\"type\":\"all\"}}], signature=[{__time:LONG, cnt:LONG, dim1:STRING, dim2:STRING, dim3:STRING, m1:FLOAT, m2:DOUBLE, unique_dim1:COMPLEX}])\n"; + String legacyExplanationWithContext = "DruidQueryRel(query=[{\"queryType\":\"scan\",\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"virtualColumns\":[],\"resultFormat\":\"compactedList\",\"batchSize\":20480,\"filter\":null,\"columns\":[\"__time\",\"cnt\",\"dim1\",\"dim2\",\"dim3\",\"m1\",\"m2\",\"unique_dim1\"],\"legacy\":false,\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\",\"sqlQueryId\":\"dummy\",\"useNativeQueryExplain\":false},\"descending\":false,\"granularity\":{\"type\":\"all\"}}], signature=[{__time:LONG, cnt:LONG, dim1:STRING, dim2:STRING, dim3:STRING, m1:FLOAT, m2:DOUBLE, unique_dim1:COMPLEX}])\n"; String explanation = "[{" + "\"query\":{\"queryType\":\"scan\"," + "\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"}," @@ -6819,7 +6819,7 @@ public void testExplainSelectStarWithOverrides() throws Exception + "\"filter\":null," + "\"columns\":[\"__time\",\"cnt\",\"dim1\",\"dim2\",\"dim3\",\"m1\",\"m2\",\"unique_dim1\"]," + "\"legacy\":false," - + "\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\",\"sqlQueryId\":\"dummy\",\"useLegacyDruidExplain\":false,\"vectorize\":\"false\",\"vectorizeVirtualColumns\":\"false\"}," + + "\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\",\"sqlQueryId\":\"dummy\",\"useNativeQueryExplain\":true,\"vectorize\":\"false\",\"vectorizeVirtualColumns\":\"false\"}," + "\"descending\":false," + "\"granularity\":{\"type\":\"all\"}}," + "\"signature\":[{\"name\":\"__time\",\"type\":\"LONG\"},{\"name\":\"cnt\",\"type\":\"LONG\"},{\"name\":\"dim1\",\"type\":\"STRING\"},{\"name\":\"dim2\",\"type\":\"STRING\"},{\"name\":\"dim3\",\"type\":\"STRING\"},{\"name\":\"m1\",\"type\":\"FLOAT\"},{\"name\":\"m2\",\"type\":\"DOUBLE\"},{\"name\":\"unique_dim1\",\"type\":\"COMPLEX\"}]" @@ -6830,27 +6830,27 @@ public void testExplainSelectStarWithOverrides() throws Exception // Test when default config and no overrides testQuery(sql, ImmutableList.of(), ImmutableList.of(new Object[]{legacyExplanation, resources})); - // Test when default config and useLegacyDruidExplain is overridden in the context + // Test when default config and useNativeQueryExplain is overridden in the context testQuery( sql, - useRegularDruidExplainContext, + useNativeQueryExplain, ImmutableList.of(), ImmutableList.of(new Object[]{explanationWithContext, resources}) ); - // Test when useLegacyDruidExplain disabled by default and no overrides + // Test when useNativeQueryExplain enabled by default and no overrides testQuery( - PLANNER_CONFIG_LEGACY_DRUID_EXPLAIN_FALSE, + PLANNER_CONFIG_NATIVE_QUERY_EXPLAIN, sql, CalciteTests.REGULAR_USER_AUTH_RESULT, ImmutableList.of(), ImmutableList.of(new Object[]{explanation, resources}) ); - // Test when useLegacyDruidExplain disabled by default but is overriden in the context + // Test when useNativeQueryExplain enabled by default but is overriden in the context testQuery( - PLANNER_CONFIG_LEGACY_DRUID_EXPLAIN_FALSE, - useLegacyDruidExplainContext, + PLANNER_CONFIG_NATIVE_QUERY_EXPLAIN, + useRegularExplainContext, sql, CalciteTests.REGULAR_USER_AUTH_RESULT, ImmutableList.of(), diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteSelectQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteSelectQueryTest.java index 905ed75ee07f..a4aaab8041c9 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteSelectQueryTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteSelectQueryTest.java @@ -439,7 +439,7 @@ public void testExplainSelectConstantExpression() throws Exception ) ); testQuery( - PLANNER_CONFIG_LEGACY_DRUID_EXPLAIN_FALSE, + PLANNER_CONFIG_NATIVE_QUERY_EXPLAIN, query, CalciteTests.REGULAR_USER_AUTH_RESULT, ImmutableList.of(), @@ -1138,7 +1138,7 @@ public void testExplainSelectStar() throws Exception ) ); testQuery( - PLANNER_CONFIG_LEGACY_DRUID_EXPLAIN_FALSE, + PLANNER_CONFIG_NATIVE_QUERY_EXPLAIN, query, CalciteTests.REGULAR_USER_AUTH_RESULT, ImmutableList.of(),