diff --git a/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/quantiles/sql/DoublesSketchSqlAggregatorTest.java b/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/quantiles/sql/DoublesSketchSqlAggregatorTest.java index ba1eb347fe71..9cc90617347d 100644 --- a/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/quantiles/sql/DoublesSketchSqlAggregatorTest.java +++ b/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/quantiles/sql/DoublesSketchSqlAggregatorTest.java @@ -27,6 +27,7 @@ import org.apache.druid.java.util.common.Intervals; import org.apache.druid.java.util.common.granularity.Granularities; import org.apache.druid.query.Druids; +import org.apache.druid.query.JoinAlgorithm; import org.apache.druid.query.JoinDataSource; import org.apache.druid.query.QueryContexts; import org.apache.druid.query.QueryDataSource; @@ -390,7 +391,8 @@ public void testSubqueryWithNestedGroupBy() JoinType.INNER, null, TestExprMacroTable.INSTANCE, - null + null, + JoinAlgorithm.BROADCAST ) ) .intervals(querySegmentSpec(Intervals.ETERNITY)) diff --git a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/indexing/error/BroadcastTablesTooLargeFault.java b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/indexing/error/BroadcastTablesTooLargeFault.java index 6912fefa6b65..31337cbcfe69 100644 --- a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/indexing/error/BroadcastTablesTooLargeFault.java +++ b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/indexing/error/BroadcastTablesTooLargeFault.java @@ -24,7 +24,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonTypeName; import org.apache.druid.java.util.common.StringUtils; -import org.apache.druid.sql.calcite.planner.JoinAlgorithm; +import org.apache.druid.query.JoinAlgorithm; import org.apache.druid.sql.calcite.planner.PlannerContext; import javax.annotation.Nullable; diff --git a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/querykit/BroadcastJoinSegmentMapFnProcessor.java b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/querykit/BroadcastJoinSegmentMapFnProcessor.java index cbb79c45702b..b47a450aa000 100644 --- a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/querykit/BroadcastJoinSegmentMapFnProcessor.java +++ b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/querykit/BroadcastJoinSegmentMapFnProcessor.java @@ -38,11 +38,11 @@ import org.apache.druid.msq.input.ReadableInput; import org.apache.druid.query.DataSource; import org.apache.druid.query.InlineDataSource; +import org.apache.druid.query.JoinAlgorithm; import org.apache.druid.query.Query; import org.apache.druid.segment.ColumnValueSelector; import org.apache.druid.segment.Cursor; import org.apache.druid.segment.SegmentReference; -import org.apache.druid.sql.calcite.planner.JoinAlgorithm; import org.apache.druid.sql.calcite.planner.PlannerContext; import java.io.IOException; diff --git a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/querykit/DataSourcePlan.java b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/querykit/DataSourcePlan.java index 6b8fbcb6f352..d20adc1e0062 100644 --- a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/querykit/DataSourcePlan.java +++ b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/querykit/DataSourcePlan.java @@ -47,6 +47,7 @@ import org.apache.druid.query.DataSource; import org.apache.druid.query.FilteredDataSource; import org.apache.druid.query.InlineDataSource; +import org.apache.druid.query.JoinAlgorithm; import org.apache.druid.query.JoinDataSource; import org.apache.druid.query.LookupDataSource; import org.apache.druid.query.QueryContext; @@ -65,8 +66,6 @@ import org.apache.druid.segment.join.JoinConditionAnalysis; import org.apache.druid.sql.calcite.external.ExternalDataSource; import org.apache.druid.sql.calcite.parser.DruidSqlInsert; -import org.apache.druid.sql.calcite.planner.JoinAlgorithm; -import org.apache.druid.sql.calcite.planner.PlannerContext; import org.joda.time.Interval; import javax.annotation.Nullable; @@ -212,10 +211,11 @@ public static DataSourcePlan forDataSource( broadcast ); } else if (dataSource instanceof JoinDataSource) { - final JoinAlgorithm preferredJoinAlgorithm = PlannerContext.getJoinAlgorithm(queryContext); + JoinDataSource joinDataSource = (JoinDataSource) dataSource; + final JoinAlgorithm preferredJoinAlgorithm = joinDataSource.getJoinAlgorithm(); final JoinAlgorithm deducedJoinAlgorithm = deduceJoinAlgorithm( preferredJoinAlgorithm, - ((JoinDataSource) dataSource) + joinDataSource ); switch (deducedJoinAlgorithm) { @@ -223,7 +223,7 @@ public static DataSourcePlan forDataSource( return forBroadcastHashJoin( queryKitSpec, queryContext, - (JoinDataSource) dataSource, + joinDataSource, querySegmentSpec, filter, filterFields, @@ -234,7 +234,7 @@ public static DataSourcePlan forDataSource( case SORT_MERGE: return forSortMergeJoin( queryKitSpec, - (JoinDataSource) dataSource, + joinDataSource, querySegmentSpec, minStageNumber, broadcast @@ -615,7 +615,8 @@ private static DataSourcePlan forBroadcastHashJoin( clause.getJoinType(), // First JoinDataSource (i == 0) involves the base table, so we need to propagate the base table filter. i == 0 ? analysis.getJoinBaseTableFilter().orElse(null) : null, - dataSource.getJoinableFactoryWrapper() + dataSource.getJoinableFactoryWrapper(), + clause.getJoinAlgorithm() ); inputSpecs.addAll(clausePlan.getInputSpecs()); clausePlan.getBroadcastInputs().intStream().forEach(n -> broadcastInputs.add(n + shift)); diff --git a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/exec/MSQSelectTest.java b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/exec/MSQSelectTest.java index 09dcbfcda144..6b904e5f1495 100644 --- a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/exec/MSQSelectTest.java +++ b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/exec/MSQSelectTest.java @@ -50,6 +50,7 @@ import org.apache.druid.msq.test.MSQTestBase; import org.apache.druid.msq.util.MultiStageQueryContext; import org.apache.druid.query.InlineDataSource; +import org.apache.druid.query.JoinAlgorithm; import org.apache.druid.query.LookupDataSource; import org.apache.druid.query.OrderBy; import org.apache.druid.query.Query; @@ -81,7 +82,6 @@ import org.apache.druid.sql.calcite.filtration.Filtration; import org.apache.druid.sql.calcite.planner.ColumnMapping; import org.apache.druid.sql.calcite.planner.ColumnMappings; -import org.apache.druid.sql.calcite.planner.JoinAlgorithm; import org.apache.druid.sql.calcite.planner.PlannerContext; import org.apache.druid.sql.calcite.util.CalciteTests; import org.hamcrest.CoreMatchers; @@ -1043,7 +1043,9 @@ private void testJoin(String contextName, Map context, final Joi DruidExpression.ofColumn(ColumnType.FLOAT, "m1"), DruidExpression.ofColumn(ColumnType.FLOAT, "j0.m1") ), - JoinType.INNER + JoinType.INNER, + null, + joinAlgorithm ) ) .setInterval(querySegmentSpec(Filtration.eternity())) @@ -2523,7 +2525,9 @@ public void testJoinUsesDifferentAlgorithm(String contextName, Map_0", + "input" : [ { + "type" : "table", + "dataSource" : "wikipedia", + "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ], + "filter" : { + "type" : "equals", + "column" : "cityName", + "matchValueType" : "STRING", + "matchValue" : "New York" + }, + "filterFields" : [ "cityName" ] + } ], + "processor" : { + "type" : "scan", + "query" : { + "queryType" : "scan", + "dataSource" : { + "type" : "inputNumber", + "inputNumber" : 0 + }, + "intervals" : { + "type" : "intervals", + "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ] + }, + "resultFormat" : "compactedList", + "filter" : { + "type" : "equals", + "column" : "cityName", + "matchValueType" : "STRING", + "matchValue" : "New York" + }, + "columns" : [ "cityName" ], + "context" : { + "scanSignature" : "[{\"name\":\"cityName\",\"type\":\"STRING\"}]", + "sqlInsertSegmentGranularity" : null, + "sqlQueryId" : __SQL_QUERY_ID__ + "sqlStringifyArrays" : false + }, + "columnTypes" : [ "STRING" ], + "granularity" : { + "type" : "all" + }, + "legacy" : false + } + }, + "signature" : [ { + "name" : "__boost", + "type" : "LONG" + }, { + "name" : "cityName", + "type" : "STRING" + } ], + "shuffleSpec" : { + "type" : "maxCount", + "clusterBy" : { + "columns" : [ { + "columnName" : "__boost", + "order" : "ASCENDING" + } ] + }, + "partitions" : 1 + }, + "maxWorkerCount" : 1 + }, + "phase" : "FINISHED", + "workerCount" : 1, + "partitionCount" : 1, + "shuffle" : "globalSort", + "output" : "localStorage", + "startTime" : __TIMESTAMP__ + "duration" : __DURATION__ + "sort" : true +}, { + "stageNumber" : 1, + "definition" : { + "id" : "_1", + "input" : [ { + "type" : "table", + "dataSource" : "wikipedia", + "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ] + } ], + "processor" : { + "type" : "scan", + "query" : { + "queryType" : "scan", + "dataSource" : { + "type" : "inputNumber", + "inputNumber" : 0 + }, + "intervals" : { + "type" : "intervals", + "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ] + }, + "resultFormat" : "compactedList", + "columns" : [ "cityName", "countryName" ], + "context" : { + "scanSignature" : "[{\"name\":\"cityName\",\"type\":\"STRING\"},{\"name\":\"countryName\",\"type\":\"STRING\"}]", + "sqlInsertSegmentGranularity" : null, + "sqlQueryId" : __SQL_QUERY_ID__ + "sqlStringifyArrays" : false + }, + "columnTypes" : [ "STRING", "STRING" ], + "granularity" : { + "type" : "all" + }, + "legacy" : false + } + }, + "signature" : [ { + "name" : "__boost", + "type" : "LONG" + }, { + "name" : "cityName", + "type" : "STRING" + }, { + "name" : "countryName", + "type" : "STRING" + } ], + "shuffleSpec" : { + "type" : "maxCount", + "clusterBy" : { + "columns" : [ { + "columnName" : "__boost", + "order" : "ASCENDING" + } ] + }, + "partitions" : 1 + }, + "maxWorkerCount" : 1 + }, + "phase" : "FINISHED", + "workerCount" : 1, + "partitionCount" : 1, + "shuffle" : "globalSort", + "output" : "localStorage", + "startTime" : __TIMESTAMP__ + "duration" : __DURATION__ + "sort" : true +}, { + "stageNumber" : 2, + "definition" : { + "id" : "_2", + "input" : [ { + "type" : "stage", + "stage" : 0 + }, { + "type" : "stage", + "stage" : 1 + } ], + "broadcast" : [ 1 ], + "processor" : { + "type" : "scan", + "query" : { + "queryType" : "scan", + "dataSource" : { + "type" : "join", + "left" : { + "type" : "inputNumber", + "inputNumber" : 0 + }, + "right" : { + "type" : "inputNumber", + "inputNumber" : 1 + }, + "rightPrefix" : "j0.", + "condition" : "(\"cityName\" == \"j0.cityName\")", + "joinType" : "INNER" + }, + "intervals" : { + "type" : "intervals", + "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ] + }, + "virtualColumns" : [ { + "type" : "expression", + "name" : "v0", + "expression" : "'New York'", + "outputType" : "STRING" + } ], + "resultFormat" : "compactedList", + "columns" : [ "v0", "j0.countryName" ], + "context" : { + "__user" : null, + "finalize" : true, + "maxParseExceptions" : 0, + "scanSignature" : "[{\"name\":\"v0\",\"type\":\"STRING\"},{\"name\":\"j0.countryName\",\"type\":\"STRING\"}]", + "sqlQueryId" : __SQL_QUERY_ID__ + "sqlStringifyArrays" : false, + "windowFunctionOperatorTransformation" : true + }, + "columnTypes" : [ "STRING", "STRING" ], + "granularity" : { + "type" : "all" + }, + "legacy" : false + } + }, + "signature" : [ { + "name" : "__boost", + "type" : "LONG" + }, { + "name" : "v0", + "type" : "STRING" + }, { + "name" : "j0.countryName", + "type" : "STRING" + } ], + "shuffleSpec" : { + "type" : "maxCount", + "clusterBy" : { + "columns" : [ { + "columnName" : "__boost", + "order" : "ASCENDING" + } ] + }, + "partitions" : 1 + }, + "maxWorkerCount" : 1 + }, + "phase" : "FINISHED", + "workerCount" : 1, + "partitionCount" : 1, + "shuffle" : "globalSort", + "output" : "localStorage", + "startTime" : __TIMESTAMP__ + "duration" : __DURATION__ + "sort" : true +} ] +!msqPlan + +select /*+ broadcast */ w1.cityName, w2.countryName +from (select cityName, countryName from wikipedia where cityName='New York') w1 +JOIN wikipedia w2 ON w1.cityName = w2.cityName +where w1.cityName='New York'; + +LogicalJoin:[[broadcast inheritPath:[0, 0]]] + +!hints + +[ { + "stageNumber" : 0, + "definition" : { + "id" : "_0", + "input" : [ { + "type" : "table", + "dataSource" : "wikipedia", + "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ], + "filter" : { + "type" : "equals", + "column" : "cityName", + "matchValueType" : "STRING", + "matchValue" : "New York" + }, + "filterFields" : [ "cityName" ] + } ], + "processor" : { + "type" : "scan", + "query" : { + "queryType" : "scan", + "dataSource" : { + "type" : "inputNumber", + "inputNumber" : 0 + }, + "intervals" : { + "type" : "intervals", + "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ] + }, + "resultFormat" : "compactedList", + "filter" : { + "type" : "equals", + "column" : "cityName", + "matchValueType" : "STRING", + "matchValue" : "New York" + }, + "columns" : [ "cityName" ], + "context" : { + "scanSignature" : "[{\"name\":\"cityName\",\"type\":\"STRING\"}]", + "sqlInsertSegmentGranularity" : null, + "sqlQueryId" : __SQL_QUERY_ID__ + "sqlStringifyArrays" : false + }, + "columnTypes" : [ "STRING" ], + "granularity" : { + "type" : "all" + }, + "legacy" : false + } + }, + "signature" : [ { + "name" : "__boost", + "type" : "LONG" + }, { + "name" : "cityName", + "type" : "STRING" + } ], + "shuffleSpec" : { + "type" : "maxCount", + "clusterBy" : { + "columns" : [ { + "columnName" : "__boost", + "order" : "ASCENDING" + } ] + }, + "partitions" : 1 + }, + "maxWorkerCount" : 1 + }, + "phase" : "FINISHED", + "workerCount" : 1, + "partitionCount" : 1, + "shuffle" : "globalSort", + "output" : "localStorage", + "startTime" : __TIMESTAMP__ + "duration" : __DURATION__ + "sort" : true +}, { + "stageNumber" : 1, + "definition" : { + "id" : "_1", + "input" : [ { + "type" : "table", + "dataSource" : "wikipedia", + "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ] + } ], + "processor" : { + "type" : "scan", + "query" : { + "queryType" : "scan", + "dataSource" : { + "type" : "inputNumber", + "inputNumber" : 0 + }, + "intervals" : { + "type" : "intervals", + "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ] + }, + "resultFormat" : "compactedList", + "columns" : [ "cityName", "countryName" ], + "context" : { + "scanSignature" : "[{\"name\":\"cityName\",\"type\":\"STRING\"},{\"name\":\"countryName\",\"type\":\"STRING\"}]", + "sqlInsertSegmentGranularity" : null, + "sqlQueryId" : __SQL_QUERY_ID__ + "sqlStringifyArrays" : false + }, + "columnTypes" : [ "STRING", "STRING" ], + "granularity" : { + "type" : "all" + }, + "legacy" : false + } + }, + "signature" : [ { + "name" : "__boost", + "type" : "LONG" + }, { + "name" : "cityName", + "type" : "STRING" + }, { + "name" : "countryName", + "type" : "STRING" + } ], + "shuffleSpec" : { + "type" : "maxCount", + "clusterBy" : { + "columns" : [ { + "columnName" : "__boost", + "order" : "ASCENDING" + } ] + }, + "partitions" : 1 + }, + "maxWorkerCount" : 1 + }, + "phase" : "FINISHED", + "workerCount" : 1, + "partitionCount" : 1, + "shuffle" : "globalSort", + "output" : "localStorage", + "startTime" : __TIMESTAMP__ + "duration" : __DURATION__ + "sort" : true +}, { + "stageNumber" : 2, + "definition" : { + "id" : "_2", + "input" : [ { + "type" : "stage", + "stage" : 0 + }, { + "type" : "stage", + "stage" : 1 + } ], + "broadcast" : [ 1 ], + "processor" : { + "type" : "scan", + "query" : { + "queryType" : "scan", + "dataSource" : { + "type" : "join", + "left" : { + "type" : "inputNumber", + "inputNumber" : 0 + }, + "right" : { + "type" : "inputNumber", + "inputNumber" : 1 + }, + "rightPrefix" : "j0.", + "condition" : "(\"cityName\" == \"j0.cityName\")", + "joinType" : "INNER" + }, + "intervals" : { + "type" : "intervals", + "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ] + }, + "virtualColumns" : [ { + "type" : "expression", + "name" : "v0", + "expression" : "'New York'", + "outputType" : "STRING" + } ], + "resultFormat" : "compactedList", + "columns" : [ "v0", "j0.countryName" ], + "context" : { + "__user" : null, + "finalize" : true, + "maxParseExceptions" : 0, + "scanSignature" : "[{\"name\":\"v0\",\"type\":\"STRING\"},{\"name\":\"j0.countryName\",\"type\":\"STRING\"}]", + "sqlQueryId" : __SQL_QUERY_ID__ + "sqlStringifyArrays" : false, + "windowFunctionOperatorTransformation" : true + }, + "columnTypes" : [ "STRING", "STRING" ], + "granularity" : { + "type" : "all" + }, + "legacy" : false + } + }, + "signature" : [ { + "name" : "__boost", + "type" : "LONG" + }, { + "name" : "v0", + "type" : "STRING" + }, { + "name" : "j0.countryName", + "type" : "STRING" + } ], + "shuffleSpec" : { + "type" : "maxCount", + "clusterBy" : { + "columns" : [ { + "columnName" : "__boost", + "order" : "ASCENDING" + } ] + }, + "partitions" : 1 + }, + "maxWorkerCount" : 1 + }, + "phase" : "FINISHED", + "workerCount" : 1, + "partitionCount" : 1, + "shuffle" : "globalSort", + "output" : "localStorage", + "startTime" : __TIMESTAMP__ + "duration" : __DURATION__ + "sort" : true +} ] +!msqPlan + +select /*+ sort_merge */ w1.cityName, w2.countryName +from (select cityName, countryName from wikipedia where cityName='New York') w1 +JOIN wikipedia w2 ON w1.cityName = w2.cityName +where w1.cityName='New York'; + + +LogicalJoin:[[sort_merge inheritPath:[0, 0]]] + +!hints + +[ { + "stageNumber" : 0, + "definition" : { + "id" : "_0", + "input" : [ { + "type" : "table", + "dataSource" : "wikipedia", + "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ], + "filter" : { + "type" : "equals", + "column" : "cityName", + "matchValueType" : "STRING", + "matchValue" : "New York" + }, + "filterFields" : [ "cityName" ] + } ], + "processor" : { + "type" : "scan", + "query" : { + "queryType" : "scan", + "dataSource" : { + "type" : "inputNumber", + "inputNumber" : 0 + }, + "intervals" : { + "type" : "intervals", + "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ] + }, + "resultFormat" : "compactedList", + "filter" : { + "type" : "equals", + "column" : "cityName", + "matchValueType" : "STRING", + "matchValue" : "New York" + }, + "columns" : [ "cityName" ], + "context" : { + "scanSignature" : "[{\"name\":\"cityName\",\"type\":\"STRING\"}]", + "sqlInsertSegmentGranularity" : null, + "sqlQueryId" : __SQL_QUERY_ID__ + "sqlStringifyArrays" : false + }, + "columnTypes" : [ "STRING" ], + "granularity" : { + "type" : "all" + }, + "legacy" : false + } + }, + "signature" : [ { + "name" : "cityName", + "type" : "STRING" + }, { + "name" : "__boost", + "type" : "LONG" + } ], + "shuffleSpec" : { + "type" : "hash", + "clusterBy" : { + "columns" : [ { + "columnName" : "cityName", + "order" : "ASCENDING" + } ] + }, + "partitions" : 1 + }, + "maxWorkerCount" : 1 + }, + "phase" : "FINISHED", + "workerCount" : 1, + "partitionCount" : 1, + "shuffle" : "hashLocalSort", + "output" : "localStorage", + "startTime" : __TIMESTAMP__ + "duration" : __DURATION__ + "sort" : true +}, { + "stageNumber" : 1, + "definition" : { + "id" : "_1", + "input" : [ { + "type" : "table", + "dataSource" : "wikipedia", + "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ] + } ], + "processor" : { + "type" : "scan", + "query" : { + "queryType" : "scan", + "dataSource" : { + "type" : "inputNumber", + "inputNumber" : 0 + }, + "intervals" : { + "type" : "intervals", + "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ] + }, + "resultFormat" : "compactedList", + "columns" : [ "cityName", "countryName" ], + "context" : { + "scanSignature" : "[{\"name\":\"cityName\",\"type\":\"STRING\"},{\"name\":\"countryName\",\"type\":\"STRING\"}]", + "sqlInsertSegmentGranularity" : null, + "sqlQueryId" : __SQL_QUERY_ID__ + "sqlStringifyArrays" : false + }, + "columnTypes" : [ "STRING", "STRING" ], + "granularity" : { + "type" : "all" + }, + "legacy" : false + } + }, + "signature" : [ { + "name" : "cityName", + "type" : "STRING" + }, { + "name" : "__boost", + "type" : "LONG" + }, { + "name" : "countryName", + "type" : "STRING" + } ], + "shuffleSpec" : { + "type" : "hash", + "clusterBy" : { + "columns" : [ { + "columnName" : "cityName", + "order" : "ASCENDING" + } ] + }, + "partitions" : 1 + }, + "maxWorkerCount" : 1 + }, + "phase" : "FINISHED", + "workerCount" : 1, + "partitionCount" : 1, + "shuffle" : "hashLocalSort", + "output" : "localStorage", + "startTime" : __TIMESTAMP__ + "duration" : __DURATION__ + "sort" : true +}, { + "stageNumber" : 2, + "definition" : { + "id" : "_2", + "input" : [ { + "type" : "stage", + "stage" : 0 + }, { + "type" : "stage", + "stage" : 1 + } ], + "processor" : { + "type" : "sortMergeJoin", + "rightPrefix" : "j0.", + "condition" : "(\"cityName\" == \"j0.cityName\")", + "joinType" : "INNER" + }, + "signature" : [ { + "name" : "cityName", + "type" : "STRING" + }, { + "name" : "__boost", + "type" : "LONG" + }, { + "name" : "j0.cityName", + "type" : "STRING" + }, { + "name" : "j0.__boost", + "type" : "LONG" + }, { + "name" : "j0.countryName", + "type" : "STRING" + } ], + "maxWorkerCount" : 1 + }, + "phase" : "FINISHED", + "workerCount" : 1, + "partitionCount" : 1, + "output" : "localStorage", + "startTime" : __TIMESTAMP__ + "duration" : __DURATION__ +}, { + "stageNumber" : 3, + "definition" : { + "id" : "_3", + "input" : [ { + "type" : "stage", + "stage" : 2 + } ], + "processor" : { + "type" : "scan", + "query" : { + "queryType" : "scan", + "dataSource" : { + "type" : "inputNumber", + "inputNumber" : 0 + }, + "intervals" : { + "type" : "intervals", + "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ] + }, + "virtualColumns" : [ { + "type" : "expression", + "name" : "v0", + "expression" : "'New York'", + "outputType" : "STRING" + } ], + "resultFormat" : "compactedList", + "columns" : [ "v0", "j0.countryName" ], + "context" : { + "__user" : null, + "finalize" : true, + "maxParseExceptions" : 0, + "scanSignature" : "[{\"name\":\"v0\",\"type\":\"STRING\"},{\"name\":\"j0.countryName\",\"type\":\"STRING\"}]", + "sqlQueryId" : __SQL_QUERY_ID__ + "sqlStringifyArrays" : false, + "windowFunctionOperatorTransformation" : true + }, + "columnTypes" : [ "STRING", "STRING" ], + "granularity" : { + "type" : "all" + }, + "legacy" : false + } + }, + "signature" : [ { + "name" : "__boost", + "type" : "LONG" + }, { + "name" : "v0", + "type" : "STRING" + }, { + "name" : "j0.countryName", + "type" : "STRING" + } ], + "shuffleSpec" : { + "type" : "maxCount", + "clusterBy" : { + "columns" : [ { + "columnName" : "__boost", + "order" : "ASCENDING" + } ] + }, + "partitions" : 1 + }, + "maxWorkerCount" : 1 + }, + "phase" : "FINISHED", + "workerCount" : 1, + "partitionCount" : 1, + "shuffle" : "globalSort", + "output" : "localStorage", + "startTime" : __TIMESTAMP__ + "duration" : __DURATION__ + "sort" : true +} ] +!msqPlan diff --git a/extensions-core/multi-stage-query/src/test/quidem/org.apache.druid.msq.quidem.MSQQuidemTest/msqNestedJoinHint.iq b/extensions-core/multi-stage-query/src/test/quidem/org.apache.druid.msq.quidem.MSQQuidemTest/msqNestedJoinHint.iq new file mode 100644 index 000000000000..acf6240d5467 --- /dev/null +++ b/extensions-core/multi-stage-query/src/test/quidem/org.apache.druid.msq.quidem.MSQQuidemTest/msqNestedJoinHint.iq @@ -0,0 +1,913 @@ +!use druidtest://?componentSupplier=DrillWindowQueryMSQComponentSupplier +!set outputformat mysql + +select w1.cityName, w2.countryName +from +( + select w3.cityName AS cityName, w4.countryName AS countryName from wikipedia w3 LEFT JOIN wikipedia w4 ON w3.regionName = w4.regionName +) w1 +JOIN wikipedia w2 ON w1.cityName = w2.cityName +where w1.cityName='New York'; + +[ { + "stageNumber" : 0, + "definition" : { + "id" : "_0", + "input" : [ { + "type" : "table", + "dataSource" : "wikipedia", + "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ] + } ], + "processor" : { + "type" : "scan", + "query" : { + "queryType" : "scan", + "dataSource" : { + "type" : "inputNumber", + "inputNumber" : 0 + }, + "intervals" : { + "type" : "intervals", + "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ] + }, + "resultFormat" : "compactedList", + "columns" : [ "regionName" ], + "context" : { + "scanSignature" : "[{\"name\":\"regionName\",\"type\":\"STRING\"}]", + "sqlInsertSegmentGranularity" : null, + "sqlQueryId" : __SQL_QUERY_ID__ + "sqlStringifyArrays" : false + }, + "columnTypes" : [ "STRING" ], + "granularity" : { + "type" : "all" + }, + "legacy" : false + } + }, + "signature" : [ { + "name" : "__boost", + "type" : "LONG" + }, { + "name" : "regionName", + "type" : "STRING" + } ], + "shuffleSpec" : { + "type" : "maxCount", + "clusterBy" : { + "columns" : [ { + "columnName" : "__boost", + "order" : "ASCENDING" + } ] + }, + "partitions" : 1 + }, + "maxWorkerCount" : 1 + }, + "phase" : "FINISHED", + "workerCount" : 1, + "partitionCount" : 1, + "shuffle" : "globalSort", + "output" : "localStorage", + "startTime" : __TIMESTAMP__ + "duration" : __DURATION__ + "sort" : true +}, { + "stageNumber" : 1, + "definition" : { + "id" : "_1", + "input" : [ { + "type" : "table", + "dataSource" : "wikipedia", + "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ] + } ], + "processor" : { + "type" : "scan", + "query" : { + "queryType" : "scan", + "dataSource" : { + "type" : "inputNumber", + "inputNumber" : 0 + }, + "intervals" : { + "type" : "intervals", + "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ] + }, + "resultFormat" : "compactedList", + "columns" : [ "cityName", "countryName" ], + "context" : { + "scanSignature" : "[{\"name\":\"cityName\",\"type\":\"STRING\"},{\"name\":\"countryName\",\"type\":\"STRING\"}]", + "sqlInsertSegmentGranularity" : null, + "sqlQueryId" : __SQL_QUERY_ID__ + "sqlStringifyArrays" : false + }, + "columnTypes" : [ "STRING", "STRING" ], + "granularity" : { + "type" : "all" + }, + "legacy" : false + } + }, + "signature" : [ { + "name" : "__boost", + "type" : "LONG" + }, { + "name" : "cityName", + "type" : "STRING" + }, { + "name" : "countryName", + "type" : "STRING" + } ], + "shuffleSpec" : { + "type" : "maxCount", + "clusterBy" : { + "columns" : [ { + "columnName" : "__boost", + "order" : "ASCENDING" + } ] + }, + "partitions" : 1 + }, + "maxWorkerCount" : 1 + }, + "phase" : "FINISHED", + "workerCount" : 1, + "partitionCount" : 1, + "shuffle" : "globalSort", + "output" : "localStorage", + "startTime" : __TIMESTAMP__ + "duration" : __DURATION__ + "sort" : true +}, { + "stageNumber" : 2, + "definition" : { + "id" : "_2", + "input" : [ { + "type" : "table", + "dataSource" : "wikipedia", + "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ], + "filter" : { + "type" : "equals", + "column" : "cityName", + "matchValueType" : "STRING", + "matchValue" : "New York" + }, + "filterFields" : [ "cityName" ] + }, { + "type" : "stage", + "stage" : 0 + }, { + "type" : "stage", + "stage" : 1 + } ], + "broadcast" : [ 1, 2 ], + "processor" : { + "type" : "scan", + "query" : { + "queryType" : "scan", + "dataSource" : { + "type" : "join", + "left" : { + "type" : "join", + "left" : { + "type" : "inputNumber", + "inputNumber" : 0 + }, + "right" : { + "type" : "inputNumber", + "inputNumber" : 1 + }, + "rightPrefix" : "j0.", + "condition" : "(\"regionName\" == \"j0.regionName\")", + "joinType" : "LEFT" + }, + "right" : { + "type" : "inputNumber", + "inputNumber" : 2 + }, + "rightPrefix" : "_j0.", + "condition" : "(\"cityName\" == \"_j0.cityName\")", + "joinType" : "INNER" + }, + "intervals" : { + "type" : "intervals", + "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ] + }, + "virtualColumns" : [ { + "type" : "expression", + "name" : "v0", + "expression" : "'New York'", + "outputType" : "STRING" + } ], + "resultFormat" : "compactedList", + "filter" : { + "type" : "equals", + "column" : "cityName", + "matchValueType" : "STRING", + "matchValue" : "New York" + }, + "columns" : [ "v0", "_j0.countryName" ], + "context" : { + "__user" : null, + "finalize" : true, + "maxParseExceptions" : 0, + "scanSignature" : "[{\"name\":\"v0\",\"type\":\"STRING\"},{\"name\":\"_j0.countryName\",\"type\":\"STRING\"}]", + "sqlQueryId" : __SQL_QUERY_ID__ + "sqlStringifyArrays" : false, + "windowFunctionOperatorTransformation" : true + }, + "columnTypes" : [ "STRING", "STRING" ], + "granularity" : { + "type" : "all" + }, + "legacy" : false + } + }, + "signature" : [ { + "name" : "__boost", + "type" : "LONG" + }, { + "name" : "v0", + "type" : "STRING" + }, { + "name" : "_j0.countryName", + "type" : "STRING" + } ], + "shuffleSpec" : { + "type" : "maxCount", + "clusterBy" : { + "columns" : [ { + "columnName" : "__boost", + "order" : "ASCENDING" + } ] + }, + "partitions" : 1 + }, + "maxWorkerCount" : 1 + }, + "phase" : "FINISHED", + "workerCount" : 1, + "partitionCount" : 1, + "shuffle" : "globalSort", + "output" : "localStorage", + "startTime" : __TIMESTAMP__ + "duration" : __DURATION__ + "sort" : true +} ] +!msqPlan + +select w1.cityName, w2.countryName +from +( + select /*+ sort_merge */ w3.cityName AS cityName, w4.countryName AS countryName from wikipedia w3 LEFT JOIN wikipedia w4 ON w3.regionName = w4.regionName +) w1 +JOIN wikipedia w2 ON w1.cityName = w2.cityName +where w1.cityName='New York'; + +LogicalJoin:[[sort_merge inheritPath:[0]]] + +!hints + +[ { + "stageNumber" : 0, + "definition" : { + "id" : "_0", + "input" : [ { + "type" : "table", + "dataSource" : "wikipedia", + "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ] + } ], + "processor" : { + "type" : "scan", + "query" : { + "queryType" : "scan", + "dataSource" : { + "type" : "inputNumber", + "inputNumber" : 0 + }, + "intervals" : { + "type" : "intervals", + "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ] + }, + "resultFormat" : "compactedList", + "columns" : [ "cityName", "regionName" ], + "context" : { + "scanSignature" : "[{\"name\":\"cityName\",\"type\":\"STRING\"},{\"name\":\"regionName\",\"type\":\"STRING\"}]", + "sqlInsertSegmentGranularity" : null, + "sqlQueryId" : __SQL_QUERY_ID__ + "sqlStringifyArrays" : false + }, + "columnTypes" : [ "STRING", "STRING" ], + "granularity" : { + "type" : "all" + }, + "legacy" : false + } + }, + "signature" : [ { + "name" : "__boost", + "type" : "LONG" + }, { + "name" : "cityName", + "type" : "STRING" + }, { + "name" : "regionName", + "type" : "STRING" + } ], + "shuffleSpec" : { + "type" : "maxCount", + "clusterBy" : { + "columns" : [ { + "columnName" : "__boost", + "order" : "ASCENDING" + } ] + }, + "partitions" : 1 + }, + "maxWorkerCount" : 1 + }, + "phase" : "FINISHED", + "workerCount" : 1, + "partitionCount" : 1, + "shuffle" : "globalSort", + "output" : "localStorage", + "startTime" : __TIMESTAMP__ + "duration" : __DURATION__ + "sort" : true +}, { + "stageNumber" : 1, + "definition" : { + "id" : "_1", + "input" : [ { + "type" : "table", + "dataSource" : "wikipedia", + "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ] + } ], + "processor" : { + "type" : "scan", + "query" : { + "queryType" : "scan", + "dataSource" : { + "type" : "inputNumber", + "inputNumber" : 0 + }, + "intervals" : { + "type" : "intervals", + "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ] + }, + "resultFormat" : "compactedList", + "columns" : [ "regionName" ], + "context" : { + "scanSignature" : "[{\"name\":\"regionName\",\"type\":\"STRING\"}]", + "sqlInsertSegmentGranularity" : null, + "sqlQueryId" : __SQL_QUERY_ID__ + "sqlStringifyArrays" : false + }, + "columnTypes" : [ "STRING" ], + "granularity" : { + "type" : "all" + }, + "legacy" : false + } + }, + "signature" : [ { + "name" : "__boost", + "type" : "LONG" + }, { + "name" : "regionName", + "type" : "STRING" + } ], + "shuffleSpec" : { + "type" : "maxCount", + "clusterBy" : { + "columns" : [ { + "columnName" : "__boost", + "order" : "ASCENDING" + } ] + }, + "partitions" : 1 + }, + "maxWorkerCount" : 1 + }, + "phase" : "FINISHED", + "workerCount" : 1, + "partitionCount" : 1, + "shuffle" : "globalSort", + "output" : "localStorage", + "startTime" : __TIMESTAMP__ + "duration" : __DURATION__ + "sort" : true +}, { + "stageNumber" : 2, + "definition" : { + "id" : "_2", + "input" : [ { + "type" : "table", + "dataSource" : "wikipedia", + "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ] + } ], + "processor" : { + "type" : "scan", + "query" : { + "queryType" : "scan", + "dataSource" : { + "type" : "inputNumber", + "inputNumber" : 0 + }, + "intervals" : { + "type" : "intervals", + "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ] + }, + "resultFormat" : "compactedList", + "columns" : [ "cityName", "countryName" ], + "context" : { + "scanSignature" : "[{\"name\":\"cityName\",\"type\":\"STRING\"},{\"name\":\"countryName\",\"type\":\"STRING\"}]", + "sqlInsertSegmentGranularity" : null, + "sqlQueryId" : __SQL_QUERY_ID__ + "sqlStringifyArrays" : false + }, + "columnTypes" : [ "STRING", "STRING" ], + "granularity" : { + "type" : "all" + }, + "legacy" : false + } + }, + "signature" : [ { + "name" : "__boost", + "type" : "LONG" + }, { + "name" : "cityName", + "type" : "STRING" + }, { + "name" : "countryName", + "type" : "STRING" + } ], + "shuffleSpec" : { + "type" : "maxCount", + "clusterBy" : { + "columns" : [ { + "columnName" : "__boost", + "order" : "ASCENDING" + } ] + }, + "partitions" : 1 + }, + "maxWorkerCount" : 1 + }, + "phase" : "FINISHED", + "workerCount" : 1, + "partitionCount" : 1, + "shuffle" : "globalSort", + "output" : "localStorage", + "startTime" : __TIMESTAMP__ + "duration" : __DURATION__ + "sort" : true +}, { + "stageNumber" : 3, + "definition" : { + "id" : "_3", + "input" : [ { + "type" : "stage", + "stage" : 0 + }, { + "type" : "stage", + "stage" : 1 + }, { + "type" : "stage", + "stage" : 2 + } ], + "broadcast" : [ 1, 2 ], + "processor" : { + "type" : "scan", + "query" : { + "queryType" : "scan", + "dataSource" : { + "type" : "join", + "left" : { + "type" : "join", + "left" : { + "type" : "inputNumber", + "inputNumber" : 0 + }, + "right" : { + "type" : "inputNumber", + "inputNumber" : 1 + }, + "rightPrefix" : "j0.", + "condition" : "(\"regionName\" == \"j0.regionName\")", + "joinType" : "LEFT", + "joinAlgorithm" : "sortMerge" + }, + "right" : { + "type" : "inputNumber", + "inputNumber" : 2 + }, + "rightPrefix" : "_j0.", + "condition" : "(\"cityName\" == \"_j0.cityName\")", + "joinType" : "INNER" + }, + "intervals" : { + "type" : "intervals", + "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ] + }, + "virtualColumns" : [ { + "type" : "expression", + "name" : "v0", + "expression" : "'New York'", + "outputType" : "STRING" + } ], + "resultFormat" : "compactedList", + "filter" : { + "type" : "equals", + "column" : "cityName", + "matchValueType" : "STRING", + "matchValue" : "New York" + }, + "columns" : [ "v0", "_j0.countryName" ], + "context" : { + "__user" : null, + "finalize" : true, + "maxParseExceptions" : 0, + "scanSignature" : "[{\"name\":\"v0\",\"type\":\"STRING\"},{\"name\":\"_j0.countryName\",\"type\":\"STRING\"}]", + "sqlQueryId" : __SQL_QUERY_ID__ + "sqlStringifyArrays" : false, + "windowFunctionOperatorTransformation" : true + }, + "columnTypes" : [ "STRING", "STRING" ], + "granularity" : { + "type" : "all" + }, + "legacy" : false + } + }, + "signature" : [ { + "name" : "__boost", + "type" : "LONG" + }, { + "name" : "v0", + "type" : "STRING" + }, { + "name" : "_j0.countryName", + "type" : "STRING" + } ], + "shuffleSpec" : { + "type" : "maxCount", + "clusterBy" : { + "columns" : [ { + "columnName" : "__boost", + "order" : "ASCENDING" + } ] + }, + "partitions" : 1 + }, + "maxWorkerCount" : 1 + }, + "phase" : "FINISHED", + "workerCount" : 1, + "partitionCount" : 1, + "shuffle" : "globalSort", + "output" : "localStorage", + "startTime" : __TIMESTAMP__ + "duration" : __DURATION__ + "sort" : true +} ] +!msqPlan + +select /*+ sort_merge */ w1.cityName, w2.countryName +from +( + select /*+ broadcast */ w3.cityName AS cityName, w4.countryName AS countryName from wikipedia w3 LEFT JOIN wikipedia w4 ON w3.regionName = w4.regionName +) w1 +JOIN wikipedia w2 ON w1.cityName = w2.cityName +where w1.cityName='New York'; + +[ { + "stageNumber" : 0, + "definition" : { + "id" : "_0", + "input" : [ { + "type" : "table", + "dataSource" : "wikipedia", + "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ] + } ], + "processor" : { + "type" : "scan", + "query" : { + "queryType" : "scan", + "dataSource" : { + "type" : "inputNumber", + "inputNumber" : 0 + }, + "intervals" : { + "type" : "intervals", + "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ] + }, + "resultFormat" : "compactedList", + "columns" : [ "regionName" ], + "context" : { + "scanSignature" : "[{\"name\":\"regionName\",\"type\":\"STRING\"}]", + "sqlInsertSegmentGranularity" : null, + "sqlQueryId" : __SQL_QUERY_ID__ + "sqlStringifyArrays" : false + }, + "columnTypes" : [ "STRING" ], + "granularity" : { + "type" : "all" + }, + "legacy" : false + } + }, + "signature" : [ { + "name" : "__boost", + "type" : "LONG" + }, { + "name" : "regionName", + "type" : "STRING" + } ], + "shuffleSpec" : { + "type" : "maxCount", + "clusterBy" : { + "columns" : [ { + "columnName" : "__boost", + "order" : "ASCENDING" + } ] + }, + "partitions" : 1 + }, + "maxWorkerCount" : 1 + }, + "phase" : "FINISHED", + "workerCount" : 1, + "partitionCount" : 1, + "shuffle" : "globalSort", + "output" : "localStorage", + "startTime" : __TIMESTAMP__ + "duration" : __DURATION__ + "sort" : true +}, { + "stageNumber" : 1, + "definition" : { + "id" : "_1", + "input" : [ { + "type" : "table", + "dataSource" : "wikipedia", + "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ] + }, { + "type" : "stage", + "stage" : 0 + } ], + "broadcast" : [ 1 ], + "processor" : { + "type" : "scan", + "query" : { + "queryType" : "scan", + "dataSource" : { + "type" : "join", + "left" : { + "type" : "inputNumber", + "inputNumber" : 0 + }, + "right" : { + "type" : "inputNumber", + "inputNumber" : 1 + }, + "rightPrefix" : "j0.", + "condition" : "(\"regionName\" == \"j0.regionName\")", + "joinType" : "LEFT" + }, + "intervals" : { + "type" : "intervals", + "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ] + }, + "resultFormat" : "compactedList", + "columns" : [ "cityName" ], + "context" : { + "scanSignature" : "[{\"name\":\"cityName\",\"type\":\"STRING\"}]", + "sqlInsertSegmentGranularity" : null, + "sqlQueryId" : __SQL_QUERY_ID__ + "sqlStringifyArrays" : false + }, + "columnTypes" : [ "STRING" ], + "granularity" : { + "type" : "all" + }, + "legacy" : false + } + }, + "signature" : [ { + "name" : "cityName", + "type" : "STRING" + }, { + "name" : "__boost", + "type" : "LONG" + } ], + "shuffleSpec" : { + "type" : "hash", + "clusterBy" : { + "columns" : [ { + "columnName" : "cityName", + "order" : "ASCENDING" + } ] + }, + "partitions" : 1 + }, + "maxWorkerCount" : 1 + }, + "phase" : "FINISHED", + "workerCount" : 1, + "partitionCount" : 1, + "shuffle" : "hashLocalSort", + "output" : "localStorage", + "startTime" : __TIMESTAMP__ + "duration" : __DURATION__ + "sort" : true +}, { + "stageNumber" : 2, + "definition" : { + "id" : "_2", + "input" : [ { + "type" : "table", + "dataSource" : "wikipedia", + "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ] + } ], + "processor" : { + "type" : "scan", + "query" : { + "queryType" : "scan", + "dataSource" : { + "type" : "inputNumber", + "inputNumber" : 0 + }, + "intervals" : { + "type" : "intervals", + "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ] + }, + "resultFormat" : "compactedList", + "columns" : [ "cityName", "countryName" ], + "context" : { + "scanSignature" : "[{\"name\":\"cityName\",\"type\":\"STRING\"},{\"name\":\"countryName\",\"type\":\"STRING\"}]", + "sqlInsertSegmentGranularity" : null, + "sqlQueryId" : __SQL_QUERY_ID__ + "sqlStringifyArrays" : false + }, + "columnTypes" : [ "STRING", "STRING" ], + "granularity" : { + "type" : "all" + }, + "legacy" : false + } + }, + "signature" : [ { + "name" : "cityName", + "type" : "STRING" + }, { + "name" : "__boost", + "type" : "LONG" + }, { + "name" : "countryName", + "type" : "STRING" + } ], + "shuffleSpec" : { + "type" : "hash", + "clusterBy" : { + "columns" : [ { + "columnName" : "cityName", + "order" : "ASCENDING" + } ] + }, + "partitions" : 1 + }, + "maxWorkerCount" : 1 + }, + "phase" : "FINISHED", + "workerCount" : 1, + "partitionCount" : 1, + "shuffle" : "hashLocalSort", + "output" : "localStorage", + "startTime" : __TIMESTAMP__ + "duration" : __DURATION__ + "sort" : true +}, { + "stageNumber" : 3, + "definition" : { + "id" : "_3", + "input" : [ { + "type" : "stage", + "stage" : 1 + }, { + "type" : "stage", + "stage" : 2 + } ], + "processor" : { + "type" : "sortMergeJoin", + "rightPrefix" : "_j0.", + "condition" : "(\"cityName\" == \"_j0.cityName\")", + "joinType" : "INNER" + }, + "signature" : [ { + "name" : "cityName", + "type" : "STRING" + }, { + "name" : "__boost", + "type" : "LONG" + }, { + "name" : "_j0.cityName", + "type" : "STRING" + }, { + "name" : "_j0.__boost", + "type" : "LONG" + }, { + "name" : "_j0.countryName", + "type" : "STRING" + } ], + "maxWorkerCount" : 1 + }, + "phase" : "FINISHED", + "workerCount" : 1, + "partitionCount" : 1, + "output" : "localStorage", + "startTime" : __TIMESTAMP__ + "duration" : __DURATION__ +}, { + "stageNumber" : 4, + "definition" : { + "id" : "_4", + "input" : [ { + "type" : "stage", + "stage" : 3 + } ], + "processor" : { + "type" : "scan", + "query" : { + "queryType" : "scan", + "dataSource" : { + "type" : "inputNumber", + "inputNumber" : 0 + }, + "intervals" : { + "type" : "intervals", + "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ] + }, + "virtualColumns" : [ { + "type" : "expression", + "name" : "v0", + "expression" : "'New York'", + "outputType" : "STRING" + } ], + "resultFormat" : "compactedList", + "filter" : { + "type" : "equals", + "column" : "cityName", + "matchValueType" : "STRING", + "matchValue" : "New York" + }, + "columns" : [ "v0", "_j0.countryName" ], + "context" : { + "__user" : null, + "finalize" : true, + "maxParseExceptions" : 0, + "scanSignature" : "[{\"name\":\"v0\",\"type\":\"STRING\"},{\"name\":\"_j0.countryName\",\"type\":\"STRING\"}]", + "sqlQueryId" : __SQL_QUERY_ID__ + "sqlStringifyArrays" : false, + "windowFunctionOperatorTransformation" : true + }, + "columnTypes" : [ "STRING", "STRING" ], + "granularity" : { + "type" : "all" + }, + "legacy" : false + } + }, + "signature" : [ { + "name" : "__boost", + "type" : "LONG" + }, { + "name" : "v0", + "type" : "STRING" + }, { + "name" : "_j0.countryName", + "type" : "STRING" + } ], + "shuffleSpec" : { + "type" : "maxCount", + "clusterBy" : { + "columns" : [ { + "columnName" : "__boost", + "order" : "ASCENDING" + } ] + }, + "partitions" : 1 + }, + "maxWorkerCount" : 1 + }, + "phase" : "FINISHED", + "workerCount" : 1, + "partitionCount" : 1, + "shuffle" : "globalSort", + "output" : "localStorage", + "startTime" : __TIMESTAMP__ + "duration" : __DURATION__ + "sort" : true +} ] +!msqPlan diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/planner/JoinAlgorithm.java b/processing/src/main/java/org/apache/druid/query/JoinAlgorithm.java similarity index 97% rename from sql/src/main/java/org/apache/druid/sql/calcite/planner/JoinAlgorithm.java rename to processing/src/main/java/org/apache/druid/query/JoinAlgorithm.java index 2e53795c4ffa..94cd209e5780 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/planner/JoinAlgorithm.java +++ b/processing/src/main/java/org/apache/druid/query/JoinAlgorithm.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.druid.sql.calcite.planner; +package org.apache.druid.query; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonValue; diff --git a/processing/src/main/java/org/apache/druid/query/JoinDataSource.java b/processing/src/main/java/org/apache/druid/query/JoinDataSource.java index 220f18a94855..2eb459cf8cc3 100644 --- a/processing/src/main/java/org/apache/druid/query/JoinDataSource.java +++ b/processing/src/main/java/org/apache/druid/query/JoinDataSource.java @@ -96,6 +96,7 @@ public class JoinDataSource implements DataSource private final DimFilter leftFilter; @Nullable private final JoinableFactoryWrapper joinableFactoryWrapper; + private final JoinAlgorithm joinAlgorithm; private static final Logger log = new Logger(JoinDataSource.class); private final DataSourceAnalysis analysis; @@ -106,7 +107,8 @@ private JoinDataSource( JoinConditionAnalysis conditionAnalysis, JoinType joinType, @Nullable DimFilter leftFilter, - @Nullable JoinableFactoryWrapper joinableFactoryWrapper + @Nullable JoinableFactoryWrapper joinableFactoryWrapper, + JoinAlgorithm joinAlgorithm ) { this.left = Preconditions.checkNotNull(left, "left"); @@ -116,6 +118,7 @@ private JoinDataSource( this.joinType = Preconditions.checkNotNull(joinType, "joinType"); this.leftFilter = validateLeftFilter(left, leftFilter); this.joinableFactoryWrapper = joinableFactoryWrapper; + this.joinAlgorithm = JoinAlgorithm.BROADCAST.equals(joinAlgorithm) ? null : joinAlgorithm; this.analysis = this.getAnalysisForDataSource(); } @@ -132,7 +135,8 @@ public static JoinDataSource create( @JsonProperty("joinType") JoinType joinType, @Nullable @JsonProperty("leftFilter") DimFilter leftFilter, @JacksonInject ExprMacroTable macroTable, - @Nullable @JacksonInject JoinableFactoryWrapper joinableFactoryWrapper + @Nullable @JacksonInject JoinableFactoryWrapper joinableFactoryWrapper, + @Nullable @JsonProperty("joinAlgorithm") JoinAlgorithm joinAlgorithm ) { return new JoinDataSource( @@ -146,7 +150,8 @@ public static JoinDataSource create( ), joinType, leftFilter, - joinableFactoryWrapper + joinableFactoryWrapper, + joinAlgorithm ); } @@ -160,7 +165,8 @@ public static JoinDataSource create( final JoinConditionAnalysis conditionAnalysis, final JoinType joinType, final DimFilter leftFilter, - @Nullable final JoinableFactoryWrapper joinableFactoryWrapper + @Nullable final JoinableFactoryWrapper joinableFactoryWrapper, + @Nullable final JoinAlgorithm joinAlgorithm ) { return new JoinDataSource( @@ -170,11 +176,11 @@ public static JoinDataSource create( conditionAnalysis, joinType, leftFilter, - joinableFactoryWrapper + joinableFactoryWrapper, + joinAlgorithm ); } - @Override public Set getTableNames() { @@ -253,7 +259,8 @@ public DataSource withChildren(List children) conditionAnalysis, joinType, leftFilter, - joinableFactoryWrapper + joinableFactoryWrapper, + joinAlgorithm ); } @@ -320,7 +327,8 @@ public DataSource withUpdatedDataSource(DataSource newSource) clause.getCondition(), clause.getJoinType(), joinBaseFilter, - this.joinableFactoryWrapper + this.joinableFactoryWrapper, + clause.getJoinAlgorithm() ); joinBaseFilter = null; } @@ -363,6 +371,18 @@ public DataSourceAnalysis getAnalysis() return analysis; } + @JsonProperty("joinAlgorithm") + @JsonInclude(Include.NON_NULL) + private JoinAlgorithm getJoinAlgorithmForSerialization() + { + return joinAlgorithm; + } + + public JoinAlgorithm getJoinAlgorithm() + { + return joinAlgorithm == null ? JoinAlgorithm.BROADCAST : joinAlgorithm; + } + @Override public boolean equals(Object o) { @@ -378,13 +398,14 @@ public boolean equals(Object o) Objects.equals(rightPrefix, that.rightPrefix) && Objects.equals(conditionAnalysis, that.conditionAnalysis) && Objects.equals(leftFilter, that.leftFilter) && + joinAlgorithm == that.joinAlgorithm && joinType == that.joinType; } @Override public int hashCode() { - return Objects.hash(left, right, rightPrefix, conditionAnalysis, joinType, leftFilter); + return Objects.hash(left, right, rightPrefix, conditionAnalysis, joinType, leftFilter, joinAlgorithm); } @Override @@ -397,6 +418,7 @@ public String toString() ", condition=" + conditionAnalysis + ", joinType=" + joinType + ", leftFilter=" + leftFilter + + ", joinAlgorithm=" + joinAlgorithm + '}'; } @@ -539,7 +561,8 @@ private static Triple> flattenJoi joinDataSource.getRightPrefix(), joinDataSource.getRight(), joinDataSource.getJoinType(), - joinDataSource.getConditionAnalysis() + joinDataSource.getConditionAnalysis(), + joinDataSource.getJoinAlgorithm() ) ); } else if (current instanceof UnnestDataSource) { diff --git a/processing/src/main/java/org/apache/druid/query/planning/PreJoinableClause.java b/processing/src/main/java/org/apache/druid/query/planning/PreJoinableClause.java index 44e3f77262fb..ee3ed01020f1 100644 --- a/processing/src/main/java/org/apache/druid/query/planning/PreJoinableClause.java +++ b/processing/src/main/java/org/apache/druid/query/planning/PreJoinableClause.java @@ -21,10 +21,12 @@ import com.google.common.base.Preconditions; import org.apache.druid.query.DataSource; +import org.apache.druid.query.JoinAlgorithm; import org.apache.druid.segment.join.JoinConditionAnalysis; import org.apache.druid.segment.join.JoinPrefixUtils; import org.apache.druid.segment.join.JoinType; +import javax.annotation.Nullable; import java.util.Objects; /** @@ -38,18 +40,21 @@ public class PreJoinableClause private final DataSource dataSource; private final JoinType joinType; private final JoinConditionAnalysis condition; + private final JoinAlgorithm joinAlgorithm; public PreJoinableClause( final String prefix, final DataSource dataSource, final JoinType joinType, - final JoinConditionAnalysis condition + final JoinConditionAnalysis condition, + @Nullable final JoinAlgorithm joinAlgorithm ) { this.prefix = JoinPrefixUtils.validatePrefix(prefix); this.dataSource = Preconditions.checkNotNull(dataSource, "dataSource"); this.joinType = Preconditions.checkNotNull(joinType, "joinType"); this.condition = Preconditions.checkNotNull(condition, "condition"); + this.joinAlgorithm = joinAlgorithm; } public String getPrefix() @@ -72,6 +77,12 @@ public JoinConditionAnalysis getCondition() return condition; } + @Nullable + public JoinAlgorithm getJoinAlgorithm() + { + return joinAlgorithm; + } + @Override public boolean equals(Object o) { @@ -82,16 +93,17 @@ public boolean equals(Object o) return false; } PreJoinableClause that = (PreJoinableClause) o; - return Objects.equals(prefix, that.prefix) && - Objects.equals(dataSource, that.dataSource) && - joinType == that.joinType && - Objects.equals(condition, that.condition); + return Objects.equals(prefix, that.prefix) + && Objects.equals(dataSource, that.dataSource) + && joinType == that.joinType + && Objects.equals(condition, that.condition) + && joinAlgorithm == that.joinAlgorithm; } @Override public int hashCode() { - return Objects.hash(prefix, dataSource, joinType, condition); + return Objects.hash(prefix, dataSource, joinType, condition, joinAlgorithm); } @Override @@ -102,6 +114,7 @@ public String toString() ", dataSource=" + dataSource + ", joinType=" + joinType + ", condition=" + condition + + ", joinAlgorithm=" + joinAlgorithm + '}'; } } diff --git a/processing/src/test/java/org/apache/druid/query/JoinDataSourceTest.java b/processing/src/test/java/org/apache/druid/query/JoinDataSourceTest.java index b821bc49c4e7..ac3786282cf5 100644 --- a/processing/src/test/java/org/apache/druid/query/JoinDataSourceTest.java +++ b/processing/src/test/java/org/apache/druid/query/JoinDataSourceTest.java @@ -65,7 +65,9 @@ public class JoinDataSourceTest JoinType.LEFT, null, ExprMacroTable.nil(), - null + null, + JoinAlgorithm.BROADCAST + ); private final JoinDataSource joinTableToTable = JoinDataSource.create( fooTable, @@ -75,7 +77,9 @@ public class JoinDataSourceTest JoinType.LEFT, null, ExprMacroTable.nil(), - null + null, + JoinAlgorithm.BROADCAST + ); @Rule public ExpectedException expectedException = ExpectedException.none(); @@ -182,7 +186,8 @@ public void test_serde() throws Exception JoinType.LEFT, TrueDimFilter.instance(), ExprMacroTable.nil(), - joinableFactoryWrapper + joinableFactoryWrapper, + JoinAlgorithm.BROADCAST ); final JoinDataSource deserialized = (JoinDataSource) jsonMapper.readValue( @@ -206,7 +211,8 @@ public void testException_leftFilterOnNonTableSource() JoinType.LEFT, TrueDimFilter.instance(), ExprMacroTable.nil(), - null + null, + JoinAlgorithm.BROADCAST ); } @@ -221,7 +227,8 @@ public void testLeftFilter() JoinType.LEFT, TrueDimFilter.instance(), ExprMacroTable.nil(), - null + null, + JoinAlgorithm.BROADCAST ); Assert.assertEquals(TrueDimFilter.instance(), dataSource.getLeftFilter()); } @@ -237,7 +244,8 @@ public void testVirtualColumnCandidates() JoinType.LEFT, null, ExprMacroTable.nil(), - null + null, + JoinAlgorithm.BROADCAST ); Assert.assertEquals(dataSource.getVirtualColumnCandidates(), ImmutableSet.of("x")); } @@ -253,7 +261,8 @@ public void test_computeJoinDataSourceCacheKey_noHashJoin() JoinType.LEFT, null, ExprMacroTable.nil(), - NOOP_JOINABLE_FACTORY_WRAPPER + NOOP_JOINABLE_FACTORY_WRAPPER, + JoinAlgorithm.BROADCAST ); Optional cacheKey = Optional.ofNullable(dataSource.getCacheKey()); @@ -274,7 +283,8 @@ public void test_computeJoinDataSourceCacheKey_sameKeyForSameJoin() JoinType.LEFT, null, ExprMacroTable.nil(), - joinableFactoryWrapper + joinableFactoryWrapper, + JoinAlgorithm.BROADCAST ); JoinDataSource joinDataSource1 = JoinDataSource.create( @@ -285,7 +295,8 @@ public void test_computeJoinDataSourceCacheKey_sameKeyForSameJoin() JoinType.LEFT, null, ExprMacroTable.nil(), - joinableFactoryWrapper + joinableFactoryWrapper, + JoinAlgorithm.BROADCAST ); byte[] cacheKey1 = joinDataSource.getCacheKey(); @@ -309,7 +320,8 @@ public void test_computeJoinDataSourceCacheKey_keyChangesWithTables() JoinType.LEFT, null, ExprMacroTable.nil(), - joinableFactoryWrapper + joinableFactoryWrapper, + JoinAlgorithm.BROADCAST ); JoinDataSource joinDataSource1 = JoinDataSource.create( @@ -320,7 +332,8 @@ public void test_computeJoinDataSourceCacheKey_keyChangesWithTables() JoinType.LEFT, null, ExprMacroTable.nil(), - joinableFactoryWrapper + joinableFactoryWrapper, + JoinAlgorithm.BROADCAST ); byte[] cacheKey1 = joinDataSource.getCacheKey(); @@ -344,7 +357,8 @@ public void test_computeJoinDataSourceCacheKey_keyChangesWithExpressions() JoinType.LEFT, null, ExprMacroTable.nil(), - joinableFactoryWrapper + joinableFactoryWrapper, + JoinAlgorithm.BROADCAST ); JoinDataSource joinDataSource1 = JoinDataSource.create( @@ -355,7 +369,8 @@ public void test_computeJoinDataSourceCacheKey_keyChangesWithExpressions() JoinType.LEFT, null, ExprMacroTable.nil(), - joinableFactoryWrapper + joinableFactoryWrapper, + JoinAlgorithm.BROADCAST ); byte[] cacheKey1 = joinDataSource.getCacheKey(); @@ -379,7 +394,8 @@ public void test_computeJoinDataSourceCacheKey_keyChangesWithJoinType() JoinType.LEFT, null, ExprMacroTable.nil(), - joinableFactoryWrapper + joinableFactoryWrapper, + JoinAlgorithm.BROADCAST ); JoinDataSource joinDataSource1 = JoinDataSource.create( @@ -390,7 +406,8 @@ public void test_computeJoinDataSourceCacheKey_keyChangesWithJoinType() JoinType.INNER, null, ExprMacroTable.nil(), - joinableFactoryWrapper + joinableFactoryWrapper, + JoinAlgorithm.BROADCAST ); byte[] cacheKey1 = joinDataSource.getCacheKey(); @@ -414,7 +431,8 @@ public void test_computeJoinDataSourceCacheKey_keyChangesWithPrefix() JoinType.LEFT, null, ExprMacroTable.nil(), - joinableFactoryWrapper + joinableFactoryWrapper, + JoinAlgorithm.BROADCAST ); JoinDataSource joinDataSource1 = JoinDataSource.create( @@ -425,7 +443,8 @@ public void test_computeJoinDataSourceCacheKey_keyChangesWithPrefix() JoinType.LEFT, null, ExprMacroTable.nil(), - joinableFactoryWrapper + joinableFactoryWrapper, + JoinAlgorithm.BROADCAST ); byte[] cacheKey1 = joinDataSource.getCacheKey(); @@ -451,7 +470,8 @@ public void testGetAnalysisWithUnnestDS() JoinType.LEFT, null, ExprMacroTable.nil(), - null + null, + JoinAlgorithm.BROADCAST ); DataSourceAnalysis analysis = dataSource.getAnalysis(); Assert.assertEquals("table1", analysis.getBaseDataSource().getTableNames().iterator().next()); @@ -475,7 +495,8 @@ public void testGetAnalysisWithFilteredDS() JoinType.LEFT, null, ExprMacroTable.nil(), - null + null, + JoinAlgorithm.BROADCAST ); DataSourceAnalysis analysis = dataSource.getAnalysis(); Assert.assertEquals("table1", analysis.getBaseDataSource().getTableNames().iterator().next()); @@ -496,7 +517,8 @@ public void test_computeJoinDataSourceCacheKey_keyChangesWithBaseFilter() JoinType.LEFT, expectedInDimFilter, ExprMacroTable.nil(), - joinableFactoryWrapper + joinableFactoryWrapper, + JoinAlgorithm.BROADCAST ); JoinDataSource joinDataSource1 = JoinDataSource.create( @@ -507,7 +529,8 @@ public void test_computeJoinDataSourceCacheKey_keyChangesWithBaseFilter() JoinType.LEFT, null, ExprMacroTable.nil(), - joinableFactoryWrapper + joinableFactoryWrapper, + JoinAlgorithm.BROADCAST ); byte[] cacheKey1 = joinDataSource.getCacheKey(); @@ -531,7 +554,8 @@ public void test_computeJoinDataSourceCacheKey_cachingUnsupported() JoinType.LEFT, null, ExprMacroTable.nil(), - joinableFactoryWrapper + joinableFactoryWrapper, + JoinAlgorithm.BROADCAST ); byte[] cacheKey1 = joinDataSource.getCacheKey(); diff --git a/processing/src/test/java/org/apache/druid/query/QueriesTest.java b/processing/src/test/java/org/apache/druid/query/QueriesTest.java index b83e256d7a80..444e67e2e462 100644 --- a/processing/src/test/java/org/apache/druid/query/QueriesTest.java +++ b/processing/src/test/java/org/apache/druid/query/QueriesTest.java @@ -426,7 +426,8 @@ public void testWithBaseDataSourceSubQueryStackWithJoinOnUnion() JoinType.INNER, null, ExprMacroTable.nil(), - null + null, + JoinAlgorithm.BROADCAST ) ) .intervals("2000/3000") @@ -464,7 +465,8 @@ public void testWithBaseDataSourceSubQueryStackWithJoinOnUnion() JoinType.INNER, null, ExprMacroTable.nil(), - null + null, + JoinAlgorithm.BROADCAST ) ) .intervals("2000/3000") @@ -503,7 +505,9 @@ public void testWithBaseDataSourcedBaseFilterWithMultiJoin() JoinType.INNER, TrueDimFilter.instance(), ExprMacroTable.nil(), - null + null, + JoinAlgorithm.BROADCAST + ), new TableDataSource("foo_outer"), "j0.", @@ -511,7 +515,9 @@ public void testWithBaseDataSourcedBaseFilterWithMultiJoin() JoinType.INNER, null, ExprMacroTable.nil(), - null + null, + JoinAlgorithm.BROADCAST + ) ) @@ -538,7 +544,9 @@ public void testWithBaseDataSourcedBaseFilterWithMultiJoin() JoinType.INNER, TrueDimFilter.instance(), ExprMacroTable.nil(), - null + null, + JoinAlgorithm.BROADCAST + ), new TableDataSource("foo_outer"), "j0.", @@ -546,7 +554,9 @@ public void testWithBaseDataSourcedBaseFilterWithMultiJoin() JoinType.INNER, null, ExprMacroTable.nil(), - null + null, + JoinAlgorithm.BROADCAST + ) ) diff --git a/processing/src/test/java/org/apache/druid/query/metadata/SegmentMetadataQueryTest.java b/processing/src/test/java/org/apache/druid/query/metadata/SegmentMetadataQueryTest.java index 8e39a640acb3..9bcf69f0aa8b 100644 --- a/processing/src/test/java/org/apache/druid/query/metadata/SegmentMetadataQueryTest.java +++ b/processing/src/test/java/org/apache/druid/query/metadata/SegmentMetadataQueryTest.java @@ -39,6 +39,7 @@ import org.apache.druid.query.Druids; import org.apache.druid.query.FinalizeResultsQueryRunner; import org.apache.druid.query.InlineDataSource; +import org.apache.druid.query.JoinAlgorithm; import org.apache.druid.query.JoinDataSource; import org.apache.druid.query.LookupDataSource; import org.apache.druid.query.Query; @@ -1601,7 +1602,8 @@ public void testSegmentMetadataQueryWithInvalidDatasourceTypes() JoinType.LEFT, null, ExprMacroTable.nil(), - null + null, + JoinAlgorithm.BROADCAST ), new LegacySegmentSpec("2015-01-01/2015-01-02"), null, @@ -1616,7 +1618,7 @@ public void testSegmentMetadataQueryWithInvalidDatasourceTypes() DruidExceptionMatcher .invalidInput() .expectMessageIs( - "Invalid dataSource type [JoinDataSource{left=table1, right=table2, rightPrefix='j.', condition=x == \"j.x\", joinType=LEFT, leftFilter=null}]. SegmentMetadataQuery only supports table or union datasources.") + "Invalid dataSource type [JoinDataSource{left=table1, right=table2, rightPrefix='j.', condition=x == \"j.x\", joinType=LEFT, leftFilter=null, joinAlgorithm=null}]. SegmentMetadataQuery only supports table or union datasources.") ); } diff --git a/processing/src/test/java/org/apache/druid/query/planning/DataSourceAnalysisTest.java b/processing/src/test/java/org/apache/druid/query/planning/DataSourceAnalysisTest.java index fc85b4e3fb6c..522e58daf1f5 100644 --- a/processing/src/test/java/org/apache/druid/query/planning/DataSourceAnalysisTest.java +++ b/processing/src/test/java/org/apache/druid/query/planning/DataSourceAnalysisTest.java @@ -27,6 +27,7 @@ import org.apache.druid.math.expr.ExprMacroTable; import org.apache.druid.query.DataSource; import org.apache.druid.query.InlineDataSource; +import org.apache.druid.query.JoinAlgorithm; import org.apache.druid.query.JoinDataSource; import org.apache.druid.query.LookupDataSource; import org.apache.druid.query.QueryDataSource; @@ -245,9 +246,9 @@ public void testJoinSimpleLeftLeaning() Assert.assertEquals(Optional.empty(), analysis.getBaseQuerySegmentSpec()); Assert.assertEquals( ImmutableList.of( - new PreJoinableClause("1.", LOOKUP_LOOKYLOO, JoinType.INNER, joinClause("1.")), - new PreJoinableClause("2.", INLINE, JoinType.LEFT, joinClause("2.")), - new PreJoinableClause("3.", subquery(LOOKUP_LOOKYLOO), JoinType.FULL, joinClause("3.")) + new PreJoinableClause("1.", LOOKUP_LOOKYLOO, JoinType.INNER, joinClause("1."), JoinAlgorithm.BROADCAST), + new PreJoinableClause("2.", INLINE, JoinType.LEFT, joinClause("2."), JoinAlgorithm.BROADCAST), + new PreJoinableClause("3.", subquery(LOOKUP_LOOKYLOO), JoinType.FULL, joinClause("3."), JoinAlgorithm.BROADCAST) ), analysis.getPreJoinableClauses() ); @@ -296,9 +297,9 @@ public void testJoinSimpleLeftLeaningWithLeftFilter() Assert.assertEquals(Optional.empty(), analysis.getBaseQuerySegmentSpec()); Assert.assertEquals( ImmutableList.of( - new PreJoinableClause("1.", LOOKUP_LOOKYLOO, JoinType.INNER, joinClause("1.")), - new PreJoinableClause("2.", INLINE, JoinType.LEFT, joinClause("2.")), - new PreJoinableClause("3.", subquery(LOOKUP_LOOKYLOO), JoinType.FULL, joinClause("3.")) + new PreJoinableClause("1.", LOOKUP_LOOKYLOO, JoinType.INNER, joinClause("1."), JoinAlgorithm.BROADCAST), + new PreJoinableClause("2.", INLINE, JoinType.LEFT, joinClause("2."), JoinAlgorithm.BROADCAST), + new PreJoinableClause("3.", subquery(LOOKUP_LOOKYLOO), JoinType.FULL, joinClause("3."), JoinAlgorithm.BROADCAST) ), analysis.getPreJoinableClauses() ); @@ -353,7 +354,7 @@ public void testJoinSimpleRightLeaning() Assert.assertEquals(Optional.empty(), analysis.getBaseQuerySegmentSpec()); Assert.assertEquals( ImmutableList.of( - new PreJoinableClause("3.", rightLeaningJoinStack, JoinType.RIGHT, joinClause("3.")) + new PreJoinableClause("3.", rightLeaningJoinStack, JoinType.RIGHT, joinClause("3."), JoinAlgorithm.BROADCAST) ), analysis.getPreJoinableClauses() ); @@ -403,7 +404,7 @@ public void testJoinSimpleRightLeaningWithLeftFilter() Assert.assertEquals(Optional.empty(), analysis.getBaseQuerySegmentSpec()); Assert.assertEquals( ImmutableList.of( - new PreJoinableClause("3.", rightLeaningJoinStack, JoinType.RIGHT, joinClause("3.")) + new PreJoinableClause("3.", rightLeaningJoinStack, JoinType.RIGHT, joinClause("3."), JoinAlgorithm.BROADCAST) ), analysis.getPreJoinableClauses() ); @@ -437,7 +438,7 @@ public void testJoinOverTableSubquery() Assert.assertEquals(Optional.empty(), analysis.getBaseUnionDataSource()); Assert.assertEquals( ImmutableList.of( - new PreJoinableClause("1.", subquery(TABLE_FOO), JoinType.INNER, joinClause("1.")) + new PreJoinableClause("1.", subquery(TABLE_FOO), JoinType.INNER, joinClause("1."), JoinAlgorithm.BROADCAST) ), analysis.getPreJoinableClauses() ); @@ -471,7 +472,7 @@ public void testJoinTableUnionToLookup() Assert.assertEquals(Optional.empty(), analysis.getBaseQuerySegmentSpec()); Assert.assertEquals( ImmutableList.of( - new PreJoinableClause("1.", LOOKUP_LOOKYLOO, JoinType.INNER, joinClause("1.")) + new PreJoinableClause("1.", LOOKUP_LOOKYLOO, JoinType.INNER, joinClause("1."), JoinAlgorithm.BROADCAST) ), analysis.getPreJoinableClauses() ); @@ -526,7 +527,7 @@ public void testJoinUnderTopLevelSubqueries() ); Assert.assertEquals( ImmutableList.of( - new PreJoinableClause("1.", LOOKUP_LOOKYLOO, JoinType.INNER, joinClause("1.")) + new PreJoinableClause("1.", LOOKUP_LOOKYLOO, JoinType.INNER, joinClause("1."), JoinAlgorithm.BROADCAST) ), analysis.getPreJoinableClauses() ); @@ -559,7 +560,7 @@ public void testJoinLookupToLookup() Assert.assertEquals(Optional.empty(), analysis.getJoinBaseTableFilter()); Assert.assertEquals( ImmutableList.of( - new PreJoinableClause("1.", LOOKUP_LOOKYLOO, JoinType.INNER, joinClause("1.")) + new PreJoinableClause("1.", LOOKUP_LOOKYLOO, JoinType.INNER, joinClause("1."), JoinAlgorithm.BROADCAST) ), analysis.getPreJoinableClauses() ); @@ -592,7 +593,7 @@ public void testJoinLookupToTable() Assert.assertEquals(Optional.empty(), analysis.getJoinBaseTableFilter()); Assert.assertEquals( ImmutableList.of( - new PreJoinableClause("1.", TABLE_FOO, JoinType.INNER, joinClause("1.")) + new PreJoinableClause("1.", TABLE_FOO, JoinType.INNER, joinClause("1."), JoinAlgorithm.BROADCAST) ), analysis.getPreJoinableClauses() ); @@ -633,7 +634,8 @@ private static JoinDataSource join( joinType, dimFilter, ExprMacroTable.nil(), - null + null, + JoinAlgorithm.BROADCAST ); } diff --git a/processing/src/test/java/org/apache/druid/query/planning/PreJoinableClauseTest.java b/processing/src/test/java/org/apache/druid/query/planning/PreJoinableClauseTest.java index 80762a758c83..a9fbaae6a056 100644 --- a/processing/src/test/java/org/apache/druid/query/planning/PreJoinableClauseTest.java +++ b/processing/src/test/java/org/apache/druid/query/planning/PreJoinableClauseTest.java @@ -33,7 +33,8 @@ public class PreJoinableClauseTest "j.", new TableDataSource("foo"), JoinType.LEFT, - JoinConditionAnalysis.forExpression("x == \"j.x\"", "j.", ExprMacroTable.nil()) + JoinConditionAnalysis.forExpression("x == \"j.x\"", "j.", ExprMacroTable.nil()), + null ); @Test diff --git a/server/src/test/java/org/apache/druid/server/ClientQuerySegmentWalkerTest.java b/server/src/test/java/org/apache/druid/server/ClientQuerySegmentWalkerTest.java index 7a8e2248367d..f9c6c02cca33 100644 --- a/server/src/test/java/org/apache/druid/server/ClientQuerySegmentWalkerTest.java +++ b/server/src/test/java/org/apache/druid/server/ClientQuerySegmentWalkerTest.java @@ -42,6 +42,7 @@ import org.apache.druid.query.FrameBasedInlineDataSource; import org.apache.druid.query.GlobalTableDataSource; import org.apache.druid.query.InlineDataSource; +import org.apache.druid.query.JoinAlgorithm; import org.apache.druid.query.JoinDataSource; import org.apache.druid.query.Query; import org.apache.druid.query.QueryContexts; @@ -552,7 +553,8 @@ public void testJoinOnGroupByOnTable() new JoinableFactoryWrapper(QueryStackTests.makeJoinableFactoryFromDefault( null, null, - null)) + null)), + JoinAlgorithm.BROADCAST ) ) .setGranularity(Granularities.ALL) @@ -624,7 +626,8 @@ public void testJoinOnGroupByOnUnionOfTables() new JoinableFactoryWrapper(QueryStackTests.makeJoinableFactoryFromDefault( null, null, - null)) + null)), + JoinAlgorithm.BROADCAST ) ) .setGranularity(Granularities.ALL) @@ -801,7 +804,8 @@ public void testJoinOnTableErrorCantInlineTable() JoinType.INNER, null, ExprMacroTable.nil(), - null + null, + JoinAlgorithm.BROADCAST ) ) .setGranularity(Granularities.ALL) diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/planner/CalcitePlanner.java b/sql/src/main/java/org/apache/druid/sql/calcite/planner/CalcitePlanner.java index 8eb9541961c6..559d4bbaa3c6 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/planner/CalcitePlanner.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/planner/CalcitePlanner.java @@ -235,7 +235,7 @@ public SqlNode parse(final Reader reader) throws SqlParseException @Override public SqlNode validate(SqlNode sqlNode) throws ValidationException { - Hook.PARSE_TREE.run(new Object[] {null, sqlNode}); + Hook.PARSE_TREE.run(new Object[]{null, sqlNode}); ensure(CalcitePlanner.State.STATE_3_PARSED); this.validator = createSqlValidator(createCatalogReader()); try { @@ -295,10 +295,11 @@ public RelRoot rel(SqlNode sql) rexBuilder ); final SqlToRelConverter.Config config = - sqlToRelConverterConfig.withTrimUnusedFields(false); + sqlToRelConverterConfig.withTrimUnusedFields(false) + .withHintStrategyTable(DruidHint.HINT_STRATEGY_TABLE); final SqlToRelConverter sqlToRelConverter = new DruidSqlToRelConverter(this, validator, - createCatalogReader(), cluster, convertletTable, config + createCatalogReader(), cluster, convertletTable, config ); RelRoot root = sqlToRelConverter.convertQuery(sql, false, true); diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/planner/CalciteRulesManager.java b/sql/src/main/java/org/apache/druid/sql/calcite/planner/CalciteRulesManager.java index d6dd1310e6c5..ea4efd16d91a 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/planner/CalciteRulesManager.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/planner/CalciteRulesManager.java @@ -50,6 +50,7 @@ import org.apache.calcite.tools.Programs; import org.apache.calcite.tools.RelBuilder; import org.apache.druid.java.util.common.logger.Logger; +import org.apache.druid.query.JoinAlgorithm; import org.apache.druid.sql.calcite.external.ExternalTableScanRule; import org.apache.druid.sql.calcite.rule.AggregatePullUpLookupRule; import org.apache.druid.sql.calcite.rule.CaseToCoalesceRule; diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/planner/DruidHint.java b/sql/src/main/java/org/apache/druid/sql/calcite/planner/DruidHint.java new file mode 100644 index 000000000000..a5c57c95e282 --- /dev/null +++ b/sql/src/main/java/org/apache/druid/sql/calcite/planner/DruidHint.java @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.sql.calcite.planner; + +import org.apache.calcite.rel.hint.HintPredicates; +import org.apache.calcite.rel.hint.HintStrategyTable; +import org.apache.druid.query.JoinAlgorithm; + +import javax.annotation.Nullable; + +public abstract class DruidHint +{ + public abstract static class DruidJoinHint extends DruidHint + { + @Nullable + public static DruidJoinHint fromString(String hintName) + { + switch (hintName) { + case BroadcastJoinHint.BROADCAST_JOIN: + return new BroadcastJoinHint(); + case SortMergeJoinHint.SORT_MERGE_JOIN: + return new SortMergeJoinHint(); + default: + return null; + } + } + + abstract JoinAlgorithm asJoinAlgorithm(); + + abstract String id(); + + public static class SortMergeJoinHint extends DruidJoinHint + { + static final String SORT_MERGE_JOIN = "sort_merge"; + + @Override + String id() + { + return SORT_MERGE_JOIN; + } + + @Override + public JoinAlgorithm asJoinAlgorithm() + { + return JoinAlgorithm.SORT_MERGE; + } + } + + public static class BroadcastJoinHint extends DruidJoinHint + { + static final String BROADCAST_JOIN = "broadcast"; + + @Override + String id() + { + return BROADCAST_JOIN; + } + + @Override + public JoinAlgorithm asJoinAlgorithm() + { + return JoinAlgorithm.BROADCAST; + } + } + } + + public static final HintStrategyTable HINT_STRATEGY_TABLE = createHintStrategies(); + + /** + * Creates hint strategies. + * + * @return HintStrategyTable instance + */ + private static HintStrategyTable createHintStrategies() + { + return HintStrategyTable.builder() + .hintStrategy(DruidJoinHint.SortMergeJoinHint.SORT_MERGE_JOIN, HintPredicates.JOIN) + .hintStrategy(DruidJoinHint.BroadcastJoinHint.BROADCAST_JOIN, HintPredicates.JOIN) + .build(); + } +} diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/planner/PlannerContext.java b/sql/src/main/java/org/apache/druid/sql/calcite/planner/PlannerContext.java index c02cb3cd8bd7..124b602ae2fa 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/planner/PlannerContext.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/planner/PlannerContext.java @@ -37,6 +37,7 @@ import org.apache.druid.java.util.common.granularity.Granularities; import org.apache.druid.math.expr.Expr; import org.apache.druid.math.expr.ExprMacroTable; +import org.apache.druid.query.JoinAlgorithm; import org.apache.druid.query.QueryContext; import org.apache.druid.query.QueryContexts; import org.apache.druid.query.filter.InDimFilter; diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/planner/QueryUtils.java b/sql/src/main/java/org/apache/druid/sql/calcite/planner/QueryUtils.java index e0a5cce94187..e7de26f6f6f1 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/planner/QueryUtils.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/planner/QueryUtils.java @@ -19,6 +19,9 @@ package org.apache.druid.sql.calcite.planner; +import org.apache.calcite.rel.core.Join; +import org.apache.calcite.rel.hint.RelHint; +import org.apache.druid.query.JoinAlgorithm; import org.apache.druid.sql.calcite.rel.DruidQuery; import java.util.ArrayList; @@ -55,4 +58,21 @@ public static List buildColumnMappings( return columnMappings; } + + public static JoinAlgorithm getJoinAlgorithm(Join join, PlannerContext plannerContext) + { + RelHint closestHint = null; + for (RelHint hint : join.getHints()) { + if ((closestHint == null || hint.inheritPath.size() < closestHint.inheritPath.size()) + && DruidHint.DruidJoinHint.fromString(hint.hintName) != null) { + closestHint = hint; + } + } + + if (closestHint != null) { + return DruidHint.DruidJoinHint.fromString(closestHint.hintName).asJoinAlgorithm(); + } else { + return plannerContext.getJoinAlgorithm(); + } + } } diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/planner/QueryValidations.java b/sql/src/main/java/org/apache/druid/sql/calcite/planner/QueryValidations.java index c516496af31e..27c5ca7e9e82 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/planner/QueryValidations.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/planner/QueryValidations.java @@ -25,6 +25,7 @@ import org.apache.calcite.rel.logical.LogicalJoin; import org.apache.calcite.tools.ValidationException; import org.apache.druid.java.util.common.StringUtils; +import org.apache.druid.query.JoinAlgorithm; import org.apache.druid.sql.calcite.run.EngineFeature; /** diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/rel/DruidJoinQueryRel.java b/sql/src/main/java/org/apache/druid/sql/calcite/rel/DruidJoinQueryRel.java index 5615da3344fe..312ba1885dd9 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/rel/DruidJoinQueryRel.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/rel/DruidJoinQueryRel.java @@ -55,6 +55,7 @@ import org.apache.druid.sql.calcite.planner.Calcites; import org.apache.druid.sql.calcite.planner.PlannerConfig; import org.apache.druid.sql.calcite.planner.PlannerContext; +import org.apache.druid.sql.calcite.planner.QueryUtils; import org.apache.druid.sql.calcite.planner.querygen.SourceDescProducer.SourceDesc; import org.apache.druid.sql.calcite.table.RowSignatures; @@ -151,7 +152,7 @@ private SourceDesc buildLeftSourceDesc() final DruidQuery leftQuery = Preconditions.checkNotNull(leftDruidRel.toDruidQuery(false), "leftQuery"); final RowSignature leftSignature = leftQuery.getOutputRowSignature(); final DataSource leftDataSource; - if (computeLeftRequiresSubquery(getPlannerContext(), leftDruidRel)) { + if (computeLeftRequiresSubquery(getPlannerContext(), leftDruidRel, joinRel)) { leftDataSource = new QueryDataSource(leftQuery.getQuery()); if (leftFilter != null) { throw new ISE("Filter on left table is supposed to be null if left child is a query source"); @@ -225,7 +226,8 @@ public static SourceDesc buildJoinSourceDesc(final SourceDesc leftDesc, final So ), toDruidJoinType(joinRel.getJoinType()), getDimFilter(plannerContext, leftDesc.rowSignature, leftFilter), - plannerContext.getJoinableFactoryWrapper() + plannerContext.getJoinableFactoryWrapper(), + QueryUtils.getJoinAlgorithm(joinRel, plannerContext) ); SourceDesc sourceDesc = new SourceDesc(joinDataSource, signature, virtualColumnRegistry); @@ -360,7 +362,7 @@ public RelOptCost computeSelfCost(final RelOptPlanner planner, final RelMetadata joinCost *= CostEstimates.MULTIPLIER_OUTER_QUERY; } else { // Penalize subqueries if we don't have to do them. - if (computeLeftRequiresSubquery(getPlannerContext(), getSomeDruidChild(left))) { + if (computeLeftRequiresSubquery(getPlannerContext(), getSomeDruidChild(left), joinRel)) { joinCost += CostEstimates.COST_SUBQUERY; } else { if (joinRel.getJoinType() == JoinRelType.INNER && plannerConfig.isComputeInnerJoinCostAsFilter()) { @@ -400,9 +402,9 @@ public static JoinType toDruidJoinType(JoinRelType calciteJoinType) } } - public static boolean computeLeftRequiresSubquery(final PlannerContext plannerContext, final DruidRel left) + public static boolean computeLeftRequiresSubquery(final PlannerContext plannerContext, final DruidRel left, final Join joinRel) { - if (plannerContext.getJoinAlgorithm().requiresSubquery()) { + if (QueryUtils.getJoinAlgorithm(joinRel, plannerContext).requiresSubquery()) { return true; } diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/rel/DruidQuery.java b/sql/src/main/java/org/apache/druid/sql/calcite/rel/DruidQuery.java index 5d6fc2d2899a..72f72cd6d932 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/rel/DruidQuery.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/rel/DruidQuery.java @@ -873,7 +873,8 @@ static Pair getFiltration( joinDataSource.getConditionAnalysis(), joinDataSource.getJoinType(), leftFiltration.getDimFilter(), - joinableFactoryWrapper + joinableFactoryWrapper, + joinDataSource.getJoinAlgorithm() ); return Pair.of(newDataSource, queryFiltration); } else { diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/rule/DruidJoinRule.java b/sql/src/main/java/org/apache/druid/sql/calcite/rule/DruidJoinRule.java index 8d6230afcc8f..d76230da2a5e 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/rule/DruidJoinRule.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/rule/DruidJoinRule.java @@ -49,8 +49,10 @@ import org.apache.druid.common.config.NullHandling; import org.apache.druid.error.DruidException; import org.apache.druid.java.util.common.StringUtils; +import org.apache.druid.query.JoinAlgorithm; import org.apache.druid.query.LookupDataSource; import org.apache.druid.sql.calcite.planner.PlannerContext; +import org.apache.druid.sql.calcite.planner.QueryUtils; import org.apache.druid.sql.calcite.rel.DruidJoinQueryRel; import org.apache.druid.sql.calcite.rel.DruidQueryRel; import org.apache.druid.sql.calcite.rel.DruidRel; @@ -130,7 +132,8 @@ public void onMatch(RelOptRuleCall call) plannerContext.setPlanningError(conditionAnalysis.errorStr); final boolean isLeftDirectAccessPossible = enableLeftScanDirect && (left instanceof DruidQueryRel); - if (!plannerContext.getJoinAlgorithm().requiresSubquery() + final JoinAlgorithm joinAlgorithm = QueryUtils.getJoinAlgorithm(join, plannerContext); + if (!joinAlgorithm.requiresSubquery() && left.getPartialDruidQuery().stage() == PartialDruidQuery.Stage.SELECT_PROJECT && (isLeftDirectAccessPossible || left.getPartialDruidQuery().getWhereFilter() == null)) { // Swap the left-side projection above the join, so the left side is a simple scan or mapping. This helps us @@ -153,7 +156,7 @@ public void onMatch(RelOptRuleCall call) leftFilter = null; } - if (!plannerContext.getJoinAlgorithm().requiresSubquery() + if (!joinAlgorithm.requiresSubquery() && right.getPartialDruidQuery().stage() == PartialDruidQuery.Stage.SELECT_PROJECT && right.getPartialDruidQuery().getWhereFilter() == null && !right.getPartialDruidQuery().getSelectProject().isMapping() diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/run/EngineFeature.java b/sql/src/main/java/org/apache/druid/sql/calcite/run/EngineFeature.java index 97e81c2ef5c3..9bf4fac8e1e7 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/run/EngineFeature.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/run/EngineFeature.java @@ -19,6 +19,7 @@ package org.apache.druid.sql.calcite.run; +import org.apache.druid.query.JoinAlgorithm; import org.apache.druid.sql.calcite.external.ExternalDataSource; /** @@ -103,7 +104,7 @@ public enum EngineFeature UNNEST, /** - * Planner is permitted to use {@link org.apache.druid.sql.calcite.planner.JoinAlgorithm#BROADCAST} with RIGHT + * Planner is permitted to use {@link JoinAlgorithm#BROADCAST} with RIGHT * and FULL join. Not guaranteed to produce correct results in either the native or MSQ engines, but we allow * it in native for two reasons: legacy (the docs caution against it, but it's always been allowed), and the fact * that it actually *does* generate correct results in native when the join is processed on the Broker. It is much diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/run/NativeSqlEngine.java b/sql/src/main/java/org/apache/druid/sql/calcite/run/NativeSqlEngine.java index 4f3d86b1b420..16228a19e127 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/run/NativeSqlEngine.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/run/NativeSqlEngine.java @@ -27,12 +27,12 @@ import org.apache.calcite.rel.type.RelDataTypeFactory; import org.apache.druid.error.InvalidSqlInput; import org.apache.druid.guice.LazySingleton; +import org.apache.druid.query.JoinAlgorithm; import org.apache.druid.query.groupby.GroupByQuery; import org.apache.druid.query.timeboundary.TimeBoundaryQuery; import org.apache.druid.server.QueryLifecycleFactory; import org.apache.druid.sql.calcite.parser.DruidSqlInsert; import org.apache.druid.sql.calcite.parser.DruidSqlReplace; -import org.apache.druid.sql.calcite.planner.JoinAlgorithm; import org.apache.druid.sql.calcite.planner.PlannerContext; import org.apache.druid.sql.calcite.rel.DruidQuery; import org.apache.druid.sql.destination.IngestDestination; diff --git a/sql/src/test/java/org/apache/druid/quidem/DruidQuidemCommandHandler.java b/sql/src/test/java/org/apache/druid/quidem/DruidQuidemCommandHandler.java index a183ba9ecad1..5bf6d18050f5 100644 --- a/sql/src/test/java/org/apache/druid/quidem/DruidQuidemCommandHandler.java +++ b/sql/src/test/java/org/apache/druid/quidem/DruidQuidemCommandHandler.java @@ -26,7 +26,9 @@ import net.hydromatic.quidem.CommandHandler; import net.hydromatic.quidem.Quidem.SqlCommand; import org.apache.calcite.plan.RelOptUtil; +import org.apache.calcite.rel.RelHomogeneousShuttle; import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.hint.Hintable; import org.apache.calcite.runtime.Hook; import org.apache.calcite.sql.SqlExplainFormat; import org.apache.calcite.sql.SqlExplainLevel; @@ -59,6 +61,9 @@ public Command parseCommand(List lines, List content, String lin if (line.startsWith("druidPlan")) { return new DruidPlanCommand(lines, content); } + if (line.startsWith("hints")) { + return new HintPlanCommand(lines, content); + } if (line.startsWith("nativePlan")) { return new NativePlanCommand(lines, content); } @@ -202,10 +207,16 @@ protected final void executeExplain(Context context) throws IOException if (node instanceof DruidRel) { node = ((DruidRel) node).unwrapLogicalPlan(); } - String str = RelOptUtil.dumpPlan("", node, SqlExplainFormat.TEXT, SqlExplainLevel.EXPPLAN_ATTRIBUTES); + String str = convertRelToString(node); context.echo(ImmutableList.of(str)); } } + + protected String convertRelToString(RelNode node) + { + String str = RelOptUtil.dumpPlan("", node, SqlExplainFormat.TEXT, SqlExplainLevel.EXPPLAN_ATTRIBUTES); + return str; + } } /** @@ -245,6 +256,54 @@ static class DruidPlanCommand extends AbstractRelPlanCommand } } + static class HintPlanCommand extends AbstractRelPlanCommand + { + HintPlanCommand(List lines, List content) + { + super(lines, content, DruidHook.DRUID_PLAN); + } + + @Override + protected String convertRelToString(RelNode node) + { + final HintCollector collector = new HintCollector(); + node.accept(collector); + return collector.getCollectedHintsAsString(); + } + + private static class HintCollector extends RelHomogeneousShuttle + { + private final List hintsCollect; + + HintCollector() + { + this.hintsCollect = new ArrayList<>(); + } + + @Override + public RelNode visit(RelNode relNode) + { + if (relNode instanceof Hintable) { + Hintable hintableRelNode = (Hintable) relNode; + if (!hintableRelNode.getHints().isEmpty()) { + this.hintsCollect.add(relNode.getClass().getSimpleName() + ":" + hintableRelNode.getHints()); + } + } + return super.visit(relNode); + } + + public String getCollectedHintsAsString() + { + StringBuilder builder = new StringBuilder(); + for (String hintLine : hintsCollect) { + builder.append(hintLine).append("\n"); + } + + return builder.toString(); + } + } + } + static class ConvertedPlanCommand extends AbstractRelPlanCommand { ConvertedPlanCommand(List lines, List content) diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/BaseCalciteQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/BaseCalciteQueryTest.java index 95d3d5228938..30a5c6d2cc7a 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/BaseCalciteQueryTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/BaseCalciteQueryTest.java @@ -46,6 +46,7 @@ import org.apache.druid.math.expr.ExpressionProcessing; import org.apache.druid.query.DataSource; import org.apache.druid.query.Druids; +import org.apache.druid.query.JoinAlgorithm; import org.apache.druid.query.JoinDataSource; import org.apache.druid.query.Query; import org.apache.druid.query.QueryContexts; @@ -588,7 +589,8 @@ public static JoinDataSource join( String rightPrefix, String condition, JoinType joinType, - DimFilter filter + DimFilter filter, + JoinAlgorithm joinAlgorithm ) { return JoinDataSource.create( @@ -599,7 +601,28 @@ public static JoinDataSource join( joinType, filter, CalciteTests.createExprMacroTable(), - CalciteTests.createJoinableFactoryWrapper() + CalciteTests.createJoinableFactoryWrapper(), + joinAlgorithm + ); + } + + public static JoinDataSource join( + DataSource left, + DataSource right, + String rightPrefix, + String condition, + JoinType joinType, + DimFilter filter + ) + { + return join( + left, + right, + rightPrefix, + condition, + joinType, + filter, + JoinAlgorithm.BROADCAST ); } diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteJoinQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteJoinQueryTest.java index 02a1a3fe2816..fb4be084efcf 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteJoinQueryTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteJoinQueryTest.java @@ -39,6 +39,7 @@ import org.apache.druid.query.FilteredDataSource; import org.apache.druid.query.GlobalTableDataSource; import org.apache.druid.query.InlineDataSource; +import org.apache.druid.query.JoinAlgorithm; import org.apache.druid.query.JoinDataSource; import org.apache.druid.query.LookupDataSource; import org.apache.druid.query.Order; @@ -163,7 +164,8 @@ public void testInnerJoinWithLimitAndAlias() JoinType.INNER, null, ExprMacroTable.nil(), - CalciteTests.createJoinableFactoryWrapper() + CalciteTests.createJoinableFactoryWrapper(), + JoinAlgorithm.BROADCAST ) ) .columns("d0") @@ -222,7 +224,8 @@ public void testExactTopNOnInnerJoinWithLimit() JoinType.INNER, null, ExprMacroTable.nil(), - CalciteTests.createJoinableFactoryWrapper() + CalciteTests.createJoinableFactoryWrapper(), + JoinAlgorithm.BROADCAST ) ) .context(context) @@ -3624,7 +3627,8 @@ public void testLeftJoinRightTableCanBeEmpty() JoinType.LEFT, null, ExprMacroTable.nil(), - CalciteTests.createJoinableFactoryWrapper() + CalciteTests.createJoinableFactoryWrapper(), + JoinAlgorithm.BROADCAST ) ) .setInterval(querySegmentSpec(Filtration.eternity())) @@ -5725,7 +5729,8 @@ public void testPlanWithInFilterMoreThanInSubQueryThreshold() JoinType.INNER, null, ExprMacroTable.nil(), - CalciteTests.createJoinableFactoryWrapper() + CalciteTests.createJoinableFactoryWrapper(), + JoinAlgorithm.BROADCAST ) ) .columns("l1") @@ -5973,7 +5978,8 @@ public void testJoinWithAliasAndOrderByNoGroupBy() JoinType.INNER, null, ExprMacroTable.nil(), - CalciteTests.createJoinableFactoryWrapper() + CalciteTests.createJoinableFactoryWrapper(), + JoinAlgorithm.BROADCAST ) ) .columns("__time") diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java index f2a85b4a4a90..b4bda89e13b3 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java @@ -41,6 +41,7 @@ import org.apache.druid.math.expr.ExprMacroTable; import org.apache.druid.query.Druids; import org.apache.druid.query.InlineDataSource; +import org.apache.druid.query.JoinAlgorithm; import org.apache.druid.query.JoinDataSource; import org.apache.druid.query.LookupDataSource; import org.apache.druid.query.OperatorFactoryBuilders; @@ -15542,7 +15543,8 @@ public void testOrderByAlongWithInternalScanQuery() JoinType.INNER, null, ExprMacroTable.nil(), - CalciteTests.createJoinableFactoryWrapper() + CalciteTests.createJoinableFactoryWrapper(), + JoinAlgorithm.BROADCAST ) ) .context(QUERY_CONTEXT_DEFAULT) @@ -15585,7 +15587,8 @@ public void testOrderByAlongWithInternalScanQueryNoDistinct() JoinType.INNER, null, ExprMacroTable.nil(), - CalciteTests.createJoinableFactoryWrapper() + CalciteTests.createJoinableFactoryWrapper(), + JoinAlgorithm.BROADCAST ) ) .context(QUERY_CONTEXT_DEFAULT) diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteSubqueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteSubqueryTest.java index 8117e6ff02bf..38ac3f2b2912 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteSubqueryTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteSubqueryTest.java @@ -36,6 +36,7 @@ import org.apache.druid.java.util.common.granularity.PeriodGranularity; import org.apache.druid.math.expr.ExprMacroTable; import org.apache.druid.query.Druids; +import org.apache.druid.query.JoinAlgorithm; import org.apache.druid.query.JoinDataSource; import org.apache.druid.query.Order; import org.apache.druid.query.QueryContexts; @@ -977,7 +978,8 @@ public void testJoinWithTimeDimension(String testName, Map query JoinType.INNER, null, ExprMacroTable.nil(), - CalciteTests.createJoinableFactoryWrapper() + CalciteTests.createJoinableFactoryWrapper(), + JoinAlgorithm.BROADCAST )) .intervals(querySegmentSpec(Filtration.eternity())) .granularity(Granularities.ALL) @@ -1108,7 +1110,8 @@ public void testJoinWithSubqueries(String testName, Map queryCon JoinType.LEFT, null, ExprMacroTable.nil(), - CalciteTests.createJoinableFactoryWrapper() + CalciteTests.createJoinableFactoryWrapper(), + JoinAlgorithm.BROADCAST ), new QueryDataSource( newScanQueryBuilder() @@ -1128,7 +1131,8 @@ public void testJoinWithSubqueries(String testName, Map queryCon JoinType.LEFT, null, ExprMacroTable.nil(), - CalciteTests.createJoinableFactoryWrapper() + CalciteTests.createJoinableFactoryWrapper(), + JoinAlgorithm.BROADCAST ) ) .columns("dim1", "j0.dim2") @@ -1141,7 +1145,8 @@ public void testJoinWithSubqueries(String testName, Map queryCon JoinType.FULL, null, ExprMacroTable.nil(), - CalciteTests.createJoinableFactoryWrapper() + CalciteTests.createJoinableFactoryWrapper(), + JoinAlgorithm.BROADCAST ) ) .columns("dim1", "_j0.j0.dim2") @@ -1468,7 +1473,8 @@ public void testGroupBySubqueryWithEarliestAggregator(String testName, Map