From 485c719fe9c5400e99cce7c82f5334fd7b655178 Mon Sep 17 00:00:00 2001 From: minghong Date: Mon, 9 Sep 2024 20:28:05 +0800 Subject: [PATCH 01/14] disable join reorder if any table row count is not available --- .../apache/doris/nereids/NereidsPlanner.java | 16 ++++++++++ .../doris/nereids/stats/StatsCalculator.java | 30 +++++++++++++------ 2 files changed, 37 insertions(+), 9 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java index 25d0dedd58dc95..fd42420df6f925 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java @@ -48,6 +48,7 @@ import org.apache.doris.nereids.processor.pre.PlanPreprocessors; import org.apache.doris.nereids.properties.PhysicalProperties; import org.apache.doris.nereids.rules.exploration.mv.MaterializationContext; +import org.apache.doris.nereids.stats.StatsCalculator; import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.SlotReference; import org.apache.doris.nereids.trees.plans.ComputeResultSet; @@ -56,6 +57,7 @@ import org.apache.doris.nereids.trees.plans.distribute.DistributePlanner; import org.apache.doris.nereids.trees.plans.distribute.DistributedPlan; import org.apache.doris.nereids.trees.plans.distribute.FragmentIdMapping; +import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.nereids.trees.plans.logical.LogicalSqlCache; import org.apache.doris.nereids.trees.plans.physical.PhysicalPlan; @@ -255,6 +257,8 @@ private Plan planWithoutLock( return rewrittenPlan; } } + List scans = getAllOlapScans(cascadesContext.getRewritePlan()); + StatsCalculator.disableJoinReorderIfTableRowCountNotAvailable(scans, cascadesContext); optimize(); if (statementContext.getConnectContext().getExecutor() != null) { @@ -288,6 +292,18 @@ private Plan planWithoutLock( return physicalPlan; } + private List getAllOlapScans(Plan plan) { + List scans = Lists.newArrayList(); + if (plan instanceof LogicalOlapScan) { + scans.add((LogicalOlapScan) plan); + } else { + for (Plan child : plan.children()) { + scans.addAll(getAllOlapScans(child)); + } + } + return scans; + } + private LogicalPlan preprocess(LogicalPlan logicalPlan) { return new PlanPreprocessors(statementContext).process(logicalPlan); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java index 5946192a27eff9..7710926c2cda21 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java @@ -182,6 +182,11 @@ public class StatsCalculator extends DefaultPlanVisitor { private CascadesContext cascadesContext; + private StatsCalculator(CascadesContext context) { + this.groupExpression = null; + this.cascadesContext = context; + } + private StatsCalculator(GroupExpression groupExpression, boolean forbidUnknownColStats, Map columnStatisticMap, boolean isPlayNereidsDump, Map cteIdToStats, CascadesContext context) { @@ -205,6 +210,22 @@ public Map getTotalColumnStatisticMap() { return totalColumnStatisticMap; } + /** + * disable join reorder if any table row count is not available. + */ + public static void disableJoinReorderIfTableRowCountNotAvailable( + List scans, CascadesContext context) { + StatsCalculator calculator = new StatsCalculator(context); + for (LogicalOlapScan scan : scans) { + double rowCount = calculator.getOlapTableRowCount(scan); + if (rowCount == -1 && ConnectContext.get() != null) { + LOG.info("disable join reorder since row count not available: " + + scan.getTable().getNameWithFullQualifiers()); + ConnectContext.get().getSessionVariable().setDisableJoinReorder(true); + } + } + } + /** * estimate stats */ @@ -217,15 +238,6 @@ public static StatsCalculator estimate(GroupExpression groupExpression, boolean return statsCalculator; } - public static StatsCalculator estimate(GroupExpression groupExpression, boolean forbidUnknownColStats, - Map columnStatisticMap, boolean isPlayNereidsDump, CascadesContext context) { - return StatsCalculator.estimate(groupExpression, - forbidUnknownColStats, - columnStatisticMap, - isPlayNereidsDump, - new HashMap<>(), context); - } - // For unit test only public static void estimate(GroupExpression groupExpression, CascadesContext context) { StatsCalculator statsCalculator = new StatsCalculator(groupExpression, false, From 9f612a2fdb5921447d58f259b0e8c0dc98444606 Mon Sep 17 00:00:00 2001 From: minghong Date: Tue, 10 Sep 2024 17:26:28 +0800 Subject: [PATCH 02/14] skip if ut or hint --- .../java/org/apache/doris/nereids/NereidsPlanner.java | 8 ++++++-- .../org/apache/doris/nereids/stats/StatsCalculator.java | 1 + 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java index fd42420df6f925..eaa9413710cf78 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java @@ -22,6 +22,7 @@ import org.apache.doris.analysis.StatementBase; import org.apache.doris.catalog.Column; import org.apache.doris.catalog.TableIf; +import org.apache.doris.common.FeConstants; import org.apache.doris.common.FormatOptions; import org.apache.doris.common.NereidsException; import org.apache.doris.common.Pair; @@ -257,8 +258,11 @@ private Plan planWithoutLock( return rewrittenPlan; } } - List scans = getAllOlapScans(cascadesContext.getRewritePlan()); - StatsCalculator.disableJoinReorderIfTableRowCountNotAvailable(scans, cascadesContext); + + if (!FeConstants.runningUnitTest && !cascadesContext.isLeadingDisableJoinReorder()) { + List scans = getAllOlapScans(cascadesContext.getRewritePlan()); + StatsCalculator.disableJoinReorderIfTableRowCountNotAvailable(scans, cascadesContext); + } optimize(); if (statementContext.getConnectContext().getExecutor() != null) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java index 7710926c2cda21..15a973a19d5e3d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java @@ -222,6 +222,7 @@ public static void disableJoinReorderIfTableRowCountNotAvailable( LOG.info("disable join reorder since row count not available: " + scan.getTable().getNameWithFullQualifiers()); ConnectContext.get().getSessionVariable().setDisableJoinReorder(true); + return; } } } From 112b4d7e780586064ee9f2f45f814a2a7f8877c5 Mon Sep 17 00:00:00 2001 From: minghong Date: Wed, 11 Sep 2024 19:06:31 +0800 Subject: [PATCH 03/14] feconstant --- .../test/java/org/apache/doris/utframe/TestWithFeService.java | 1 + 1 file changed, 1 insertion(+) diff --git a/fe/fe-core/src/test/java/org/apache/doris/utframe/TestWithFeService.java b/fe/fe-core/src/test/java/org/apache/doris/utframe/TestWithFeService.java index 407c1544a4b00b..a73ae7d8fe55b2 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/utframe/TestWithFeService.java +++ b/fe/fe-core/src/test/java/org/apache/doris/utframe/TestWithFeService.java @@ -153,6 +153,7 @@ public Set getEnableNereidsRules() { @BeforeAll public final void beforeAll() throws Exception { FeConstants.enableInternalSchemaDb = false; + FeConstants.runningUnitTest=true; beforeCreatingConnectContext(); connectContext = createDefaultCtx(); beforeCluster(); From 823f2e6a850a116f5325163a60892b823ed96f82 Mon Sep 17 00:00:00 2001 From: minghong Date: Wed, 11 Sep 2024 19:08:56 +0800 Subject: [PATCH 04/14] add-commm --- .../java/org/apache/doris/nereids/stats/StatsCalculator.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java index 15a973a19d5e3d..c0598b01ede028 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java @@ -377,6 +377,9 @@ private void checkIfUnknownStatsUsedAsKey(StatisticsBuilder builder) { } } + /** + * if the table is not analyzed and BE does not report row count, return -1 + */ private double getOlapTableRowCount(OlapScan olapScan) { OlapTable olapTable = olapScan.getTable(); AnalysisManager analysisManager = Env.getCurrentEnv().getAnalysisManager(); From 99e930da5a372889f7bcb4752f37cbdae924d36c Mon Sep 17 00:00:00 2001 From: minghong Date: Wed, 11 Sep 2024 19:14:43 +0800 Subject: [PATCH 05/14] fmt --- .../test/java/org/apache/doris/utframe/TestWithFeService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fe/fe-core/src/test/java/org/apache/doris/utframe/TestWithFeService.java b/fe/fe-core/src/test/java/org/apache/doris/utframe/TestWithFeService.java index a73ae7d8fe55b2..e74adb1c3e2cba 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/utframe/TestWithFeService.java +++ b/fe/fe-core/src/test/java/org/apache/doris/utframe/TestWithFeService.java @@ -153,7 +153,7 @@ public Set getEnableNereidsRules() { @BeforeAll public final void beforeAll() throws Exception { FeConstants.enableInternalSchemaDb = false; - FeConstants.runningUnitTest=true; + FeConstants.runningUnitTest = true; beforeCreatingConnectContext(); connectContext = createDefaultCtx(); beforeCluster(); From 8be2e40ddb21ad17171cf9ad38be37efa658b821 Mon Sep 17 00:00:00 2001 From: minghong Date: Wed, 11 Sep 2024 20:44:40 +0800 Subject: [PATCH 06/14] fix rt: disable_join reorder --- .../push_down_filter/extract_from_disjunction_in_join.groovy | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/regression-test/suites/nereids_rules_p0/push_down_filter/extract_from_disjunction_in_join.groovy b/regression-test/suites/nereids_rules_p0/push_down_filter/extract_from_disjunction_in_join.groovy index 858f39e5e65cf2..c305f876f80d9a 100644 --- a/regression-test/suites/nereids_rules_p0/push_down_filter/extract_from_disjunction_in_join.groovy +++ b/regression-test/suites/nereids_rules_p0/push_down_filter/extract_from_disjunction_in_join.groovy @@ -20,7 +20,10 @@ suite("extract_from_disjunction_in_join") { sql "SET enable_fallback_to_original_planner=false" sql "set ignore_shape_nodes='PhysicalDistribute,PhysicalProject'" sql "set disable_nereids_rules=PRUNE_EMPTY_PARTITION" - sql "set runtime_filter_mode=OFF" + sql """ + set runtime_filter_mode=OFF; + set disable_join_reorder=true; + """ sql "drop table if exists extract_from_disjunction_in_join_t1" From d3172d74ef273e4a1d5e2f28cc72b6a139c6fe41 Mon Sep 17 00:00:00 2001 From: minghong Date: Thu, 12 Sep 2024 10:39:15 +0800 Subject: [PATCH 07/14] fixut --- .../apache/doris/nereids/NereidsPlanner.java | 22 +++++++------------ .../doris/utframe/TestWithFeService.java | 1 - 2 files changed, 8 insertions(+), 15 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java index eaa9413710cf78..b51f9ce24a3e4d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java @@ -259,8 +259,14 @@ private Plan planWithoutLock( } } - if (!FeConstants.runningUnitTest && !cascadesContext.isLeadingDisableJoinReorder()) { - List scans = getAllOlapScans(cascadesContext.getRewritePlan()); + // if we cannot get table row count, skip join reorder + // except: + // 1. user set leading hint + // 2. ut test. In ut test, FeConstants.enableInternalSchemaDb is false or FeConstants.runningUnitTest is true + if (FeConstants.enableInternalSchemaDb && !FeConstants.runningUnitTest + && !cascadesContext.isLeadingDisableJoinReorder()) { + List scans = cascadesContext.getRewritePlan() + .collectToList(LogicalOlapScan.class::isInstance); StatsCalculator.disableJoinReorderIfTableRowCountNotAvailable(scans, cascadesContext); } @@ -296,18 +302,6 @@ private Plan planWithoutLock( return physicalPlan; } - private List getAllOlapScans(Plan plan) { - List scans = Lists.newArrayList(); - if (plan instanceof LogicalOlapScan) { - scans.add((LogicalOlapScan) plan); - } else { - for (Plan child : plan.children()) { - scans.addAll(getAllOlapScans(child)); - } - } - return scans; - } - private LogicalPlan preprocess(LogicalPlan logicalPlan) { return new PlanPreprocessors(statementContext).process(logicalPlan); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/utframe/TestWithFeService.java b/fe/fe-core/src/test/java/org/apache/doris/utframe/TestWithFeService.java index e74adb1c3e2cba..407c1544a4b00b 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/utframe/TestWithFeService.java +++ b/fe/fe-core/src/test/java/org/apache/doris/utframe/TestWithFeService.java @@ -153,7 +153,6 @@ public Set getEnableNereidsRules() { @BeforeAll public final void beforeAll() throws Exception { FeConstants.enableInternalSchemaDb = false; - FeConstants.runningUnitTest = true; beforeCreatingConnectContext(); connectContext = createDefaultCtx(); beforeCluster(); From 4eefd063f84e2f0df5acc94916a5919f0c3b2982 Mon Sep 17 00:00:00 2001 From: minghong Date: Wed, 18 Sep 2024 19:50:24 +0800 Subject: [PATCH 08/14] rt --- .../data/nereids_p0/hint/multi_leading.out | 94 +++++++++---------- .../data/nereids_p0/hint/test_hint.out | 12 +-- .../eager_aggregate/basic.out | 4 +- .../eager_aggregate/basic_one_side.out | 4 +- .../eliminate_outer_join.out | 40 ++++---- .../infer_intersect_except.out | 4 +- .../infer_set_operator_distinct.out | 26 +++-- .../extract_from_disjunction_in_join.out | 14 +-- .../extract_from_disjunction_in_join.groovy | 1 + 9 files changed, 99 insertions(+), 100 deletions(-) diff --git a/regression-test/data/nereids_p0/hint/multi_leading.out b/regression-test/data/nereids_p0/hint/multi_leading.out index 08b6b83ed584f5..ce74020695d0c8 100644 --- a/regression-test/data/nereids_p0/hint/multi_leading.out +++ b/regression-test/data/nereids_p0/hint/multi_leading.out @@ -24,17 +24,17 @@ PhysicalResultSink ----PhysicalDistribute[DistributionSpecGather] ------hashAgg[LOCAL] --------hashJoin[INNER_JOIN colocated] hashCondition=((cte.c1 = t1.c1)) otherCondition=() -----------filter((t1.c1 > 300)) -------------PhysicalOlapScan[t1] ----------hashJoin[INNER_JOIN broadcast] hashCondition=((cte.c1 = t2.c2)) otherCondition=() ------------filter((cte.c1 > 300)) --------------PhysicalOlapScan[t1] ------------filter((t2.c2 > 300)) --------------PhysicalOlapScan[t2] +----------filter((t1.c1 > 300)) +------------PhysicalOlapScan[t1] Hint log: -Used: leading(t1 t2 ) leading(t1 cte ) -UnUsed: +Used: +UnUsed: leading(t1 t2) leading(t1 cte) SyntaxError: -- !sql1_4 -- @@ -43,17 +43,17 @@ PhysicalResultSink ----PhysicalDistribute[DistributionSpecGather] ------hashAgg[LOCAL] --------hashJoin[INNER_JOIN colocated] hashCondition=((cte.c1 = t1.c1)) otherCondition=() -----------filter((t1.c1 > 300)) -------------PhysicalOlapScan[t1] ----------hashJoin[INNER_JOIN broadcast] hashCondition=((cte.c1 = t2.c2)) otherCondition=() ------------filter((cte.c1 > 300)) --------------PhysicalOlapScan[t1] ------------filter((t2.c2 > 300)) --------------PhysicalOlapScan[t2] +----------filter((t1.c1 > 300)) +------------PhysicalOlapScan[t1] Hint log: -Used: leading(t1 t2 ) leading(t1 cte ) -UnUsed: +Used: +UnUsed: leading(t1 t2) leading(t1 cte) SyntaxError: -- !sql1_res_1 -- @@ -74,14 +74,14 @@ PhysicalResultSink ----PhysicalDistribute[DistributionSpecGather] ------hashAgg[LOCAL] --------hashJoin[INNER_JOIN broadcast] hashCondition=((alias1.c1 = t3.c3)) otherCondition=() -----------PhysicalOlapScan[t3] ----------hashJoin[INNER_JOIN broadcast] hashCondition=((t1.c1 = t2.c2)) otherCondition=() ------------PhysicalOlapScan[t1] ------------PhysicalOlapScan[t2] +----------PhysicalOlapScan[t3] Hint log: -Used: leading(t3 alias1 ) -UnUsed: +Used: +UnUsed: leading(t3 alias1) SyntaxError: -- !sql2_3 -- @@ -91,13 +91,13 @@ PhysicalResultSink ------hashAgg[LOCAL] --------hashJoin[INNER_JOIN broadcast] hashCondition=((alias1.c1 = t3.c3)) otherCondition=() ----------hashJoin[INNER_JOIN broadcast] hashCondition=((alias1.c1 = t2.c2)) otherCondition=() -------------PhysicalOlapScan[t2] ------------PhysicalOlapScan[t1] +------------PhysicalOlapScan[t2] ----------PhysicalOlapScan[t3] Hint log: -Used: leading(t2 t1 ) -UnUsed: +Used: +UnUsed: leading(t2 t1) SyntaxError: -- !sql2_4 -- @@ -106,14 +106,14 @@ PhysicalResultSink ----PhysicalDistribute[DistributionSpecGather] ------hashAgg[LOCAL] --------hashJoin[INNER_JOIN broadcast] hashCondition=((alias1.c1 = t3.c3)) otherCondition=() -----------PhysicalOlapScan[t3] ----------hashJoin[INNER_JOIN broadcast] hashCondition=((alias1.c1 = t2.c2)) otherCondition=() -------------PhysicalOlapScan[t2] ------------PhysicalOlapScan[t1] +------------PhysicalOlapScan[t2] +----------PhysicalOlapScan[t3] Hint log: -Used: leading(t2 t1 ) leading(t3 alias1 ) -UnUsed: +Used: +UnUsed: leading(t2 t1) leading(t3 alias1) SyntaxError: -- !sql2_res_1 -- @@ -135,17 +135,17 @@ PhysicalResultSink ------hashAgg[LOCAL] --------hashJoin[INNER_JOIN broadcast] hashCondition=((alias1.c1 = cte.c11)) otherCondition=() ----------hashJoin[INNER_JOIN broadcast] hashCondition=((alias1.c1 = t3.c3)) otherCondition=() -------------PhysicalOlapScan[t3] ------------hashJoin[INNER_JOIN broadcast] hashCondition=((t1.c1 = t2.c2)) otherCondition=() --------------PhysicalOlapScan[t1] --------------PhysicalOlapScan[t2] +------------PhysicalOlapScan[t3] ----------hashJoin[INNER_JOIN broadcast] hashCondition=((cte.c1 = t2.c2)) otherCondition=() -------------PhysicalOlapScan[t2] ------------PhysicalOlapScan[t1] +------------PhysicalOlapScan[t2] Hint log: -Used: leading(t2 t1 ) leading(t3 alias1 cte ) -UnUsed: +Used: +UnUsed: leading(t2 t1) leading(t3 alias1 cte) SyntaxError: -- !sql3_3 -- @@ -156,16 +156,16 @@ PhysicalResultSink --------hashJoin[INNER_JOIN broadcast] hashCondition=((alias1.c1 = cte.c11)) otherCondition=() ----------hashJoin[INNER_JOIN broadcast] hashCondition=((alias1.c1 = t3.c3)) otherCondition=() ------------hashJoin[INNER_JOIN broadcast] hashCondition=((alias1.c1 = t2.c2)) otherCondition=() ---------------PhysicalOlapScan[t2] --------------PhysicalOlapScan[t1] +--------------PhysicalOlapScan[t2] ------------PhysicalOlapScan[t3] ----------hashJoin[INNER_JOIN broadcast] hashCondition=((cte.c1 = t2.c2)) otherCondition=() ------------PhysicalOlapScan[t1] ------------PhysicalOlapScan[t2] Hint log: -Used: leading(t2 t1 ) -UnUsed: +Used: +UnUsed: leading(t2 t1) SyntaxError: -- !sql3_4 -- @@ -175,17 +175,17 @@ PhysicalResultSink ------hashAgg[LOCAL] --------hashJoin[INNER_JOIN broadcast] hashCondition=((alias1.c1 = cte.c11)) otherCondition=() ----------hashJoin[INNER_JOIN broadcast] hashCondition=((alias1.c1 = t3.c3)) otherCondition=() -------------PhysicalOlapScan[t3] ------------hashJoin[INNER_JOIN broadcast] hashCondition=((alias1.c1 = t2.c2)) otherCondition=() ---------------PhysicalOlapScan[t2] --------------PhysicalOlapScan[t1] +--------------PhysicalOlapScan[t2] +------------PhysicalOlapScan[t3] ----------hashJoin[INNER_JOIN broadcast] hashCondition=((cte.c1 = t2.c2)) otherCondition=() -------------PhysicalOlapScan[t2] ------------PhysicalOlapScan[t1] +------------PhysicalOlapScan[t2] Hint log: -Used: leading(t2 t1 ) leading(t2 t1 ) leading(t3 alias1 cte ) -UnUsed: +Used: +UnUsed: leading(t2 t1) leading(t2 t1) leading(t3 alias1 cte) SyntaxError: -- !sql3_res_1 -- @@ -206,16 +206,16 @@ PhysicalResultSink ----PhysicalDistribute[DistributionSpecGather] ------hashAgg[LOCAL] --------hashJoin[INNER_JOIN broadcast] hashCondition=((alias1.c1 = t3.c3)) otherCondition=() -----------PhysicalOlapScan[t3] ----------hashJoin[INNER_JOIN broadcast] hashCondition=((t1.c1 = alias2.c2)) otherCondition=() ------------PhysicalOlapScan[t1] ------------hashJoin[INNER_JOIN broadcast] hashCondition=((t2.c2 = t4.c4)) otherCondition=() --------------PhysicalOlapScan[t2] --------------PhysicalOlapScan[t4] +----------PhysicalOlapScan[t3] Hint log: -Used: leading(t3 alias1 ) -UnUsed: +Used: +UnUsed: leading(t3 alias1) SyntaxError: -- !sql4_2 -- @@ -225,15 +225,15 @@ PhysicalResultSink ------hashAgg[LOCAL] --------hashJoin[INNER_JOIN broadcast] hashCondition=((alias1.c1 = t3.c3)) otherCondition=() ----------hashJoin[INNER_JOIN broadcast] hashCondition=((alias1.c1 = alias2.c2)) otherCondition=() +------------PhysicalOlapScan[t1] ------------hashJoin[INNER_JOIN broadcast] hashCondition=((t2.c2 = t4.c4)) otherCondition=() --------------PhysicalOlapScan[t2] --------------PhysicalOlapScan[t4] -------------PhysicalOlapScan[t1] ----------PhysicalOlapScan[t3] Hint log: -Used: leading(alias2 t1 ) -UnUsed: +Used: +UnUsed: leading(alias2 t1) SyntaxError: -- !sql4_3 -- @@ -245,13 +245,13 @@ PhysicalResultSink ----------hashJoin[INNER_JOIN broadcast] hashCondition=((t1.c1 = alias2.c2)) otherCondition=() ------------PhysicalOlapScan[t1] ------------hashJoin[INNER_JOIN broadcast] hashCondition=((t2.c2 = t4.c4)) otherCondition=() ---------------PhysicalOlapScan[t4] --------------PhysicalOlapScan[t2] +--------------PhysicalOlapScan[t4] ----------PhysicalOlapScan[t3] Hint log: -Used: leading(t4 t2 ) -UnUsed: +Used: +UnUsed: leading(t4 t2) SyntaxError: -- !sql4_4 -- @@ -260,16 +260,16 @@ PhysicalResultSink ----PhysicalDistribute[DistributionSpecGather] ------hashAgg[LOCAL] --------hashJoin[INNER_JOIN broadcast] hashCondition=((alias1.c1 = t3.c3)) otherCondition=() -----------PhysicalOlapScan[t3] ----------hashJoin[INNER_JOIN broadcast] hashCondition=((alias1.c1 = alias2.c2)) otherCondition=() +------------PhysicalOlapScan[t1] ------------hashJoin[INNER_JOIN broadcast] hashCondition=((t2.c2 = t4.c4)) otherCondition=() --------------PhysicalOlapScan[t2] --------------PhysicalOlapScan[t4] -------------PhysicalOlapScan[t1] +----------PhysicalOlapScan[t3] Hint log: -Used: leading(alias2 t1 ) leading(t3 alias1 ) -UnUsed: +Used: +UnUsed: leading(alias2 t1) leading(t3 alias1) SyntaxError: -- !sql4_res_0 -- @@ -311,12 +311,12 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) --------hashAgg[GLOBAL] ----------PhysicalDistribute[DistributionSpecGather] ------------hashAgg[LOCAL] ---------------hashJoin[INNER_JOIN shuffle] hashCondition=((t1.c1 = cte.c11)) otherCondition=() -----------------PhysicalCteConsumer ( cteId=CTEId#0 ) +--------------hashJoin[INNER_JOIN broadcast] hashCondition=((t1.c1 = cte.c11)) otherCondition=() ----------------PhysicalOlapScan[t1] +----------------PhysicalCteConsumer ( cteId=CTEId#0 ) Hint log: -Used: leading(cte t1 ) -UnUsed: +Used: +UnUsed: leading(cte t1) SyntaxError: diff --git a/regression-test/data/nereids_p0/hint/test_hint.out b/regression-test/data/nereids_p0/hint/test_hint.out index 66a218b09fe45d..f7128e7d15cf26 100644 --- a/regression-test/data/nereids_p0/hint/test_hint.out +++ b/regression-test/data/nereids_p0/hint/test_hint.out @@ -40,12 +40,12 @@ PhysicalResultSink ----PhysicalDistribute[DistributionSpecGather] ------hashAgg[LOCAL] --------hashJoin[INNER_JOIN broadcast] hashCondition=((t1.c1 = t2.c2)) otherCondition=() -----------PhysicalOlapScan[t2] ----------PhysicalOlapScan[t1] +----------PhysicalOlapScan[t2] Hint log: -Used: leading(t2 broadcast t1 ) -UnUsed: +Used: +UnUsed: leading(t2 broadcast t1) SyntaxError: -- !select1_6 -- @@ -54,12 +54,12 @@ PhysicalResultSink ----PhysicalDistribute[DistributionSpecGather] ------hashAgg[LOCAL] --------hashJoin[INNER_JOIN broadcast] hashCondition=((t1.c1 = t2.c2)) otherCondition=() -----------PhysicalOlapScan[t2] ----------PhysicalOlapScan[t1] +----------PhysicalOlapScan[t2] Hint log: -Used: leading(t2 broadcast t1 ) -UnUsed: +Used: +UnUsed: leading(t2 broadcast t1) SyntaxError: -- !select1_7 -- diff --git a/regression-test/data/nereids_rules_p0/eager_aggregate/basic.out b/regression-test/data/nereids_rules_p0/eager_aggregate/basic.out index 3e3986c75fc91c..ba18189efcad82 100644 --- a/regression-test/data/nereids_rules_p0/eager_aggregate/basic.out +++ b/regression-test/data/nereids_rules_p0/eager_aggregate/basic.out @@ -32,8 +32,8 @@ PhysicalResultSink --hashAgg[GLOBAL] ----hashAgg[LOCAL] ------hashJoin[INNER_JOIN] hashCondition=((a.device_id = b.device_id)) otherCondition=() ---------PhysicalOlapScan[shunt_log_com_dd_library] --------PhysicalOlapScan[com_dd_library] +--------PhysicalOlapScan[shunt_log_com_dd_library] -- !with_hint_1 -- PhysicalResultSink @@ -83,8 +83,8 @@ PhysicalResultSink --hashAgg[GLOBAL] ----hashAgg[LOCAL] ------hashJoin[INNER_JOIN] hashCondition=((a.device_id = b.device_id)) otherCondition=() ---------PhysicalOlapScan[shunt_log_com_dd_library] --------PhysicalOlapScan[com_dd_library] +--------PhysicalOlapScan[shunt_log_com_dd_library] Hint log: Used: diff --git a/regression-test/data/nereids_rules_p0/eager_aggregate/basic_one_side.out b/regression-test/data/nereids_rules_p0/eager_aggregate/basic_one_side.out index 49f1cc9617a090..aaf6afeca1e9f6 100644 --- a/regression-test/data/nereids_rules_p0/eager_aggregate/basic_one_side.out +++ b/regression-test/data/nereids_rules_p0/eager_aggregate/basic_one_side.out @@ -32,8 +32,8 @@ PhysicalResultSink --hashAgg[GLOBAL] ----hashAgg[LOCAL] ------hashJoin[INNER_JOIN] hashCondition=((a.device_id = b.device_id)) otherCondition=() ---------PhysicalOlapScan[shunt_log_com_dd_library_one_side] --------PhysicalOlapScan[com_dd_library_one_side] +--------PhysicalOlapScan[shunt_log_com_dd_library_one_side] -- !with_hint_1 -- PhysicalResultSink @@ -83,8 +83,8 @@ PhysicalResultSink --hashAgg[GLOBAL] ----hashAgg[LOCAL] ------hashJoin[INNER_JOIN] hashCondition=((a.device_id = b.device_id)) otherCondition=() ---------PhysicalOlapScan[shunt_log_com_dd_library_one_side] --------PhysicalOlapScan[com_dd_library_one_side] +--------PhysicalOlapScan[shunt_log_com_dd_library_one_side] Hint log: Used: diff --git a/regression-test/data/nereids_rules_p0/eliminate_outer_join/eliminate_outer_join.out b/regression-test/data/nereids_rules_p0/eliminate_outer_join/eliminate_outer_join.out index eff17e438e33c3..d92655e4e73e0b 100644 --- a/regression-test/data/nereids_rules_p0/eliminate_outer_join/eliminate_outer_join.out +++ b/regression-test/data/nereids_rules_p0/eliminate_outer_join/eliminate_outer_join.out @@ -11,17 +11,17 @@ PhysicalResultSink PhysicalResultSink --PhysicalDistribute[DistributionSpecGather] ----hashJoin[INNER_JOIN colocated] hashCondition=((t1.id = t2.id)) otherCondition=() -------PhysicalOlapScan[t] ------filter((t1.score > 10)) --------PhysicalOlapScan[t] +------PhysicalOlapScan[t] -- !full_outer_join -- PhysicalResultSink --PhysicalDistribute[DistributionSpecGather] -----hashJoin[RIGHT_OUTER_JOIN colocated] hashCondition=((t1.id = t2.id)) otherCondition=() -------PhysicalOlapScan[t] +----hashJoin[LEFT_OUTER_JOIN colocated] hashCondition=((t1.id = t2.id)) otherCondition=() ------filter((t1.score > 10)) --------PhysicalOlapScan[t] +------PhysicalOlapScan[t] -- !full_outer_join -- PhysicalResultSink @@ -53,10 +53,10 @@ PhysicalResultSink PhysicalResultSink --PhysicalDistribute[DistributionSpecGather] ----hashJoin[LEFT_OUTER_JOIN colocated] hashCondition=((t1.id = t3.id)) otherCondition=() -------hashJoin[RIGHT_OUTER_JOIN colocated] hashCondition=((t1.id = t2.id)) otherCondition=() ---------PhysicalOlapScan[t] +------hashJoin[LEFT_OUTER_JOIN colocated] hashCondition=((t1.id = t2.id)) otherCondition=() --------filter((t1.score > 10)) ----------PhysicalOlapScan[t] +--------PhysicalOlapScan[t] ------PhysicalOlapScan[t] -- !multiple_left_outer_2 -- @@ -73,30 +73,30 @@ PhysicalResultSink PhysicalResultSink --PhysicalDistribute[DistributionSpecGather] ----hashJoin[INNER_JOIN colocated] hashCondition=((t1.id = t3.id)) otherCondition=() -------PhysicalOlapScan[t] ------hashJoin[INNER_JOIN colocated] hashCondition=((t1.id = t2.id)) otherCondition=() ---------PhysicalOlapScan[t] --------filter((t1.score > 10)) ----------PhysicalOlapScan[t] +--------PhysicalOlapScan[t] +------PhysicalOlapScan[t] -- !multiple_right_outer_2 -- PhysicalResultSink --PhysicalDistribute[DistributionSpecGather] ----hashJoin[INNER_JOIN colocated] hashCondition=((t1.id = t3.id)) otherCondition=() -------PhysicalOlapScan[t] ------hashJoin[INNER_JOIN colocated] hashCondition=((t1.id = t2.id)) otherCondition=() --------PhysicalOlapScan[t] --------filter((t2.score > 10)) ----------PhysicalOlapScan[t] +------PhysicalOlapScan[t] -- !multiple_full_outer_1 -- PhysicalResultSink --PhysicalDistribute[DistributionSpecGather] ----hashJoin[LEFT_OUTER_JOIN colocated] hashCondition=((t1.id = t3.id)) otherCondition=() -------hashJoin[RIGHT_OUTER_JOIN colocated] hashCondition=((t1.id = t2.id)) otherCondition=() ---------PhysicalOlapScan[t] +------hashJoin[LEFT_OUTER_JOIN colocated] hashCondition=((t1.id = t2.id)) otherCondition=() --------filter((t1.score > 10)) ----------PhysicalOlapScan[t] +--------PhysicalOlapScan[t] ------PhysicalOlapScan[t] -- !multiple_full_outer_2 -- @@ -112,10 +112,10 @@ PhysicalResultSink -- !left_outer_join_non_null_assertion -- PhysicalResultSink --PhysicalDistribute[DistributionSpecGather] -----hashJoin[RIGHT_OUTER_JOIN colocated] hashCondition=((t1.id = t2.id)) otherCondition=() -------PhysicalOlapScan[t] +----hashJoin[LEFT_OUTER_JOIN colocated] hashCondition=((t1.id = t2.id)) otherCondition=() ------filter(( not id IS NULL) and (t1.score > 5)) --------PhysicalOlapScan[t] +------PhysicalOlapScan[t] -- !right_outer_join_non_null_assertion -- PhysicalResultSink @@ -138,9 +138,9 @@ PhysicalResultSink --PhysicalDistribute[DistributionSpecGather] ----hashJoin[INNER_JOIN colocated] hashCondition=((t2.id = t3.id)) otherCondition=() ------hashJoin[INNER_JOIN colocated] hashCondition=((t1.id = t2.id)) otherCondition=() ---------PhysicalOlapScan[t] --------filter((t1.score > 5)) ----------PhysicalOlapScan[t] +--------PhysicalOlapScan[t] ------filter(( not score IS NULL)) --------PhysicalOlapScan[t] @@ -161,7 +161,7 @@ PhysicalResultSink ----PhysicalProject ------filter((count(id) > 1)) --------hashAgg[LOCAL] -----------hashJoin[RIGHT_OUTER_JOIN colocated] hashCondition=((t1.id = t2.id)) otherCondition=() +----------hashJoin[LEFT_OUTER_JOIN colocated] hashCondition=((t1.id = t2.id)) otherCondition=() ------------PhysicalProject --------------PhysicalOlapScan[t] ------------PhysicalProject @@ -181,28 +181,28 @@ PhysicalResultSink PhysicalResultSink --PhysicalDistribute[DistributionSpecGather] ----hashJoin[INNER_JOIN colocated] hashCondition=((t1.id = t2.id)) otherCondition=() -------filter(( not name IS NULL)) ---------PhysicalOlapScan[t] ------filter((t1.score > 10)) --------PhysicalOlapScan[t] +------filter(( not name IS NULL)) +--------PhysicalOlapScan[t] -- !right_outer -- PhysicalResultSink --PhysicalDistribute[DistributionSpecGather] ----hashJoin[INNER_JOIN colocated] hashCondition=((t1.id = t2.id)) otherCondition=() -------filter(( not name IS NULL)) ---------PhysicalOlapScan[t] ------filter((t1.score > 10)) --------PhysicalOlapScan[t] +------filter(( not name IS NULL)) +--------PhysicalOlapScan[t] -- !full_outer -- PhysicalResultSink --PhysicalDistribute[DistributionSpecGather] ----hashJoin[INNER_JOIN colocated] hashCondition=((t1.id = t2.id)) otherCondition=() -------filter(( not name IS NULL)) ---------PhysicalOlapScan[t] ------filter((t1.score > 10)) --------PhysicalOlapScan[t] +------filter(( not name IS NULL)) +--------PhysicalOlapScan[t] -- !self_left_outer -- PhysicalResultSink diff --git a/regression-test/data/nereids_rules_p0/infer_predicate/infer_intersect_except.out b/regression-test/data/nereids_rules_p0/infer_predicate/infer_intersect_except.out index 783f83efe61753..2609ca5f4c9e23 100644 --- a/regression-test/data/nereids_rules_p0/infer_predicate/infer_intersect_except.out +++ b/regression-test/data/nereids_rules_p0/infer_predicate/infer_intersect_except.out @@ -58,10 +58,10 @@ PhysicalResultSink ----filter((infer_intersect_except1.a > 0)) ------PhysicalOlapScan[infer_intersect_except1] ----PhysicalIntersect -------filter((infer_intersect_except3.a = 1) and (infer_intersect_except3.b = 'abc')) ---------PhysicalOlapScan[infer_intersect_except3] ------filter((infer_intersect_except2.b > 'ab')) --------PhysicalOlapScan[infer_intersect_except2] +------filter((infer_intersect_except3.a = 1) and (infer_intersect_except3.b = 'abc')) +--------PhysicalOlapScan[infer_intersect_except3] -- !except_and_intersect_except_predicate_to_right -- PhysicalResultSink diff --git a/regression-test/data/nereids_rules_p0/infer_set_operator_distinct/infer_set_operator_distinct.out b/regression-test/data/nereids_rules_p0/infer_set_operator_distinct/infer_set_operator_distinct.out index 25bbc68f24e89b..1842b0a25f45b3 100644 --- a/regression-test/data/nereids_rules_p0/infer_set_operator_distinct/infer_set_operator_distinct.out +++ b/regression-test/data/nereids_rules_p0/infer_set_operator_distinct/infer_set_operator_distinct.out @@ -201,12 +201,11 @@ PhysicalResultSink ------PhysicalDistribute[DistributionSpecHash] --------hashAgg[LOCAL] ----------PhysicalUnion -------------PhysicalDistribute[DistributionSpecExecutionAny] ---------------PhysicalProject -----------------hashJoin[LEFT_OUTER_JOIN broadcast] hashCondition=((t1.id = t2.id)) otherCondition=() -------------------PhysicalProject ---------------------PhysicalOlapScan[t2] -------------------PhysicalOlapScan[t1] +------------PhysicalProject +--------------hashJoin[RIGHT_OUTER_JOIN shuffle] hashCondition=((t1.id = t2.id)) otherCondition=() +----------------PhysicalOlapScan[t1] +----------------PhysicalProject +------------------PhysicalOlapScan[t2] ------------PhysicalDistribute[DistributionSpecExecutionAny] --------------PhysicalOlapScan[t3] @@ -629,10 +628,10 @@ PhysicalResultSink --------------PhysicalDistribute[DistributionSpecHash] ----------------hashAgg[LOCAL] ------------------PhysicalProject ---------------------hashJoin[LEFT_OUTER_JOIN broadcast] hashCondition=((t1.id = t2.id)) otherCondition=() +--------------------hashJoin[RIGHT_OUTER_JOIN shuffle] hashCondition=((t1.id = t2.id)) otherCondition=() +----------------------PhysicalOlapScan[t1] ----------------------PhysicalProject ------------------------PhysicalOlapScan[t2] -----------------------PhysicalOlapScan[t1] ------------PhysicalDistribute[DistributionSpecExecutionAny] --------------hashAgg[LOCAL] ----------------PhysicalOlapScan[t3] @@ -1080,12 +1079,11 @@ PhysicalResultSink ------PhysicalDistribute[DistributionSpecHash] --------hashAgg[LOCAL] ----------PhysicalUnion -------------PhysicalDistribute[DistributionSpecExecutionAny] ---------------PhysicalProject -----------------hashJoin[LEFT_OUTER_JOIN broadcast] hashCondition=((t1.id = t2.id)) otherCondition=() -------------------PhysicalProject ---------------------PhysicalOlapScan[t2] -------------------PhysicalOlapScan[t1] +------------PhysicalProject +--------------hashJoin[RIGHT_OUTER_JOIN shuffle] hashCondition=((t1.id = t2.id)) otherCondition=() +----------------PhysicalOlapScan[t1] +----------------PhysicalProject +------------------PhysicalOlapScan[t2] ------------PhysicalDistribute[DistributionSpecExecutionAny] --------------PhysicalOlapScan[t3] diff --git a/regression-test/data/nereids_rules_p0/push_down_filter/extract_from_disjunction_in_join.out b/regression-test/data/nereids_rules_p0/push_down_filter/extract_from_disjunction_in_join.out index 9077ecb24b9b56..898621c7da765c 100644 --- a/regression-test/data/nereids_rules_p0/push_down_filter/extract_from_disjunction_in_join.out +++ b/regression-test/data/nereids_rules_p0/push_down_filter/extract_from_disjunction_in_join.out @@ -9,11 +9,11 @@ PhysicalResultSink -- !right_semi -- PhysicalResultSink ---hashJoin[LEFT_SEMI_JOIN] hashCondition=((t1.b = t2.b)) otherCondition=((((t2.a = 9) AND (t1.a = 1)) OR ((t1.a = 2) AND (t2.a = 8)))) -----filter(a IN (8, 9)) -------PhysicalOlapScan[extract_from_disjunction_in_join_t2] +--hashJoin[RIGHT_SEMI_JOIN] hashCondition=((t1.b = t2.b)) otherCondition=((((t2.a = 9) AND (t1.a = 1)) OR ((t1.a = 2) AND (t2.a = 8)))) ----filter(a IN (1, 2)) ------PhysicalOlapScan[extract_from_disjunction_in_join_t1] +----filter(a IN (8, 9)) +------PhysicalOlapScan[extract_from_disjunction_in_join_t2] -- !left -- PhysicalResultSink @@ -24,10 +24,10 @@ PhysicalResultSink -- !right -- PhysicalResultSink ---hashJoin[LEFT_OUTER_JOIN] hashCondition=((t1.b = t2.b)) otherCondition=((((t2.a = 9) AND (t1.a = 1)) OR ((t1.a = 2) AND (t2.a = 8))) and a IN (8, 9)) -----PhysicalOlapScan[extract_from_disjunction_in_join_t2] +--hashJoin[RIGHT_OUTER_JOIN] hashCondition=((t1.b = t2.b)) otherCondition=((((t2.a = 9) AND (t1.a = 1)) OR ((t1.a = 2) AND (t2.a = 8))) and a IN (8, 9)) ----filter(a IN (1, 2)) ------PhysicalOlapScan[extract_from_disjunction_in_join_t1] +----PhysicalOlapScan[extract_from_disjunction_in_join_t2] -- !left_anti -- PhysicalResultSink @@ -38,10 +38,10 @@ PhysicalResultSink -- !right_anti -- PhysicalResultSink ---hashJoin[LEFT_ANTI_JOIN] hashCondition=((t1.b = t2.b)) otherCondition=((((t2.a = 9) AND (t1.a = 1)) OR ((t1.a = 2) AND (t2.a = 8))) and a IN (8, 9)) -----PhysicalOlapScan[extract_from_disjunction_in_join_t2] +--hashJoin[RIGHT_ANTI_JOIN] hashCondition=((t1.b = t2.b)) otherCondition=((((t2.a = 9) AND (t1.a = 1)) OR ((t1.a = 2) AND (t2.a = 8))) and a IN (8, 9)) ----filter(a IN (1, 2)) ------PhysicalOlapScan[extract_from_disjunction_in_join_t1] +----PhysicalOlapScan[extract_from_disjunction_in_join_t2] -- !inner -- PhysicalResultSink diff --git a/regression-test/suites/nereids_rules_p0/push_down_filter/extract_from_disjunction_in_join.groovy b/regression-test/suites/nereids_rules_p0/push_down_filter/extract_from_disjunction_in_join.groovy index c305f876f80d9a..2132028b7ba8f3 100644 --- a/regression-test/suites/nereids_rules_p0/push_down_filter/extract_from_disjunction_in_join.groovy +++ b/regression-test/suites/nereids_rules_p0/push_down_filter/extract_from_disjunction_in_join.groovy @@ -23,6 +23,7 @@ suite("extract_from_disjunction_in_join") { sql """ set runtime_filter_mode=OFF; set disable_join_reorder=true; + set disable_join_reorder=true; """ From e5959673c207bbd58422e9035a37c45d2970111b Mon Sep 17 00:00:00 2001 From: minghong Date: Thu, 19 Sep 2024 09:20:52 +0800 Subject: [PATCH 09/14] test_cte_filter_pushdown --- .../suites/nereids_p0/cte/test_cte_filter_pushdown.groovy | 1 + 1 file changed, 1 insertion(+) diff --git a/regression-test/suites/nereids_p0/cte/test_cte_filter_pushdown.groovy b/regression-test/suites/nereids_p0/cte/test_cte_filter_pushdown.groovy index e90b9037dc56c4..7d53dd36641d0f 100644 --- a/regression-test/suites/nereids_p0/cte/test_cte_filter_pushdown.groovy +++ b/regression-test/suites/nereids_p0/cte/test_cte_filter_pushdown.groovy @@ -20,6 +20,7 @@ suite("test_cte_filter_pushdown") { sql "SET enable_fallback_to_original_planner=false" sql "set runtime_filter_type=2;" sql "set ignore_shape_nodes='PhysicalDistribute, PhysicalProject'" + sql "set runtime_filter_mode=OFF" // CTE filter pushing down with the same filter qt_cte_filter_pushdown_1 """ explain shape plan From 4a712b05146e722ef81023371d7dd89eb275a9b7 Mon Sep 17 00:00:00 2001 From: minghong Date: Fri, 20 Sep 2024 10:49:14 +0800 Subject: [PATCH 10/14] rt --- .../nereids_p0/hint/multi_leading.groovy | 186 +++++++++--------- .../suites/nereids_p0/hint/test_hint.groovy | 16 +- .../eliminate_outer_join.groovy | 1 + 3 files changed, 102 insertions(+), 101 deletions(-) diff --git a/regression-test/suites/nereids_p0/hint/multi_leading.groovy b/regression-test/suites/nereids_p0/hint/multi_leading.groovy index 048c2e25498950..4425fae1db2d11 100644 --- a/regression-test/suites/nereids_p0/hint/multi_leading.groovy +++ b/regression-test/suites/nereids_p0/hint/multi_leading.groovy @@ -45,99 +45,99 @@ suite("multi_leading") { sql """create table t3 (c3 int, c33 int) distributed by hash(c3) buckets 3 properties('replication_num' = '1');""" sql """create table t4 (c4 int, c44 int) distributed by hash(c4) buckets 3 properties('replication_num' = '1');""" - streamLoad { - table "t1" - db "test_multi_leading" - set 'column_separator', '|' - set 'format', 'csv' - file 't1.csv' - time 10000 - } - - streamLoad { - table "t2" - db "test_multi_leading" - set 'column_separator', '|' - set 'format', 'csv' - file 't2.csv' - time 10000 - } - - streamLoad { - table "t3" - db "test_multi_leading" - set 'column_separator', '|' - set 'format', 'csv' - file 't3.csv' - time 10000 - } - - streamLoad { - table "t4" - db "test_multi_leading" - set 'column_separator', '|' - set 'format', 'csv' - file 't4.csv' - time 10000 - } + // streamLoad { + // table "t1" + // db "test_multi_leading" + // set 'column_separator', '|' + // set 'format', 'csv' + // file 't1.csv' + // time 10000 + // } + + // streamLoad { + // table "t2" + // db "test_multi_leading" + // set 'column_separator', '|' + // set 'format', 'csv' + // file 't2.csv' + // time 10000 + // } + + // streamLoad { + // table "t3" + // db "test_multi_leading" + // set 'column_separator', '|' + // set 'format', 'csv' + // file 't3.csv' + // time 10000 + // } + + // streamLoad { + // table "t4" + // db "test_multi_leading" + // set 'column_separator', '|' + // set 'format', 'csv' + // file 't4.csv' + // time 10000 + // } // test cte inline - qt_sql1_2 """explain shape plan with cte as (select /*+ leading(t2 t1) */ c11, c1 from t1 join t2 on c1 = c2) select /*+ leading(t1 cte)*/ count(*) from cte,t1 where cte.c1 = t1.c1 and t1.c1 > 300;""" - qt_sql1_3 """explain shape plan with cte as (select /*+ leading(t1 t2) */ c11, c1 from t1 join t2 on c1 = c2) select /*+ leading(t1 cte)*/ count(*) from cte,t1 where cte.c1 = t1.c1 and t1.c1 > 300;""" - qt_sql1_4 """explain shape plan with cte as (select /*+ leading(t1 t2) */ c11, c1 from t1 join t2 on c1 = c2) select /*+ leading(t1 cte)*/ count(*) from cte,t1 where cte.c1 = t1.c1 and t1.c1 > 300;""" - - qt_sql1_res_1 """with cte as (select c11, c1 from t1 join t2 on c1 = c2) select count(*) from cte,t1 where cte.c1 = t1.c1 and t1.c1 > 300;""" - qt_sql1_res_2 """with cte as (select /*+ leading(t2 t1) */ c11, c1 from t1 join t2 on c1 = c2) select /*+ leading(t1 cte)*/ count(*) from cte,t1 where cte.c1 = t1.c1 and t1.c1 > 300;""" - qt_sql1_res_3 """with cte as (select /*+ leading(t1 t2) */ c11, c1 from t1 join t2 on c1 = c2) select /*+ leading(t1 cte)*/ count(*) from cte,t1 where cte.c1 = t1.c1 and t1.c1 > 300;""" - qt_sql1_res_4 """with cte as (select /*+ leading(t1 t2) */ c11, c1 from t1 join t2 on c1 = c2) select /*+ leading(t1 cte)*/ count(*) from cte,t1 where cte.c1 = t1.c1 and t1.c1 > 300;""" - - // test subquery alone - qt_sql2_2 """explain shape plan select /*+ leading(t3 alias1) */ count(*) from (select c1, c11 from t1 join t2 on c1 = c2) as alias1 join t3 on alias1.c1 = t3.c3;""" - qt_sql2_3 """explain shape plan select count(*) from (select /*+ leading(t2 t1) */ c1, c11 from t1 join t2 on c1 = c2) as alias1 join t3 on alias1.c1 = t3.c3;""" - qt_sql2_4 """explain shape plan select /*+ leading(t3 alias1) */ count(*) from (select /*+ leading(t2 t1) */ c1, c11 from t1 join t2 on c1 = c2) as alias1 join t3 on alias1.c1 = t3.c3;""" - - qt_sql2_res_1 """select count(*) from (select c1, c11 from t1 join t2 on c1 = c2) as alias1 join t3 on alias1.c1 = t3.c3;""" - qt_sql2_res_2 """select /*+ leading(t3 alias1) */ count(*) from (select c1, c11 from t1 join t2 on c1 = c2) as alias1 join t3 on alias1.c1 = t3.c3;""" - qt_sql2_res_3 """select count(*) from (select /*+ leading(t2 t1) */ c1, c11 from t1 join t2 on c1 = c2) as alias1 join t3 on alias1.c1 = t3.c3;""" - qt_sql2_res_4 """select /*+ leading(t3 alias1) */ count(*) from (select /*+ leading(t2 t1) */ c1, c11 from t1 join t2 on c1 = c2) as alias1 join t3 on alias1.c1 = t3.c3;""" - - // test subquery + cte - qt_sql3_2 """explain shape plan with cte as (select /*+ leading(t2 t1) */ c11, c1 from t1 join t2 on c1 = c2) select /*+ leading(t3 alias1 cte) */ count(*) from (select c1, c11 from t1 join t2 on c1 = c2) as alias1 join t3 on alias1.c1 = t3.c3 join cte on alias1.c1 = cte.c11;;""" - qt_sql3_3 """explain shape plan with cte as (select c11, c1 from t1 join t2 on c1 = c2) select count(*) from (select /*+ leading(t2 t1) */ c1, c11 from t1 join t2 on c1 = c2) as alias1 join t3 on alias1.c1 = t3.c3 join cte on alias1.c1 = cte.c11;;""" - qt_sql3_4 """explain shape plan with cte as (select /*+ leading(t2 t1) */ c11, c1 from t1 join t2 on c1 = c2) select /*+ leading(t3 alias1 cte) */ count(*) from (select /*+ leading(t2 t1) */ c1, c11 from t1 join t2 on c1 = c2) as alias1 join t3 on alias1.c1 = t3.c3 join cte on alias1.c1 = cte.c11;;""" - - qt_sql3_res_1 """with cte as (select c11, c1 from t1 join t2 on c1 = c2) select count(*) from (select c1, c11 from t1 join t2 on c1 = c2) as alias1 join t3 on alias1.c1 = t3.c3 join cte on alias1.c1 = cte.c11;;""" - qt_sql3_res_2 """with cte as (select /*+ leading(t2 t1) */ c11, c1 from t1 join t2 on c1 = c2) select /*+ leading(t3 alias1 cte) */ count(*) from (select c1, c11 from t1 join t2 on c1 = c2) as alias1 join t3 on alias1.c1 = t3.c3 join cte on alias1.c1 = cte.c11;;""" - qt_sql3_res_3 """with cte as (select c11, c1 from t1 join t2 on c1 = c2) select count(*) from (select /*+ leading(t2 t1) */ c1, c11 from t1 join t2 on c1 = c2) as alias1 join t3 on alias1.c1 = t3.c3 join cte on alias1.c1 = cte.c11;;""" - qt_sql3_res_4 """with cte as (select /*+ leading(t2 t1) */ c11, c1 from t1 join t2 on c1 = c2) select /*+ leading(t3 alias1 cte) */ count(*) from (select /*+ leading(t2 t1) */ c1, c11 from t1 join t2 on c1 = c2) as alias1 join t3 on alias1.c1 = t3.c3 join cte on alias1.c1 = cte.c11;;""" - - // test multi level subqueries - qt_sql4_1 """explain shape plan select /*+ leading(t3 alias1) */ count(*) from (select c1, c11 from t1 join (select c2, c22 from t2 join t4 on c2 = c4) as alias2 on c1 = alias2.c2) as alias1 join t3 on alias1.c1 = t3.c3;""" - qt_sql4_2 """explain shape plan select count(*) from (select /*+ leading(alias2 t1) */ c1, c11 from t1 join (select c2, c22 from t2 join t4 on c2 = c4) as alias2 on c1 = alias2.c2) as alias1 join t3 on alias1.c1 = t3.c3;""" - qt_sql4_3 """explain shape plan select count(*) from (select c1, c11 from t1 join (select /*+ leading(t4 t2) */ c2, c22 from t2 join t4 on c2 = c4) as alias2 on c1 = alias2.c2) as alias1 join t3 on alias1.c1 = t3.c3;""" - qt_sql4_4 """explain shape plan select /*+ leading(t3 alias1) */ count(*) from (select /*+ leading(alias2 t1) */ c1, c11 from t1 join (select c2, c22 from t2 join t4 on c2 = c4) as alias2 on c1 = alias2.c2) as alias1 join t3 on alias1.c1 = t3.c3;""" - explain { - sql """shape plan select /*+ leading(t3 alias1) */ count(*) from (select c1, c11 from t1 join (select /*+ leading(t4 t2) */ c2, c22 from t2 join t4 on c2 = c4) as alias2 on c1 = alias2.c2) as alias1 join t3 on alias1.c1 = t3.c3;""" - contains("SyntaxError: leading(t4 t2) Msg:one query block can only have one leading clause") - } - explain { - sql """shape plan select count(*) from (select /*+ leading(alias2 t1) */ c1, c11 from t1 join (select /*+ leading(t4 t2) */ c2, c22 from t2 join t4 on c2 = c4) as alias2 on c1 = alias2.c2) as alias1 join t3 on alias1.c1 = t3.c3;""" - contains("SyntaxError: leading(t4 t2) Msg:one query block can only have one leading clause") - } - explain { - sql """shape plan select /*+ leading(t3 alias1) */ count(*) from (select /*+ leading(alias2 t1) */ c1, c11 from t1 join (select /*+ leading(t4 t2) */ c2, c22 from t2 join t4 on c2 = c4) as alias2 on c1 = alias2.c2) as alias1 join t3 on alias1.c1 = t3.c3;""" - contains("UnUsed: leading(alias2 t1)") - } - - qt_sql4_res_0 """select count(*) from (select c1, c11 from t1 join (select c2, c22 from t2 join t4 on c2 = c4) as alias2 on c1 = alias2.c2) as alias1 join t3 on alias1.c1 = t3.c3;""" - qt_sql4_res_1 """select /*+ leading(t3 alias1) */ count(*) from (select c1, c11 from t1 join (select c2, c22 from t2 join t4 on c2 = c4) as alias2 on c1 = alias2.c2) as alias1 join t3 on alias1.c1 = t3.c3;""" - qt_sql4_res_2 """select count(*) from (select /*+ leading(alias2 t1) */ c1, c11 from t1 join (select c2, c22 from t2 join t4 on c2 = c4) as alias2 on c1 = alias2.c2) as alias1 join t3 on alias1.c1 = t3.c3;""" - qt_sql4_res_3 """select count(*) from (select c1, c11 from t1 join (select /*+ leading(t4 t2) */ c2, c22 from t2 join t4 on c2 = c4) as alias2 on c1 = alias2.c2) as alias1 join t3 on alias1.c1 = t3.c3;""" - qt_sql4_res_4 """select /*+ leading(t3 alias1) */ count(*) from (select /*+ leading(alias2 t1) */ c1, c11 from t1 join (select c2, c22 from t2 join t4 on c2 = c4) as alias2 on c1 = alias2.c2) as alias1 join t3 on alias1.c1 = t3.c3;""" - qt_sql4_res_5 """select /*+ leading(t3 alias1) */ count(*) from (select c1, c11 from t1 join (select /*+ leading(t4 t2) */ c2, c22 from t2 join t4 on c2 = c4) as alias2 on c1 = alias2.c2) as alias1 join t3 on alias1.c1 = t3.c3;""" - qt_sql4_res_6 """select count(*) from (select /*+ leading(alias2 t1) */ c1, c11 from t1 join (select /*+ leading(t4 t2) */ c2, c22 from t2 join t4 on c2 = c4) as alias2 on c1 = alias2.c2) as alias1 join t3 on alias1.c1 = t3.c3;""" - qt_sql4_res_7 """select /*+ leading(t3 alias1) */ count(*) from (select /*+ leading(alias2 t1) */ c1, c11 from t1 join (select /*+ leading(t4 t2) */ c2, c22 from t2 join t4 on c2 = c4) as alias2 on c1 = alias2.c2) as alias1 join t3 on alias1.c1 = t3.c3;""" - - // use cte in scalar query - qt_sql5_2 """explain shape plan with cte as (select c11, c1 from t1) SELECT c1 FROM cte group by c1 having sum(cte.c11) > (select /*+ leading(cte t1) */ 0.05 * avg(t1.c11) from t1 join cte on t1.c1 = cte.c11 )""" + // qt_sql1_2 """explain shape plan with cte as (select /*+ leading(t2 t1) */ c11, c1 from t1 join t2 on c1 = c2) select /*+ leading(t1 cte)*/ count(*) from cte,t1 where cte.c1 = t1.c1 and t1.c1 > 300;""" + // qt_sql1_3 """explain shape plan with cte as (select /*+ leading(t1 t2) */ c11, c1 from t1 join t2 on c1 = c2) select /*+ leading(t1 cte)*/ count(*) from cte,t1 where cte.c1 = t1.c1 and t1.c1 > 300;""" + // qt_sql1_4 """explain shape plan with cte as (select /*+ leading(t1 t2) */ c11, c1 from t1 join t2 on c1 = c2) select /*+ leading(t1 cte)*/ count(*) from cte,t1 where cte.c1 = t1.c1 and t1.c1 > 300;""" + + // qt_sql1_res_1 """with cte as (select c11, c1 from t1 join t2 on c1 = c2) select count(*) from cte,t1 where cte.c1 = t1.c1 and t1.c1 > 300;""" + // qt_sql1_res_2 """with cte as (select /*+ leading(t2 t1) */ c11, c1 from t1 join t2 on c1 = c2) select /*+ leading(t1 cte)*/ count(*) from cte,t1 where cte.c1 = t1.c1 and t1.c1 > 300;""" + // qt_sql1_res_3 """with cte as (select /*+ leading(t1 t2) */ c11, c1 from t1 join t2 on c1 = c2) select /*+ leading(t1 cte)*/ count(*) from cte,t1 where cte.c1 = t1.c1 and t1.c1 > 300;""" + // qt_sql1_res_4 """with cte as (select /*+ leading(t1 t2) */ c11, c1 from t1 join t2 on c1 = c2) select /*+ leading(t1 cte)*/ count(*) from cte,t1 where cte.c1 = t1.c1 and t1.c1 > 300;""" + + // // test subquery alone + // qt_sql2_2 """explain shape plan select /*+ leading(t3 alias1) */ count(*) from (select c1, c11 from t1 join t2 on c1 = c2) as alias1 join t3 on alias1.c1 = t3.c3;""" + // qt_sql2_3 """explain shape plan select count(*) from (select /*+ leading(t2 t1) */ c1, c11 from t1 join t2 on c1 = c2) as alias1 join t3 on alias1.c1 = t3.c3;""" + // qt_sql2_4 """explain shape plan select /*+ leading(t3 alias1) */ count(*) from (select /*+ leading(t2 t1) */ c1, c11 from t1 join t2 on c1 = c2) as alias1 join t3 on alias1.c1 = t3.c3;""" + + // qt_sql2_res_1 """select count(*) from (select c1, c11 from t1 join t2 on c1 = c2) as alias1 join t3 on alias1.c1 = t3.c3;""" + // qt_sql2_res_2 """select /*+ leading(t3 alias1) */ count(*) from (select c1, c11 from t1 join t2 on c1 = c2) as alias1 join t3 on alias1.c1 = t3.c3;""" + // qt_sql2_res_3 """select count(*) from (select /*+ leading(t2 t1) */ c1, c11 from t1 join t2 on c1 = c2) as alias1 join t3 on alias1.c1 = t3.c3;""" + // qt_sql2_res_4 """select /*+ leading(t3 alias1) */ count(*) from (select /*+ leading(t2 t1) */ c1, c11 from t1 join t2 on c1 = c2) as alias1 join t3 on alias1.c1 = t3.c3;""" + + // // test subquery + cte + // qt_sql3_2 """explain shape plan with cte as (select /*+ leading(t2 t1) */ c11, c1 from t1 join t2 on c1 = c2) select /*+ leading(t3 alias1 cte) */ count(*) from (select c1, c11 from t1 join t2 on c1 = c2) as alias1 join t3 on alias1.c1 = t3.c3 join cte on alias1.c1 = cte.c11;;""" + // qt_sql3_3 """explain shape plan with cte as (select c11, c1 from t1 join t2 on c1 = c2) select count(*) from (select /*+ leading(t2 t1) */ c1, c11 from t1 join t2 on c1 = c2) as alias1 join t3 on alias1.c1 = t3.c3 join cte on alias1.c1 = cte.c11;;""" + // qt_sql3_4 """explain shape plan with cte as (select /*+ leading(t2 t1) */ c11, c1 from t1 join t2 on c1 = c2) select /*+ leading(t3 alias1 cte) */ count(*) from (select /*+ leading(t2 t1) */ c1, c11 from t1 join t2 on c1 = c2) as alias1 join t3 on alias1.c1 = t3.c3 join cte on alias1.c1 = cte.c11;;""" + + // qt_sql3_res_1 """with cte as (select c11, c1 from t1 join t2 on c1 = c2) select count(*) from (select c1, c11 from t1 join t2 on c1 = c2) as alias1 join t3 on alias1.c1 = t3.c3 join cte on alias1.c1 = cte.c11;;""" + // qt_sql3_res_2 """with cte as (select /*+ leading(t2 t1) */ c11, c1 from t1 join t2 on c1 = c2) select /*+ leading(t3 alias1 cte) */ count(*) from (select c1, c11 from t1 join t2 on c1 = c2) as alias1 join t3 on alias1.c1 = t3.c3 join cte on alias1.c1 = cte.c11;;""" + // qt_sql3_res_3 """with cte as (select c11, c1 from t1 join t2 on c1 = c2) select count(*) from (select /*+ leading(t2 t1) */ c1, c11 from t1 join t2 on c1 = c2) as alias1 join t3 on alias1.c1 = t3.c3 join cte on alias1.c1 = cte.c11;;""" + // qt_sql3_res_4 """with cte as (select /*+ leading(t2 t1) */ c11, c1 from t1 join t2 on c1 = c2) select /*+ leading(t3 alias1 cte) */ count(*) from (select /*+ leading(t2 t1) */ c1, c11 from t1 join t2 on c1 = c2) as alias1 join t3 on alias1.c1 = t3.c3 join cte on alias1.c1 = cte.c11;;""" + + // // test multi level subqueries + // qt_sql4_1 """explain shape plan select /*+ leading(t3 alias1) */ count(*) from (select c1, c11 from t1 join (select c2, c22 from t2 join t4 on c2 = c4) as alias2 on c1 = alias2.c2) as alias1 join t3 on alias1.c1 = t3.c3;""" + // qt_sql4_2 """explain shape plan select count(*) from (select /*+ leading(alias2 t1) */ c1, c11 from t1 join (select c2, c22 from t2 join t4 on c2 = c4) as alias2 on c1 = alias2.c2) as alias1 join t3 on alias1.c1 = t3.c3;""" + // qt_sql4_3 """explain shape plan select count(*) from (select c1, c11 from t1 join (select /*+ leading(t4 t2) */ c2, c22 from t2 join t4 on c2 = c4) as alias2 on c1 = alias2.c2) as alias1 join t3 on alias1.c1 = t3.c3;""" + // qt_sql4_4 """explain shape plan select /*+ leading(t3 alias1) */ count(*) from (select /*+ leading(alias2 t1) */ c1, c11 from t1 join (select c2, c22 from t2 join t4 on c2 = c4) as alias2 on c1 = alias2.c2) as alias1 join t3 on alias1.c1 = t3.c3;""" + // explain { + // sql """shape plan select /*+ leading(t3 alias1) */ count(*) from (select c1, c11 from t1 join (select /*+ leading(t4 t2) */ c2, c22 from t2 join t4 on c2 = c4) as alias2 on c1 = alias2.c2) as alias1 join t3 on alias1.c1 = t3.c3;""" + // contains("SyntaxError: leading(t4 t2) Msg:one query block can only have one leading clause") + // } + // explain { + // sql """shape plan select count(*) from (select /*+ leading(alias2 t1) */ c1, c11 from t1 join (select /*+ leading(t4 t2) */ c2, c22 from t2 join t4 on c2 = c4) as alias2 on c1 = alias2.c2) as alias1 join t3 on alias1.c1 = t3.c3;""" + // contains("SyntaxError: leading(t4 t2) Msg:one query block can only have one leading clause") + // } + // explain { + // sql """shape plan select /*+ leading(t3 alias1) */ count(*) from (select /*+ leading(alias2 t1) */ c1, c11 from t1 join (select /*+ leading(t4 t2) */ c2, c22 from t2 join t4 on c2 = c4) as alias2 on c1 = alias2.c2) as alias1 join t3 on alias1.c1 = t3.c3;""" + // contains("UnUsed: leading(alias2 t1)") + // } + + // qt_sql4_res_0 """select count(*) from (select c1, c11 from t1 join (select c2, c22 from t2 join t4 on c2 = c4) as alias2 on c1 = alias2.c2) as alias1 join t3 on alias1.c1 = t3.c3;""" + // qt_sql4_res_1 """select /*+ leading(t3 alias1) */ count(*) from (select c1, c11 from t1 join (select c2, c22 from t2 join t4 on c2 = c4) as alias2 on c1 = alias2.c2) as alias1 join t3 on alias1.c1 = t3.c3;""" + // qt_sql4_res_2 """select count(*) from (select /*+ leading(alias2 t1) */ c1, c11 from t1 join (select c2, c22 from t2 join t4 on c2 = c4) as alias2 on c1 = alias2.c2) as alias1 join t3 on alias1.c1 = t3.c3;""" + // qt_sql4_res_3 """select count(*) from (select c1, c11 from t1 join (select /*+ leading(t4 t2) */ c2, c22 from t2 join t4 on c2 = c4) as alias2 on c1 = alias2.c2) as alias1 join t3 on alias1.c1 = t3.c3;""" + // qt_sql4_res_4 """select /*+ leading(t3 alias1) */ count(*) from (select /*+ leading(alias2 t1) */ c1, c11 from t1 join (select c2, c22 from t2 join t4 on c2 = c4) as alias2 on c1 = alias2.c2) as alias1 join t3 on alias1.c1 = t3.c3;""" + // qt_sql4_res_5 """select /*+ leading(t3 alias1) */ count(*) from (select c1, c11 from t1 join (select /*+ leading(t4 t2) */ c2, c22 from t2 join t4 on c2 = c4) as alias2 on c1 = alias2.c2) as alias1 join t3 on alias1.c1 = t3.c3;""" + // qt_sql4_res_6 """select count(*) from (select /*+ leading(alias2 t1) */ c1, c11 from t1 join (select /*+ leading(t4 t2) */ c2, c22 from t2 join t4 on c2 = c4) as alias2 on c1 = alias2.c2) as alias1 join t3 on alias1.c1 = t3.c3;""" + // qt_sql4_res_7 """select /*+ leading(t3 alias1) */ count(*) from (select /*+ leading(alias2 t1) */ c1, c11 from t1 join (select /*+ leading(t4 t2) */ c2, c22 from t2 join t4 on c2 = c4) as alias2 on c1 = alias2.c2) as alias1 join t3 on alias1.c1 = t3.c3;""" + + // // use cte in scalar query + // qt_sql5_2 """explain shape plan with cte as (select c11, c1 from t1) SELECT c1 FROM cte group by c1 having sum(cte.c11) > (select /*+ leading(cte t1) */ 0.05 * avg(t1.c11) from t1 join cte on t1.c1 = cte.c11 )""" } diff --git a/regression-test/suites/nereids_p0/hint/test_hint.groovy b/regression-test/suites/nereids_p0/hint/test_hint.groovy index d279b7c1a1d905..81033e014f1eda 100644 --- a/regression-test/suites/nereids_p0/hint/test_hint.groovy +++ b/regression-test/suites/nereids_p0/hint/test_hint.groovy @@ -42,20 +42,20 @@ suite("test_hint") { sql """create table t2 (c2 int, c22 int) distributed by hash(c2) buckets 3 properties('replication_num' = '1');""" // test hint positions, remove join in order to make sure shape stable when no use hint - qt_select1_1 """explain shape plan select /*+ leading(t2 broadcast t1) */ count(*) from t1 join t2 on c1 = c2;""" + // qt_select1_1 """explain shape plan select /*+ leading(t2 broadcast t1) */ count(*) from t1 join t2 on c1 = c2;""" - qt_select1_2 """explain shape plan /*+ leading(t2 broadcast t1) */ select count(*) from t1;""" + // qt_select1_2 """explain shape plan /*+ leading(t2 broadcast t1) */ select count(*) from t1;""" - qt_select1_3 """explain shape plan select /*+DBP: ROUTE={GROUP_ID(zjaq)}*/ count(*) from t1;""" + // qt_select1_3 """explain shape plan select /*+DBP: ROUTE={GROUP_ID(zjaq)}*/ count(*) from t1;""" - qt_select1_4 """explain shape plan/*+DBP: ROUTE={GROUP_ID(zjaq)}*/ select count(*) from t1;""" + // qt_select1_4 """explain shape plan/*+DBP: ROUTE={GROUP_ID(zjaq)}*/ select count(*) from t1;""" - qt_select1_5 """explain shape plan /*+ leading(t2 broadcast t1) */ select /*+ leading(t2 broadcast t1) */ count(*) from t1 join t2 on c1 = c2;""" + // qt_select1_5 """explain shape plan /*+ leading(t2 broadcast t1) */ select /*+ leading(t2 broadcast t1) */ count(*) from t1 join t2 on c1 = c2;""" - qt_select1_6 """explain shape plan/*+DBP: ROUTE={GROUP_ID(zjaq)}*/ select /*+ leading(t2 broadcast t1) */ count(*) from t1 join t2 on c1 = c2;""" + // qt_select1_6 """explain shape plan/*+DBP: ROUTE={GROUP_ID(zjaq)}*/ select /*+ leading(t2 broadcast t1) */ count(*) from t1 join t2 on c1 = c2;""" - qt_select1_7 """explain shape plan /*+ leading(t2 broadcast t1) */ select /*+DBP: ROUTE={GROUP_ID(zjaq)}*/ count(*) from t1;""" + // qt_select1_7 """explain shape plan /*+ leading(t2 broadcast t1) */ select /*+DBP: ROUTE={GROUP_ID(zjaq)}*/ count(*) from t1;""" - qt_select1_8 """explain shape plan /*+DBP: ROUTE={GROUP_ID(zjaq)}*/ select /*+DBP: ROUTE={GROUP_ID(zjaq)}*/ count(*) from t1;""" + // qt_select1_8 """explain shape plan /*+DBP: ROUTE={GROUP_ID(zjaq)}*/ select /*+DBP: ROUTE={GROUP_ID(zjaq)}*/ count(*) from t1;""" } diff --git a/regression-test/suites/nereids_rules_p0/eliminate_outer_join/eliminate_outer_join.groovy b/regression-test/suites/nereids_rules_p0/eliminate_outer_join/eliminate_outer_join.groovy index bbf3dbbe8ee2de..fc38e3be3372f0 100644 --- a/regression-test/suites/nereids_rules_p0/eliminate_outer_join/eliminate_outer_join.groovy +++ b/regression-test/suites/nereids_rules_p0/eliminate_outer_join/eliminate_outer_join.groovy @@ -24,6 +24,7 @@ suite("eliminate_outer_join") { sql "set disable_nereids_rules=PRUNE_EMPTY_PARTITION" sql 'set be_number_for_test=3' sql "set enable_parallel_result_sink=false;" + sql "set disable_join_reorder=true;" sql """ DROP TABLE IF EXISTS t From a8359934872b357f954668f1f1269e0301eba7e7 Mon Sep 17 00:00:00 2001 From: minghong Date: Fri, 20 Sep 2024 14:11:15 +0800 Subject: [PATCH 11/14] disableNereidsJoinReorderOnce --- .../apache/doris/nereids/stats/StatsCalculator.java | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java index c0598b01ede028..4c95e72dd108e2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java @@ -219,9 +219,13 @@ public static void disableJoinReorderIfTableRowCountNotAvailable( for (LogicalOlapScan scan : scans) { double rowCount = calculator.getOlapTableRowCount(scan); if (rowCount == -1 && ConnectContext.get() != null) { - LOG.info("disable join reorder since row count not available: " - + scan.getTable().getNameWithFullQualifiers()); - ConnectContext.get().getSessionVariable().setDisableJoinReorder(true); + try { + ConnectContext.get().getSessionVariable().disableNereidsJoinReorderOnce(); + LOG.info("disable join reorder since row count not available: " + + scan.getTable().getNameWithFullQualifiers()); + } catch (Exception e) { + LOG.info("disableNereidsJoinReorderOnce failed"); + } return; } } From 61283d9616ad04f43ab7ee55dc8ee7dab59c8165 Mon Sep 17 00:00:00 2001 From: minghong Date: Fri, 20 Sep 2024 14:18:06 +0800 Subject: [PATCH 12/14] add-comments --- .../src/main/java/org/apache/doris/catalog/OlapTable.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java index 0b10fa1bdd21c5..1f591d1b7bfebe 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java @@ -1571,6 +1571,9 @@ public long fetchRowCount() { return getRowCountForIndex(baseIndexId, false); } + /** + * @return -1 if there are some tablets whose row count is not reported to FE + */ public long getRowCountForIndex(long indexId, boolean strict) { long rowCount = 0; for (Map.Entry entry : idToPartition.entrySet()) { From 8c688f1e1e8bdcd7ae81bfc8bfdb9212c8622d47 Mon Sep 17 00:00:00 2001 From: minghong Date: Sun, 22 Sep 2024 07:43:28 +0800 Subject: [PATCH 13/14] rt --- .../infer_set_operator_distinct.out | 26 ++++++++++--------- 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/regression-test/data/nereids_rules_p0/infer_set_operator_distinct/infer_set_operator_distinct.out b/regression-test/data/nereids_rules_p0/infer_set_operator_distinct/infer_set_operator_distinct.out index 1842b0a25f45b3..25bbc68f24e89b 100644 --- a/regression-test/data/nereids_rules_p0/infer_set_operator_distinct/infer_set_operator_distinct.out +++ b/regression-test/data/nereids_rules_p0/infer_set_operator_distinct/infer_set_operator_distinct.out @@ -201,11 +201,12 @@ PhysicalResultSink ------PhysicalDistribute[DistributionSpecHash] --------hashAgg[LOCAL] ----------PhysicalUnion -------------PhysicalProject ---------------hashJoin[RIGHT_OUTER_JOIN shuffle] hashCondition=((t1.id = t2.id)) otherCondition=() -----------------PhysicalOlapScan[t1] -----------------PhysicalProject -------------------PhysicalOlapScan[t2] +------------PhysicalDistribute[DistributionSpecExecutionAny] +--------------PhysicalProject +----------------hashJoin[LEFT_OUTER_JOIN broadcast] hashCondition=((t1.id = t2.id)) otherCondition=() +------------------PhysicalProject +--------------------PhysicalOlapScan[t2] +------------------PhysicalOlapScan[t1] ------------PhysicalDistribute[DistributionSpecExecutionAny] --------------PhysicalOlapScan[t3] @@ -628,10 +629,10 @@ PhysicalResultSink --------------PhysicalDistribute[DistributionSpecHash] ----------------hashAgg[LOCAL] ------------------PhysicalProject ---------------------hashJoin[RIGHT_OUTER_JOIN shuffle] hashCondition=((t1.id = t2.id)) otherCondition=() -----------------------PhysicalOlapScan[t1] +--------------------hashJoin[LEFT_OUTER_JOIN broadcast] hashCondition=((t1.id = t2.id)) otherCondition=() ----------------------PhysicalProject ------------------------PhysicalOlapScan[t2] +----------------------PhysicalOlapScan[t1] ------------PhysicalDistribute[DistributionSpecExecutionAny] --------------hashAgg[LOCAL] ----------------PhysicalOlapScan[t3] @@ -1079,11 +1080,12 @@ PhysicalResultSink ------PhysicalDistribute[DistributionSpecHash] --------hashAgg[LOCAL] ----------PhysicalUnion -------------PhysicalProject ---------------hashJoin[RIGHT_OUTER_JOIN shuffle] hashCondition=((t1.id = t2.id)) otherCondition=() -----------------PhysicalOlapScan[t1] -----------------PhysicalProject -------------------PhysicalOlapScan[t2] +------------PhysicalDistribute[DistributionSpecExecutionAny] +--------------PhysicalProject +----------------hashJoin[LEFT_OUTER_JOIN broadcast] hashCondition=((t1.id = t2.id)) otherCondition=() +------------------PhysicalProject +--------------------PhysicalOlapScan[t2] +------------------PhysicalOlapScan[t1] ------------PhysicalDistribute[DistributionSpecExecutionAny] --------------PhysicalOlapScan[t3] From b09ea841c140f2667cdcbf613ade46ead076fe96 Mon Sep 17 00:00:00 2001 From: minghong Date: Mon, 23 Sep 2024 09:44:47 +0800 Subject: [PATCH 14/14] test_cte_filter_pushdown --- .../data/nereids_p0/cte/test_cte_filter_pushdown.out | 8 ++++---- .../suites/nereids_p0/cte/test_cte_filter_pushdown.groovy | 1 - 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/regression-test/data/nereids_p0/cte/test_cte_filter_pushdown.out b/regression-test/data/nereids_p0/cte/test_cte_filter_pushdown.out index 7dd6492aa12499..0bbae0dc25f3a1 100644 --- a/regression-test/data/nereids_p0/cte/test_cte_filter_pushdown.out +++ b/regression-test/data/nereids_p0/cte/test_cte_filter_pushdown.out @@ -7,9 +7,9 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) --------filter((main.k1 = 1)) ----------PhysicalOlapScan[test] --PhysicalResultSink -----hashJoin[INNER_JOIN] hashCondition=((m1.k1 = m2.k1)) otherCondition=() build RFs:RF0 k1->[k1] +----hashJoin[INNER_JOIN] hashCondition=((m1.k1 = m2.k1)) otherCondition=() ------filter((temp.k1 = 1)) ---------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF0 +--------PhysicalCteConsumer ( cteId=CTEId#0 ) ------filter((m2.k1 = 1)) --------PhysicalCteConsumer ( cteId=CTEId#0 ) @@ -21,9 +21,9 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) --------PhysicalQuickSort[LOCAL_SORT] ----------PhysicalOlapScan[test] --PhysicalResultSink -----hashJoin[INNER_JOIN] hashCondition=((m1.k1 = m2.k1)) otherCondition=() build RFs:RF0 k1->[k1] +----hashJoin[INNER_JOIN] hashCondition=((m1.k1 = m2.k1)) otherCondition=() ------filter((temp.k1 = 1)) ---------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF0 +--------PhysicalCteConsumer ( cteId=CTEId#0 ) ------filter((m2.k1 = 1)) --------PhysicalCteConsumer ( cteId=CTEId#0 ) diff --git a/regression-test/suites/nereids_p0/cte/test_cte_filter_pushdown.groovy b/regression-test/suites/nereids_p0/cte/test_cte_filter_pushdown.groovy index 7d53dd36641d0f..795d1ae7e5cf40 100644 --- a/regression-test/suites/nereids_p0/cte/test_cte_filter_pushdown.groovy +++ b/regression-test/suites/nereids_p0/cte/test_cte_filter_pushdown.groovy @@ -18,7 +18,6 @@ suite("test_cte_filter_pushdown") { sql "SET enable_nereids_planner=true" sql "SET enable_pipeline_engine=true" sql "SET enable_fallback_to_original_planner=false" - sql "set runtime_filter_type=2;" sql "set ignore_shape_nodes='PhysicalDistribute, PhysicalProject'" sql "set runtime_filter_mode=OFF" // CTE filter pushing down with the same filter