From 5684fbc92397a84afcbac3aabd8fbb0367ceba19 Mon Sep 17 00:00:00 2001 From: minghong Date: Fri, 27 Sep 2024 19:34:21 +0800 Subject: [PATCH 1/3] disable join reorder if any table row count is not available --- .../org/apache/doris/catalog/OlapTable.java | 3 ++ .../apache/doris/nereids/NereidsPlanner.java | 13 +++++ .../doris/nereids/stats/StatsCalculator.java | 48 +++++++++++++++++++ 3 files changed, 64 insertions(+) diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java index ffff643a8619cd..c3f8600b214c1d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java @@ -1277,6 +1277,9 @@ public long fetchRowCount() { return getRowCountForIndex(baseIndexId, false); } + /** + * @return -1 if there are some tablets whose row count is not reported to FE + */ public long getRowCountForIndex(long indexId, boolean strict) { long rowCount = 0; for (Map.Entry entry : idToPartition.entrySet()) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java index c17d3fa545ba30..2d3d52f9ce65b8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java @@ -23,6 +23,7 @@ import org.apache.doris.analysis.StatementBase; import org.apache.doris.catalog.Column; import org.apache.doris.catalog.Env; +import org.apache.doris.common.FeConstants; import org.apache.doris.common.NereidsException; import org.apache.doris.common.Pair; import org.apache.doris.common.util.DebugUtil; @@ -43,12 +44,14 @@ import org.apache.doris.nereids.processor.post.PlanPostProcessors; import org.apache.doris.nereids.processor.pre.PlanPreprocessors; import org.apache.doris.nereids.properties.PhysicalProperties; +import org.apache.doris.nereids.stats.StatsCalculator; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.literal.Literal; import org.apache.doris.nereids.trees.plans.AbstractPlan; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.commands.ExplainCommand.ExplainLevel; +import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.nereids.trees.plans.physical.PhysicalOneRowRelation; import org.apache.doris.nereids.trees.plans.physical.PhysicalPlan; @@ -258,6 +261,16 @@ private Plan planWithoutLock( } } + // if we cannot get table row count, skip join reorder + // except: + // 1. user set leading hint + // 2. ut test. In ut test, FeConstants.enableInternalSchemaDb is false or FeConstants.runningUnitTest is true + if (FeConstants.enableInternalSchemaDb && !FeConstants.runningUnitTest) { + List scans = cascadesContext.getRewritePlan() + .collectToList(LogicalOlapScan.class::isInstance); + StatsCalculator.disableJoinReorderIfTableRowCountNotAvailable(scans, cascadesContext); + } + optimize(); if (statementContext.getConnectContext().getExecutor() != null) { statementContext.getConnectContext().getExecutor().getSummaryProfile().setNereidsOptimizeTime(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java index 1b67b80a62d5d5..da61d442fcf024 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java @@ -18,6 +18,7 @@ package org.apache.doris.nereids.stats; import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.TableIf; import org.apache.doris.common.FeConstants; import org.apache.doris.common.Pair; @@ -179,6 +180,11 @@ private StatsCalculator(GroupExpression groupExpression, boolean forbidUnknownCo this.cascadesContext = context; } + private StatsCalculator(CascadesContext context) { + this.groupExpression = null; + this.cascadesContext = context; + } + public Map getTotalHistogramMap() { return totalHistogramMap; } @@ -207,6 +213,48 @@ public static StatsCalculator estimate(GroupExpression groupExpression, boolean return statsCalculator; } + /** + * disable join reorder if any table row count is not available. + */ + public static void disableJoinReorderIfTableRowCountNotAvailable( + List scans, CascadesContext context) { + StatsCalculator calculator = new StatsCalculator(context); + for (LogicalOlapScan scan : scans) { + double rowCount = calculator.getOlapTableRowCount(scan); + if (rowCount == -1 && ConnectContext.get() != null) { + try { + ConnectContext.get().getSessionVariable().disableNereidsJoinReorderOnce(); + LOG.info("disable join reorder since row count not available: " + + scan.getTable().getNameWithFullQualifiers()); + } catch (Exception e) { + LOG.info("disableNereidsJoinReorderOnce failed"); + } + return; + } + } + } + + /** + * if the table is not analyzed and BE does not report row count, return -1 + */ + private double getOlapTableRowCount(OlapScan olapScan) { + OlapTable olapTable = olapScan.getTable(); + AnalysisManager analysisManager = Env.getCurrentEnv().getAnalysisManager(); + TableStatsMeta tableMeta = analysisManager.findTableStatsStatus(olapScan.getTable().getId()); + double rowCount = -1; + if (tableMeta != null && tableMeta.userInjected) { + rowCount = tableMeta.getRowCount(olapScan.getSelectedIndexId()); + } else { + rowCount = olapTable.getRowCountForIndex(olapScan.getSelectedIndexId(), true); + if (rowCount == -1) { + if (tableMeta != null) { + rowCount = tableMeta.getRowCount(olapScan.getSelectedIndexId()); + } + } + } + return rowCount; + } + public static StatsCalculator estimate(GroupExpression groupExpression, boolean forbidUnknownColStats, Map columnStatisticMap, boolean isPlayNereidsDump, CascadesContext context) { return StatsCalculator.estimate(groupExpression, From 68beb85edd45fbfba8da87144e6543a29f02ded9 Mon Sep 17 00:00:00 2001 From: minghong Date: Fri, 27 Sep 2024 19:38:49 +0800 Subject: [PATCH 2/3] add hint --- .../src/main/java/org/apache/doris/nereids/NereidsPlanner.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java index 2d3d52f9ce65b8..150780d95e1ad8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java @@ -265,7 +265,7 @@ private Plan planWithoutLock( // except: // 1. user set leading hint // 2. ut test. In ut test, FeConstants.enableInternalSchemaDb is false or FeConstants.runningUnitTest is true - if (FeConstants.enableInternalSchemaDb && !FeConstants.runningUnitTest) { + if (FeConstants.enableInternalSchemaDb && !FeConstants.runningUnitTest && cascadesContext.isLeadingJoin()) { List scans = cascadesContext.getRewritePlan() .collectToList(LogicalOlapScan.class::isInstance); StatsCalculator.disableJoinReorderIfTableRowCountNotAvailable(scans, cascadesContext); From 9d7530e3e61db7b66c00e00b719794a2424b9e20 Mon Sep 17 00:00:00 2001 From: englefly Date: Fri, 27 Sep 2024 23:02:34 +0800 Subject: [PATCH 3/3] fmt --- .../doris/nereids/stats/StatsCalculator.java | 84 +++++++++---------- 1 file changed, 42 insertions(+), 42 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java index da61d442fcf024..9458407295d903 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java @@ -213,48 +213,6 @@ public static StatsCalculator estimate(GroupExpression groupExpression, boolean return statsCalculator; } - /** - * disable join reorder if any table row count is not available. - */ - public static void disableJoinReorderIfTableRowCountNotAvailable( - List scans, CascadesContext context) { - StatsCalculator calculator = new StatsCalculator(context); - for (LogicalOlapScan scan : scans) { - double rowCount = calculator.getOlapTableRowCount(scan); - if (rowCount == -1 && ConnectContext.get() != null) { - try { - ConnectContext.get().getSessionVariable().disableNereidsJoinReorderOnce(); - LOG.info("disable join reorder since row count not available: " - + scan.getTable().getNameWithFullQualifiers()); - } catch (Exception e) { - LOG.info("disableNereidsJoinReorderOnce failed"); - } - return; - } - } - } - - /** - * if the table is not analyzed and BE does not report row count, return -1 - */ - private double getOlapTableRowCount(OlapScan olapScan) { - OlapTable olapTable = olapScan.getTable(); - AnalysisManager analysisManager = Env.getCurrentEnv().getAnalysisManager(); - TableStatsMeta tableMeta = analysisManager.findTableStatsStatus(olapScan.getTable().getId()); - double rowCount = -1; - if (tableMeta != null && tableMeta.userInjected) { - rowCount = tableMeta.getRowCount(olapScan.getSelectedIndexId()); - } else { - rowCount = olapTable.getRowCountForIndex(olapScan.getSelectedIndexId(), true); - if (rowCount == -1) { - if (tableMeta != null) { - rowCount = tableMeta.getRowCount(olapScan.getSelectedIndexId()); - } - } - } - return rowCount; - } - public static StatsCalculator estimate(GroupExpression groupExpression, boolean forbidUnknownColStats, Map columnStatisticMap, boolean isPlayNereidsDump, CascadesContext context) { return StatsCalculator.estimate(groupExpression, @@ -1180,4 +1138,46 @@ public Statistics visitPhysicalCTEAnchor( return groupExpression.childStatistics(1); } + /** + * if the table is not analyzed and BE does not report row count, return -1 + */ + private double getOlapTableRowCount(OlapScan olapScan) { + OlapTable olapTable = olapScan.getTable(); + AnalysisManager analysisManager = Env.getCurrentEnv().getAnalysisManager(); + TableStatsMeta tableMeta = analysisManager.findTableStatsStatus(olapScan.getTable().getId()); + double rowCount = -1; + if (tableMeta != null && tableMeta.userInjected) { + rowCount = tableMeta.getRowCount(olapScan.getSelectedIndexId()); + } else { + rowCount = olapTable.getRowCountForIndex(olapScan.getSelectedIndexId(), true); + if (rowCount == -1) { + if (tableMeta != null) { + rowCount = tableMeta.getRowCount(olapScan.getSelectedIndexId()); + } + } + } + return rowCount; + } + + /** + * disable join reorder if any table row count is not available. + */ + public static void disableJoinReorderIfTableRowCountNotAvailable( + List scans, + CascadesContext context) { + StatsCalculator calculator = new StatsCalculator(context); + for (LogicalOlapScan scan : scans) { + double rowCount = calculator.getOlapTableRowCount(scan); + if (rowCount == -1 && ConnectContext.get() != null) { + try { + ConnectContext.get().getSessionVariable().disableNereidsJoinReorderOnce(); + LOG.info("disable join reorder since row count not available: " + + scan.getTable().getNameWithFullQualifiers()); + } catch (Exception e) { + LOG.info("disableNereidsJoinReorderOnce failed"); + } + return; + } + } + } }