From 915acd040ea3070e22ff741c5a6621fd439fafb7 Mon Sep 17 00:00:00 2001 From: EmmyMiao87 <522274284@qq.com> Date: Tue, 13 Jul 2021 20:27:36 +0800 Subject: [PATCH] [JoinReorder] Implement a better join reorder algorithm. The current JoinReorder algorithm mainly sorts according to the star model, and only considers the query association relationship between the table and the table. The problems are folllowing: 1. Only applicable to user data whose data model is a star model, data of other models cannot be sorted. 2. Regardless of the cost of the table, it is impossible to determine the size of the join table relationship, and the real query optimization ability is weak. 3. It is impossible to avoid possible time-consuming joins such as cross joins by sorting. The new JoinReorder algorithm mainly introduces a new sorting algorithm for Join The new ranking algorithm introduces the cost evaluation model to Doris. The sorting algorithm is mainly based on the following three principles: 1. The order is: Largest node, Smallest node. . . Second largest node 2. Cross join is better than Inner join 3. The right children of Outer join, semi join, and anti join do not move PlanNode's cost model evaluation mainly relies on two values: cardinality and selectivity. cardinality: cardinality, can also be simply understood as the number of rows. selectivity: selectivity, a value between 0 and 1. Predicate generally has selectivity. The cost model generally calculates the final cardinality of a PlanNode based on the pre-calculated cardinality of PlanNode and the selectivity of the predicate to which it belongs. Currently, you can configure "enable_cost_based_join_reorder" to control the opening and closing of JoinReorder. When the configuration is turned on, the new sorting algorithm will take effect, when it is turned off, the old sorting algorithm will take effect, and it is turned off by default. The new sorting algorithm currently has no cost base evaluation for external tables (odbc, es) and set calculations (intersect, except). When using these queries, it is not recommended to enable cost base join reoder. When using these queries, it is not recommended to enable cost base join reoder. At the code architecture level: 1. The new sorting algorithm occurs in the single-node execution planning stage. 2. Refactored the init and finalize phases of PlanNode to ensure that PlanNode planning and cost evaluation have been completed before the sorting algorithm occurs. Fixed #6225 Change-Id: I6a89b20d70ff1d49909900f39337fa31cc8efd1e --- fe/fe-core/src/main/cup/sql_parser.cup | 4 + .../org/apache/doris/analysis/Analyzer.java | 50 +- .../doris/analysis/BinaryPredicate.java | 24 + .../doris/analysis/DescriptorTable.java | 13 +- .../java/org/apache/doris/analysis/Expr.java | 34 +- .../org/apache/doris/analysis/FromClause.java | 66 +- .../org/apache/doris/analysis/SelectStmt.java | 5 +- .../apache/doris/analysis/SlotDescriptor.java | 12 + .../org/apache/doris/analysis/Subquery.java | 2 +- .../org/apache/doris/analysis/TableRef.java | 1 + .../doris/analysis/TupleDescriptor.java | 71 +- .../org/apache/doris/common/CheckedMath.java | 54 + .../java/org/apache/doris/load/ExportJob.java | 6 +- .../doris/load/loadv2/LoadLoadingTask.java | 2 +- .../doris/load/loadv2/LoadingTaskPlanner.java | 2 +- .../doris/load/loadv2/SparkLoadJob.java | 2 +- .../apache/doris/planner/AggregationNode.java | 47 +- .../doris/planner/AnalyticEvalNode.java | 23 +- .../apache/doris/planner/AnalyticPlanner.java | 7 +- .../doris/planner/AssertNumRowsNode.java | 18 + .../apache/doris/planner/CrossJoinNode.java | 48 +- .../apache/doris/planner/EmptySetNode.java | 10 +- .../org/apache/doris/planner/EsScanNode.java | 1 + .../apache/doris/planner/ExchangeNode.java | 19 +- .../apache/doris/planner/HashJoinNode.java | 317 +++- .../apache/doris/planner/LoadScanNode.java | 2 +- .../apache/doris/planner/MergeJoinNode.java | 170 --- .../org/apache/doris/planner/MergeNode.java | 31 +- .../apache/doris/planner/MysqlScanNode.java | 7 +- .../apache/doris/planner/OdbcScanNode.java | 7 +- .../apache/doris/planner/OlapScanNode.java | 116 +- .../org/apache/doris/planner/PlanNode.java | 227 ++- .../org/apache/doris/planner/Planner.java | 20 +- .../org/apache/doris/planner/RepeatNode.java | 17 +- .../org/apache/doris/planner/ScanNode.java | 8 + .../org/apache/doris/planner/SelectNode.java | 28 +- .../doris/planner/SetOperationNode.java | 56 +- .../doris/planner/SingleNodePlanner.java | 407 ++++- .../org/apache/doris/planner/SortNode.java | 13 + .../doris/planner/StreamLoadPlanner.java | 2 +- .../doris/planner/StreamLoadScanNode.java | 2 +- .../org/apache/doris/qe/SessionVariable.java | 7 + .../org/apache/doris/qe/StmtExecutor.java | 2 +- .../apache/doris/planner/QueryPlanTest.java | 2 + .../doris/planner/SingleNodePlannerTest.java | 1358 ++++++++++++++++- fe/fe-core/src/test/resources/log4j2.xml | 2 +- 46 files changed, 2846 insertions(+), 476 deletions(-) create mode 100644 fe/fe-core/src/main/java/org/apache/doris/common/CheckedMath.java delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/planner/MergeJoinNode.java diff --git a/fe/fe-core/src/main/cup/sql_parser.cup b/fe/fe-core/src/main/cup/sql_parser.cup index c4543364574e96..c8bef943a7f721 100644 --- a/fe/fe-core/src/main/cup/sql_parser.cup +++ b/fe/fe-core/src/main/cup/sql_parser.cup @@ -2733,6 +2733,10 @@ opt_alter_type ::= {: RESULT = ShowAlterStmt.AlterType.ROLLUP; :} + | KW_MATERIALIZED KW_VIEW + {: + RESULT = ShowAlterStmt.AlterType.ROLLUP; + :} | KW_COLUMN {: RESULT = ShowAlterStmt.AlterType.COLUMN; diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/Analyzer.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/Analyzer.java index a879f08a977e4b..deb84137b9e33b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/Analyzer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/Analyzer.java @@ -152,7 +152,7 @@ public class Analyzer { // The runtime filter that is expected to be used private final List assignedRuntimeFilters = new ArrayList<>(); - + public void setIsSubquery() { isSubquery = true; globalState.containsSubquery = true; @@ -206,8 +206,8 @@ private static class GlobalState { private final Map> eqJoinConjuncts = Maps.newHashMap(); // set of conjuncts that have been assigned to some PlanNode - private final Set assignedConjuncts = - Collections.newSetFromMap(new IdentityHashMap()); + private Set assignedConjuncts = + Collections.newSetFromMap(new IdentityHashMap()); // map from outer-joined tuple id, ie, one that is nullable in this select block, // to the last Join clause (represented by its rhs table ref) that outer-joined it @@ -849,6 +849,15 @@ private void registerConstantConjunct(TupleId id, Expr e) { } } + /** + * register expr id + * + * @param expr + */ + void registerExprId(Expr expr) { + expr.setId(globalState.conjunctIdGenerator.getNextId()); + } + /** * Register individual conjunct with all tuple and slot ids it references * and with the global conjunct list. @@ -945,6 +954,16 @@ public void createAuxEquivPredicate(Expr lhs, Expr rhs) { registerConjunct(p); } + public Set getAssignedConjuncts() { + return Sets.newHashSet(globalState.assignedConjuncts); + } + + public void setAssignedConjuncts(Set assigned) { + if (assigned != null) { + globalState.assignedConjuncts = Sets.newHashSet(assigned); + } + } + /** * Return all unassigned registered conjuncts that are fully bound by the given * (logical) tuple ids, can be evaluated by 'tupleIds' and are not tied to an @@ -952,7 +971,7 @@ public void createAuxEquivPredicate(Expr lhs, Expr rhs) { */ public List getUnassignedConjuncts(List tupleIds) { List result = Lists.newArrayList(); - for (Expr e: getUnassignedConjuncts(tupleIds, true)) { + for (Expr e : getUnassignedConjuncts(tupleIds, true)) { if (canEvalPredicate(tupleIds, e)) result.add(e); } return result; @@ -1256,7 +1275,7 @@ private void markConstantConjunct(Expr conjunct, boolean fromHavingClause) } final Expr newConjunct = conjunct.getResultValue(); if (newConjunct instanceof BoolLiteral) { - final BoolLiteral value = (BoolLiteral)newConjunct; + final BoolLiteral value = (BoolLiteral) newConjunct; if (!value.getValue()) { if (fromHavingClause) { hasEmptyResultSet_ = true; @@ -1597,6 +1616,17 @@ public ExprSubstitutionMap getChangeResSmap() { public void setChangeResSmap(ExprSubstitutionMap changeResSmap) { this.changeResSmap = changeResSmap; } + + // Load plan and query plan are the same framework + // Some Load method in doris access through http protocol, which will cause the session may be empty. + // In order to avoid the occurrence of null pointer exceptions, a check will be added here + public boolean safeIsEnableJoinReorderBasedCost() { + if (globalState.context == null) { + return false; + } + return globalState.context.getSessionVariable().isEnableJoinReorderBasedCost(); + } + /** * Returns true if predicate 'e' can be correctly evaluated by a tree materializing * 'tupleIds', otherwise false: @@ -1783,7 +1813,7 @@ public boolean containSubquery() { * materialization decision be cost-based? */ public void markRefdSlots(Analyzer analyzer, PlanNode planRoot, - List outputExprs, AnalyticInfo analyticInfo) { + List outputExprs, AnalyticInfo analyticInfo) { if (planRoot == null) { return; } @@ -1824,7 +1854,7 @@ public void markRefdSlots(Analyzer analyzer, PlanNode planRoot, /** * Column conduction, can slot a value-transfer to slot b - * + *

* TODO(zxy) Use value-transfer graph to check */ public boolean hasValueTransfer(SlotId a, SlotId b) { @@ -1834,7 +1864,7 @@ public boolean hasValueTransfer(SlotId a, SlotId b) { /** * Returns sorted slot IDs with value transfers from 'srcSid'. * Time complexity: O(V) where V = number of slots - * + *

* TODO(zxy) Use value-transfer graph to check */ public List getValueTransferTargets(SlotId srcSid) { @@ -1848,8 +1878,8 @@ public List getValueTransferTargets(SlotId srcSid) { * to an outer-joined tuple. */ public boolean hasOuterJoinedValueTransferTarget(List sids) { - for (SlotId srcSid: sids) { - for (SlotId dstSid: getValueTransferTargets(srcSid)) { + for (SlotId srcSid : sids) { + for (SlotId dstSid : getValueTransferTargets(srcSid)) { if (isOuterJoined(getTupleId(dstSid))) return true; } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/BinaryPredicate.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/BinaryPredicate.java index 8c3da5dcddc7c3..038a00e03023d0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/BinaryPredicate.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/BinaryPredicate.java @@ -622,6 +622,30 @@ private Expr compareLiteral(LiteralExpr first, LiteralExpr second) throws Analys return this; } + @Override + public void setSelectivity() { + switch(op) { + case EQ: + case EQ_FOR_NULL: { + Reference slotRefRef = new Reference(); + boolean singlePredicate = isSingleColumnPredicate(slotRefRef, null); + if (singlePredicate) { + long distinctValues = slotRefRef.getRef().getNumDistinctValues(); + if (distinctValues != -1) { + selectivity = 1.0 / distinctValues; + } + } + break; + } default: { + // Reference hive + selectivity = 1.0 / 3.0; + break; + } + } + + return; + } + @Override public int hashCode() { return 31 * super.hashCode() + Objects.hashCode(op); diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/DescriptorTable.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/DescriptorTable.java index 443dc323c07801..0619a62477fb45 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/DescriptorTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/DescriptorTable.java @@ -83,7 +83,7 @@ public TupleDescriptor copyTupleDescriptor(TupleId srcId, String debugName) { for (SlotDescriptor slot: src.getSlots()) { copySlotDescriptor(d, slot); } - d.computeMemLayout(); + d.computeStatAndMemLayout(); return d; } @@ -122,14 +122,21 @@ public void markSlotsMaterialized(List ids) { } } - // Computes physical layout parameters of all descriptors. - // Call this only after the last descriptor was added. + @Deprecated public void computeMemLayout() { for (TupleDescriptor d : tupleDescs.values()) { d.computeMemLayout(); } } + // Computes physical layout parameters of all descriptors and calculate the statistics of the tuple. + // Call this only after the last descriptor was added. + public void computeStatAndMemLayout() { + for (TupleDescriptor d : tupleDescs.values()) { + d.computeStatAndMemLayout(); + } + } + public TDescriptorTable toThrift() { TDescriptorTable result = new TDescriptorTable(); HashSet referencedTbls = Sets.newHashSet(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/Expr.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/Expr.java index 1b3dabae9f87f3..f27a3f107f1961 100755 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/Expr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/Expr.java @@ -63,7 +63,7 @@ abstract public class Expr extends TreeNode implements ParseNode, Cloneabl private static final String NEGATE_FN = "negate"; // to be used where we can't come up with a better estimate - protected static final double DEFAULT_SELECTIVITY = 0.1; + public static final double DEFAULT_SELECTIVITY = 0.1; public final static float FUNCTION_CALL_COST = 10; @@ -180,6 +180,10 @@ public boolean apply(Expr arg) { public boolean apply(Expr arg) { return arg instanceof NullLiteral; } }; + public void setSelectivity() { + selectivity = -1; + } + /* TODO(zc) public final static com.google.common.base.Predicate IS_NONDETERMINISTIC_BUILTIN_FN_PREDICATE = @@ -299,6 +303,8 @@ public double getSelectivity() { return selectivity; } + public boolean hasSelectivity() { return selectivity >= 0; } + public long getNumDistinctValues() { return numDistinctValues; } @@ -374,6 +380,9 @@ public final void analyze(Analyzer analyzer) throws AnalysisException { // Do all the analysis for the expr subclass before marking the Expr analyzed. analyzeImpl(analyzer); + if (analyzer.safeIsEnableJoinReorderBasedCost()) { + setSelectivity(); + } analysisDone(); } @@ -1420,6 +1429,29 @@ public Expr unwrapExpr(boolean implicitOnly) { return this; } + /** + * Returns the descriptor of the scan slot that directly or indirectly produces + * the values of 'this' SlotRef. Traverses the source exprs of intermediate slot + * descriptors to resolve materialization points (e.g., aggregations). + * Returns null if 'e' or any source expr of 'e' is not a SlotRef or cast SlotRef. + */ + public SlotDescriptor findSrcScanSlot() { + SlotRef slotRef = unwrapSlotRef(false); + if (slotRef == null) { + return null; + } + SlotDescriptor slotDesc = slotRef.getDesc(); + if (slotDesc.isScanSlot()) { + return slotDesc; + } + if (slotDesc.getSourceExprs().size() == 1) { + return slotDesc.getSourceExprs().get(0).findSrcScanSlot(); + } + // No known source expr, or there are several source exprs meaning the slot is + // has no single source table. + return null; + } + public static double getConstFromExpr(Expr e) throws AnalysisException{ Preconditions.checkState(e.isConstant()); double value = 0; diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/FromClause.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/FromClause.java index b5f1b12889c003..5f09a1454d0a5b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/FromClause.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/FromClause.java @@ -17,11 +17,6 @@ package org.apache.doris.analysis; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Comparator; -import java.util.Iterator; -import java.util.List; import org.apache.doris.catalog.Database; import org.apache.doris.catalog.Table; @@ -31,10 +26,16 @@ import org.apache.doris.common.ErrorReport; import org.apache.doris.common.UserException; -import com.google.common.base.Strings; import com.google.common.base.Preconditions; +import com.google.common.base.Strings; import com.google.common.collect.Lists; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.Iterator; +import java.util.List; + /** * Wraps a list of TableRef instances that form a FROM clause, allowing them to be * analyzed independently of the statement using them. To increase the flexibility of @@ -61,23 +62,6 @@ public void setNeedToSql(boolean needToSql) { this.needToSql = needToSql; } - private void sortTableRefForSubquery(Analyzer analyzer) { - Collections.sort(this.tableRefs_, new Comparator() { - @Override - public int compare(TableRef tableref1, TableRef tableref2) { - int i1 = 0; - int i2 = 0; - if (tableref1.getOnClause() != null) { - i1 = 1; - } - if (tableref2.getOnClause() != null) { - i2 = 1; - } - return i1 - i2; - } - }); - } - private void checkFromHiveTable(Analyzer analyzer) throws AnalysisException { for (TableRef tblRef : tableRefs_) { if (!(tblRef instanceof BaseTableRef)) { @@ -111,6 +95,33 @@ private void checkFromHiveTable(Analyzer analyzer) throws AnalysisException { } } + /** + * In some cases, the reorder method of select stmt will incorrectly sort the tableRef with on clause. + * The meaning of this function is to reset those tableRefs with on clauses. + * For example: + * Origin stmt: select * from t1 inner join t2 on t1.k1=t2.k1 + * After analyze: select * from t2 on t1.k1=t2.k1 inner join t1 + * + * If this statement just needs to be reanalyze (query rewriter), an error will be reported + * because the table t1 in the on clause cannot be recognized. + */ + private void sortTableRefKeepSequenceOfOnClause() { + Collections.sort(this.tableRefs_, new Comparator() { + @Override + public int compare(TableRef tableref1, TableRef tableref2) { + int i1 = 0; + int i2 = 0; + if (tableref1.getOnClause() != null) { + i1 = 1; + } + if (tableref2.getOnClause() != null) { + i2 = 1; + } + return i1 - i2; + } + }); + } + @Override public void analyze(Analyzer analyzer) throws AnalysisException, UserException { if (analyzed_) return; @@ -120,7 +131,14 @@ public void analyze(Analyzer analyzer) throws AnalysisException, UserException { return; } - sortTableRefForSubquery(analyzer); + // The order of the tables may have changed during the previous analyzer process. + // For example, a join b on xxx is changed to b on xxx join a. + // This change will cause the predicate in on clause be adjusted to the front of the association table, + // causing semantic analysis to fail. Unknown column 'column1' in 'table1' + // So we need to readjust the order of the tables here. + if (!analyzer.safeIsEnableJoinReorderBasedCost()) { + sortTableRefKeepSequenceOfOnClause(); + } // Start out with table refs to establish aliases. TableRef leftTblRef = null; // the one to the left of tblRef diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/SelectStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/SelectStmt.java index d67ad8bf1511f0..da372839625a6d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/SelectStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/SelectStmt.java @@ -515,7 +515,10 @@ public void analyze(Analyzer analyzer) throws AnalysisException, UserException { if (needToSql) { sqlString_ = toSql(); } - reorderTable(analyzer); + if (!analyzer.safeIsEnableJoinReorderBasedCost()) { + LOG.debug("use old reorder logical in select stmt"); + reorderTable(analyzer); + } resolveInlineViewRefs(analyzer); diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/SlotDescriptor.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/SlotDescriptor.java index 17f1d5a86455d7..8803bd05ba640a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/SlotDescriptor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/SlotDescriptor.java @@ -19,6 +19,8 @@ import org.apache.doris.catalog.Column; import org.apache.doris.catalog.ColumnStats; +import org.apache.doris.catalog.OlapTable; +import org.apache.doris.catalog.Table; import org.apache.doris.catalog.Type; import org.apache.doris.thrift.TSlotDescriptor; @@ -195,6 +197,8 @@ public ColumnStats getStats() { stats = new ColumnStats(); } } + // FIXME(dhc): mock ndv + stats.setNumDistinctValues(parent.getCardinality()); return stats; } @@ -290,4 +294,12 @@ public String getExplainString(String prefix) { builder.append(prefix).append("slotIdx=").append(slotIdx).append("\n"); return builder.toString(); } + + public boolean isScanSlot() { + Table table = parent.getTable(); + if ((table != null) && (table instanceof OlapTable)) { + return true; + } + return false; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/Subquery.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/Subquery.java index e6dffd8e46b5bf..2d8cb978c89590 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/Subquery.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/Subquery.java @@ -87,7 +87,7 @@ public void analyzeImpl(Analyzer parentAnalyzer) throws AnalysisException { try { stmt.analyze(analyzer); } catch (UserException e) { - throw new AnalysisException(e.getMessage()); + throw new AnalysisException(e.getMessage(), e); } // Check whether the stmt_ contains an illegal mix of un/correlated table refs. stmt.getCorrelatedTupleIds(analyzer); diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/TableRef.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/TableRef.java index 27dc380423707c..f18d57473dcf5e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/TableRef.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/TableRef.java @@ -638,6 +638,7 @@ public String getExplicitAlias() { return null; } + public boolean isAnalyzed() { return isAnalyzed; } public boolean isResolved() { return !getClass().equals(TableRef.class); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/TupleDescriptor.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/TupleDescriptor.java index 0b4553c445d707..9a7b7b7bd0ec47 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/TupleDescriptor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/TupleDescriptor.java @@ -62,18 +62,24 @@ public class TupleDescriptor { private int numNullBytes; private int numNullableSlots; + // This cardinality is only used to mock slot ndv. + // Only tuple of olap scan node has this value. + private long cardinality; + private float avgSerializedSize; // in bytes; includes serialization overhead public TupleDescriptor(TupleId id) { this.id = id; this.slots = new ArrayList(); this.debugName = ""; + this.cardinality = -1; } public TupleDescriptor(TupleId id, String debugName) { this.id = id; this.slots = new ArrayList(); this.debugName = debugName; + this.cardinality = -1; } public void addSlot(SlotDescriptor desc) { @@ -97,6 +103,14 @@ public ArrayList getSlots() { return slots; } + public void setCardinality(long cardinality) { + this.cardinality = cardinality; + } + + public long getCardinality() { + return cardinality; + } + public ArrayList getMaterializedSlots() { ArrayList result = Lists.newArrayList(); for (SlotDescriptor slot : slots) { @@ -162,6 +176,57 @@ public TTupleDescriptor toThrift() { return ttupleDesc; } + /** + * This function is mainly used to calculate the statistics of the tuple and the layout information. + * Generally, it occurs after the plan node materializes the slot and before calculating the plan node statistics. + * PlanNode.init() { + * materializedSlot(); + * tupleDesc.computeStatAndMemLayout(); + * computeStat(); + * } + */ + public void computeStatAndMemLayout() { + computeStat(); + computeMemLayout(); + } + + /** + * This function is mainly used to evaluate the statistics of the tuple, + * such as the average size of each row. + * This function will be used before the computeStat() of the plan node + * and is the pre-work for evaluating the statistics of the plan node. + * + * This function is theoretically only called once when the plan node is init. + * However, the current code structure is relatively confusing + * In order to ensure that even if it is wrongly called a second time, no error will occur, + * so it will be initialized again at the beginning of the function. + * + * In the future this function will be changed to a private function. + */ + @Deprecated + public void computeStat() { + // init stat + avgSerializedSize = 0; + + // compute stat + for (SlotDescriptor d : slots) { + if (!d.isMaterialized()) { + continue; + } + ColumnStats stats = d.getStats(); + if (stats.hasAvgSerializedSize()) { + avgSerializedSize += d.getStats().getAvgSerializedSize(); + } else { + // TODO: for computed slots, try to come up with stats estimates + avgSerializedSize += d.getType().getSlotSize(); + } + } + } + + /** + * In the future this function will be changed to a private function. + */ + @Deprecated public void computeMemLayout() { // sort slots by size List> slotsBySize = Lists.newArrayListWithCapacity(PrimitiveType.getMaxSlotSize()); @@ -173,12 +238,6 @@ public void computeMemLayout() { numNullableSlots = 0; for (SlotDescriptor d : slots) { ColumnStats stats = d.getStats(); - if (stats.hasAvgSerializedSize()) { - avgSerializedSize += d.getStats().getAvgSerializedSize(); - } else { - // TODO: for computed slots, try to come up with stats estimates - avgSerializedSize += d.getType().getSlotSize(); - } if (d.isMaterialized()) { slotsBySize.get(d.getType().getSlotSize()).add(d); if (d.getIsNullable()) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/CheckedMath.java b/fe/fe-core/src/main/java/org/apache/doris/common/CheckedMath.java new file mode 100644 index 00000000000000..2e4f641b2fd042 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/common/CheckedMath.java @@ -0,0 +1,54 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.common; + +import com.google.common.math.LongMath; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +public class CheckedMath { + + private final static Logger LOG = LogManager.getLogger(CheckedMath.class); + + /** + * Computes and returns the multiply of two longs. If an overflow occurs, + * the maximum Long value is returned (Long.MAX_VALUE). + */ + public static long checkedMultiply(long a, long b) { + try { + return LongMath.checkedMultiply(a, b); + } catch (ArithmeticException e) { + LOG.warn("overflow when multiplying longs: " + a + ", " + b); + return Long.MAX_VALUE; + } + } + + /** + * Computes and returns the sum of two longs. If an overflow occurs, + * the maximum Long value is returned (Long.MAX_VALUE). + */ + public static long checkedAdd(long a, long b) { + try { + return LongMath.checkedAdd(a, b); + } catch (ArithmeticException e) { + LOG.warn("overflow when adding longs: " + a + ", " + b); + return Long.MAX_VALUE; + } + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/ExportJob.java b/fe/fe-core/src/main/java/org/apache/doris/load/ExportJob.java index 1946c03bac5950..a07286bc76a485 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/ExportJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/ExportJob.java @@ -183,7 +183,7 @@ public ExportJob() { this.finishTimeMs = -1; this.failMsg = new ExportFailMsg(ExportFailMsg.CancelType.UNKNOWN, ""); this.analyzer = new Analyzer(Catalog.getCurrentCatalog(), null); - this.desc = new DescriptorTable(); + this.desc = analyzer.getDescTbl(); this.exportPath = ""; this.columnSeparator = "\t"; this.lineDelimiter = "\n"; @@ -286,7 +286,7 @@ private void registerToDesc() { } } } - desc.computeMemLayout(); + desc.computeStatAndMemLayout(); } private void plan() throws UserException { @@ -379,7 +379,6 @@ private ScanNode genScanNode() throws UserException { ((OlapScanNode) scanNode).setColumnFilters(Maps.newHashMap()); ((OlapScanNode) scanNode).setIsPreAggregation(false, "This an export operation"); ((OlapScanNode) scanNode).setCanTurnOnPreAggr(false); - scanNode.init(analyzer); ((OlapScanNode) scanNode).selectBestRollupByRollupSelector(analyzer); break; case ODBC: @@ -392,6 +391,7 @@ private ScanNode genScanNode() throws UserException { break; } if (scanNode != null) { + scanNode.init(analyzer); scanNode.finalize(analyzer); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadLoadingTask.java b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadLoadingTask.java index c8bebbefe50172..f9ad624eff57e2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadLoadingTask.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadLoadingTask.java @@ -106,7 +106,7 @@ public TUniqueId getLoadId() { @Override protected void executeTask() throws Exception{ - LOG.info("begin to execute loading task. load id: {} job: {}. db: {}, tbl: {}. left retry: {}", + LOG.info("begin to execute loading task. load id: {} job id: {}. db: {}, tbl: {}. left retry: {}", DebugUtil.printId(loadId), callback.getCallbackId(), db.getFullName(), table.getName(), retryTime); retryTime--; beginTime = System.nanoTime(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadingTaskPlanner.java b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadingTaskPlanner.java index 61f767026040b8..2d8d574e7d16a4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadingTaskPlanner.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadingTaskPlanner.java @@ -127,7 +127,7 @@ public void plan(TUniqueId loadId, List> fileStatusesLis scanNode.init(analyzer); scanNode.finalize(analyzer); scanNodes.add(scanNode); - descTable.computeMemLayout(); + descTable.computeStatAndMemLayout(); // 2. Olap table sink List partitionIds = getAllPartitionIds(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/SparkLoadJob.java b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/SparkLoadJob.java index a9ab2d35ea6de5..30fceb594e520f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/SparkLoadJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/SparkLoadJob.java @@ -950,7 +950,7 @@ private Expr castToSlot(SlotDescriptor slotDesc, Expr expr) throws AnalysisExcep } private void initTDescriptorTable(DescriptorTable descTable) { - descTable.computeMemLayout(); + descTable.computeStatAndMemLayout(); tDescriptorTable = descTable.toThrift(); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/AggregationNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/AggregationNode.java index 8fcba78d7be1ba..667e34e4acec58 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/AggregationNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/AggregationNode.java @@ -148,8 +148,8 @@ public void init(Analyzer analyzer) throws UserException { // conjuncts_ = orderConjunctsByCost(conjuncts_); // Compute the mem layout for both tuples here for simplicity. - aggInfo.getOutputTupleDesc().computeMemLayout(); - aggInfo.getIntermediateTupleDesc().computeMemLayout(); + aggInfo.getOutputTupleDesc().computeStatAndMemLayout(); + aggInfo.getIntermediateTupleDesc().computeStatAndMemLayout(); // do this at the end so it can take all conjuncts into account computeStats(analyzer); @@ -167,14 +167,16 @@ public void init(Analyzer analyzer) throws UserException { @Override public void computeStats(Analyzer analyzer) { super.computeStats(analyzer); + if (!analyzer.safeIsEnableJoinReorderBasedCost()) { + return; + } List groupingExprs = aggInfo.getGroupingExprs(); cardinality = 1; // cardinality: product of # of distinct values produced by grouping exprs for (Expr groupingExpr : groupingExprs) { long numDistinct = groupingExpr.getNumDistinctValues(); - // TODO: remove these before 1.0 LOG.debug("grouping expr: " + groupingExpr.toSql() + " #distinct=" + Long.toString( - numDistinct)); + numDistinct)); if (numDistinct == -1) { cardinality = -1; break; @@ -190,11 +192,42 @@ public void computeStats(Analyzer analyzer) { // some others, the estimate doesn't overshoot dramatically) cardinality *= numDistinct; } + if (cardinality > 0) { + LOG.debug("sel=" + Double.toString(computeSelectivity())); + applyConjunctsSelectivity(); + } + // if we ended up with an overflow, the estimate is certain to be wrong + if (cardinality < 0) { + cardinality = -1; + } + + capCardinalityAtLimit(); + if (LOG.isDebugEnabled()) { + LOG.debug("stats Agg: cardinality={}", cardinality); + } + } + + @Override + protected void computeOldCardinality() { + List groupingExprs = aggInfo.getGroupingExprs(); + cardinality = 1; + // cardinality: product of # of distinct values produced by grouping exprs + for (Expr groupingExpr : groupingExprs) { + long numDistinct = groupingExpr.getNumDistinctValues(); + // TODO: remove these before 1.0 + LOG.debug("grouping expr: " + groupingExpr.toSql() + " #distinct=" + Long.toString( + numDistinct)); + if (numDistinct == -1) { + cardinality = -1; + break; + } + cardinality *= numDistinct; + } // take HAVING predicate into account LOG.debug("Agg: cardinality=" + Long.toString(cardinality)); if (cardinality > 0) { - cardinality = Math.round((double) cardinality * computeSelectivity()); - LOG.debug("sel=" + Double.toString(computeSelectivity())); + cardinality = Math.round((double) cardinality * computeOldSelectivity()); + LOG.debug("sel=" + Double.toString(computeOldSelectivity())); } // if we ended up with an overflow, the estimate is certain to be wrong if (cardinality < 0) { @@ -277,6 +310,8 @@ public String getNodeExplainString(String detailPrefix, TExplainLevel detailLeve if (!conjuncts.isEmpty()) { output.append(detailPrefix + "having: ").append(getExplainString(conjuncts) + "\n"); } + output.append(detailPrefix).append(String.format( + "cardinality=%s", cardinality)).append("\n"); return output.toString(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/AnalyticEvalNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/AnalyticEvalNode.java index d06e4dc05e6c6a..239a1ce1121854 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/AnalyticEvalNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/AnalyticEvalNode.java @@ -30,15 +30,15 @@ import org.apache.doris.thrift.TPlanNodeType; import org.apache.doris.thrift.TQueryOptions; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import com.google.common.base.Joiner; import com.google.common.base.MoreObjects; import com.google.common.base.Preconditions; import com.google.common.collect.Lists; import com.google.common.collect.Sets; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import java.util.List; /** @@ -108,8 +108,8 @@ public List getOrderByElements() { @Override public void init(Analyzer analyzer) throws UserException { - analyzer.getDescTbl().computeMemLayout(); - intermediateTupleDesc.computeMemLayout(); + analyzer.getDescTbl().computeStatAndMemLayout(); + intermediateTupleDesc.computeStatAndMemLayout(); // we add the analyticInfo's smap to the combined smap of our child outputSmap = logicalToPhysicalSmap; createDefaultSmap(analyzer); @@ -138,6 +138,19 @@ public void init(Analyzer analyzer) throws UserException { @Override protected void computeStats(Analyzer analyzer) { super.computeStats(analyzer); + if (!analyzer.safeIsEnableJoinReorderBasedCost()) { + return; + } + cardinality = cardinality == -1 ? getChild(0).cardinality : cardinality; + applyConjunctsSelectivity(); + capCardinalityAtLimit(); + if (LOG.isDebugEnabled()) { + LOG.debug("stats AnalyticEval: cardinality={}", cardinality); + } + } + + @Override + protected void computeOldCardinality() { cardinality = getChild(0).cardinality; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/AnalyticPlanner.java b/fe/fe-core/src/main/java/org/apache/doris/planner/AnalyticPlanner.java index 9dbe1a3a00d897..42420ad70501c2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/AnalyticPlanner.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/AnalyticPlanner.java @@ -181,7 +181,7 @@ private void mergePartitionGroups( long ndv = Expr.getNumDistinctValues( Expr.intersect(pg1.partitionByExprs, pg2.partitionByExprs)); - if (ndv == -1 || ndv < 0 || ndv < numNodes) { + if (ndv == -1 || ndv < 1 || ndv < numNodes) { // didn't get a usable value or the number of partitions is too small continue; } @@ -228,7 +228,7 @@ private void computeInputPartitionExprs(List partitionGroups, // TODO: also look at l2 and take the max? long ndv = Expr.getNumDistinctValues(l1); - if (ndv < 0 || ndv < numNodes || ndv < maxNdv) { + if (ndv < 1 || ndv < numNodes || ndv < maxNdv) { continue; } @@ -673,8 +673,7 @@ public void init(Analyzer analyzer, String tupleName) { logicalToPhysicalSmap.put(new SlotRef(logicalOutputSlot), new SlotRef(physicalOutputSlot)); } - physicalOutputTuple.computeMemLayout(); - // if (requiresIntermediateTuple) physicalIntermediateTuple.computeMemLayout(); + physicalOutputTuple.computeStatAndMemLayout(); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/AssertNumRowsNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/AssertNumRowsNode.java index 86b978d5220de6..f8196e5df66f85 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/AssertNumRowsNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/AssertNumRowsNode.java @@ -17,12 +17,17 @@ package org.apache.doris.planner; +import org.apache.doris.analysis.Analyzer; import org.apache.doris.analysis.AssertNumRowsElement; +import org.apache.doris.common.UserException; import org.apache.doris.thrift.TAssertNumRowsNode; import org.apache.doris.thrift.TExplainLevel; import org.apache.doris.thrift.TPlanNode; import org.apache.doris.thrift.TPlanNodeType; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + /** * Assert num rows node is used to determine whether the number of rows is less then desired num of rows. * The rows are the result of subqueryString. @@ -30,6 +35,7 @@ * The cancelled reason will be reported by Backend and displayed back to the user. */ public class AssertNumRowsNode extends PlanNode { + private static final Logger LOG = LogManager.getLogger(AssertNumRowsNode.class); private long desiredNumOfRows; private String subqueryString; @@ -46,6 +52,18 @@ public AssertNumRowsNode(PlanNodeId id, PlanNode input, AssertNumRowsElement ass this.nullableTupleIds.addAll(input.getNullableTupleIds()); } + @Override + public void init(Analyzer analyzer) throws UserException { + super.init(analyzer); + super.computeStats(analyzer); + if (analyzer.safeIsEnableJoinReorderBasedCost()) { + cardinality = 1; + } + if (LOG.isDebugEnabled()) { + LOG.debug("stats AssertNumRows: cardinality={}", cardinality); + } + } + @Override public String getNodeExplainString(String prefix, TExplainLevel detailLevel) { if (detailLevel == TExplainLevel.BRIEF) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/CrossJoinNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/CrossJoinNode.java index 9246549bc3e294..31c668f2173f98 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/CrossJoinNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/CrossJoinNode.java @@ -19,6 +19,8 @@ import org.apache.doris.analysis.Analyzer; import org.apache.doris.analysis.TableRef; +import org.apache.doris.common.CheckedMath; +import org.apache.doris.common.UserException; import org.apache.doris.thrift.TExplainLevel; import org.apache.doris.thrift.TPlanNode; import org.apache.doris.thrift.TPlanNodeType; @@ -59,18 +61,42 @@ public TableRef getInnerRef() { return innerRef_; } + @Override + public void init(Analyzer analyzer) throws UserException { + super.init(analyzer); + assignedConjuncts = analyzer.getAssignedConjuncts(); + computeStats(analyzer); + } + @Override public void computeStats(Analyzer analyzer) { - super.computeStats(analyzer); - if (getChild(0).cardinality == -1 || getChild(1).cardinality == -1) { - cardinality = -1; - } else { - cardinality = getChild(0).cardinality * getChild(1).cardinality; - if (computeSelectivity() != -1) { - cardinality = Math.round(((double) cardinality) * computeSelectivity()); - } - } - LOG.debug("stats CrossJoin: cardinality={}", Long.toString(cardinality)); + super.computeStats(analyzer); + if (!analyzer.safeIsEnableJoinReorderBasedCost()) { + return; + } + if (getChild(0).cardinality == -1 || getChild(1).cardinality == -1) { + cardinality = -1; + } else { + cardinality = CheckedMath.checkedMultiply(getChild(0).cardinality, getChild(1).cardinality); + applyConjunctsSelectivity(); + capCardinalityAtLimit(); + } + if (LOG.isDebugEnabled()) { + LOG.debug("stats CrossJoin: cardinality={}", Long.toString(cardinality)); + } + } + + @Override + protected void computeOldCardinality() { + if (getChild(0).cardinality == -1 || getChild(1).cardinality == -1) { + cardinality = -1; + } else { + cardinality = getChild(0).cardinality * getChild(1).cardinality; + if (computeOldSelectivity() != -1) { + cardinality = Math.round(((double) cardinality) * computeOldSelectivity()); + } + } + LOG.debug("stats CrossJoin: cardinality={}", Long.toString(cardinality)); } @Override @@ -94,6 +120,8 @@ public String getNodeExplainString(String detailPrefix, TExplainLevel detailLeve } else { output.append(detailPrefix + "predicates is NULL."); } + output.append(detailPrefix).append(String.format( + "cardinality=%s", cardinality)).append("\n"); return output.toString(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/EmptySetNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/EmptySetNode.java index c9e5eadb4f3e76..eef43990886277 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/EmptySetNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/EmptySetNode.java @@ -26,6 +26,9 @@ import com.google.common.base.Preconditions; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + /** * Node that returns an empty result set. Used for planning query blocks with a constant * predicate evaluating to false or a limit 0. The result set will have zero rows, but @@ -33,6 +36,8 @@ * construct a valid row empty batch. */ public class EmptySetNode extends PlanNode { + private final static Logger LOG = LogManager.getLogger(EmptySetNode.class); + public EmptySetNode(PlanNodeId id, ArrayList tupleIds) { super(id, tupleIds, "EMPTYSET"); Preconditions.checkArgument(tupleIds.size() > 0); @@ -43,6 +48,9 @@ public void computeStats(Analyzer analyzer) { avgRowSize = 0; cardinality = 0; numNodes = 1; + if (LOG.isDebugEnabled()) { + LOG.debug("stats EmptySet:" + id + ", cardinality: " + cardinality); + } } @Override @@ -53,7 +61,7 @@ public void init(Analyzer analyzer) { // to be set as materialized (even though it isn't) to avoid failing precondition // checks generating the thrift for slot refs that may reference this tuple. for (TupleId id: tupleIds) analyzer.getTupleDesc(id).setIsMaterialized(true); - computeMemLayout(analyzer); + computeTupleStatAndMemLayout(analyzer); computeStats(analyzer); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/EsScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/EsScanNode.java index 6853459ab2200c..e8c2b71e3d157b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/EsScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/EsScanNode.java @@ -82,6 +82,7 @@ public void init(Analyzer analyzer) throws UserException { super.init(analyzer); assignBackends(); + computeStats(analyzer); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/ExchangeNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/ExchangeNode.java index a19aa43c91af72..91b23b782d671b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/ExchangeNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/ExchangeNode.java @@ -73,11 +73,6 @@ public ExchangeNode(PlanNodeId id, PlanNode inputNode, boolean copyConjuncts) { if (!copyConjuncts) { this.conjuncts = Lists.newArrayList(); } - if (hasLimit()) { - cardinality = Math.min(limit, inputNode.cardinality); - } else { - cardinality = inputNode.cardinality; - } // Only apply the limit at the receiver if there are multiple senders. if (inputNode.getFragment().isPartitioned()) limit = inputNode.limit; computeTupleIds(); @@ -95,6 +90,20 @@ public final void computeTupleIds() { public void init(Analyzer analyzer) throws UserException { super.init(analyzer); Preconditions.checkState(conjuncts.isEmpty()); + if (!analyzer.safeIsEnableJoinReorderBasedCost()) { + return; + } + computeStats(analyzer); + } + + @Override + protected void computeStats(Analyzer analyzer) { + Preconditions.checkState(children.size() == 1); + cardinality = children.get(0).cardinality; + capCardinalityAtLimit(); + if (LOG.isDebugEnabled()) { + LOG.debug("stats Exchange:" + id + ", cardinality: " + cardinality); + } } /** diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/HashJoinNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/HashJoinNode.java index 6b06de2a9323d1..83d2ff50e3f7d2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/HashJoinNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/HashJoinNode.java @@ -28,6 +28,10 @@ import org.apache.doris.analysis.TableRef; import org.apache.doris.analysis.TupleId; import org.apache.doris.catalog.ColumnStats; +import org.apache.doris.catalog.OlapTable; +import org.apache.doris.catalog.Table; +import org.apache.doris.common.CheckedMath; +import org.apache.doris.common.Pair; import org.apache.doris.common.UserException; import org.apache.doris.thrift.TEqJoinCondition; import org.apache.doris.thrift.TExplainLevel; @@ -35,15 +39,17 @@ import org.apache.doris.thrift.TPlanNode; import org.apache.doris.thrift.TPlanNodeType; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - import com.google.common.base.MoreObjects; import com.google.common.base.Preconditions; import com.google.common.collect.Lists; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + import java.util.ArrayList; +import java.util.LinkedHashMap; import java.util.List; +import java.util.Map; import java.util.stream.Collectors; /** @@ -59,7 +65,7 @@ public class HashJoinNode extends PlanNode { // predicates of the form 'a=b' or 'a<=>b' private List eqJoinConjuncts = Lists.newArrayList(); // join conjuncts from the JOIN clause that aren't equi-join predicates - private List otherJoinConjuncts; + private List otherJoinConjuncts; private DistributionMode distrMode; private boolean isColocate = false; //the flag for colocate join private String colocateReason = ""; // if can not do colocate join, set reason here @@ -134,23 +140,19 @@ public void setColocate(boolean colocate, String reason) { @Override public void init(Analyzer analyzer) throws UserException { - assignConjuncts(analyzer); - - // Set smap to the combined children's smaps and apply that to all conjuncts_. - createDefaultSmap(analyzer); - + super.init(analyzer); + assignedConjuncts = analyzer.getAssignedConjuncts(); // outSmap replace in outer join may cause NULL be replace by literal // so need replace the outsmap in nullableTupleID replaceOutputSmapForOuterJoin(); - computeStats(analyzer); - //assignedConjuncts = analyzr.getAssignedConjuncts(); ExprSubstitutionMap combinedChildSmap = getCombinedChildWithoutTupleIsNullSmap(); List newEqJoinConjuncts = Expr.substituteList(eqJoinConjuncts, combinedChildSmap, analyzer, false); eqJoinConjuncts = newEqJoinConjuncts.stream() .map(entity -> (BinaryPredicate) entity).collect(Collectors.toList()); + assignedConjuncts = analyzer.getAssignedConjuncts(); otherJoinConjuncts = Expr.substituteList(otherJoinConjuncts, combinedChildSmap, analyzer, false); } @@ -179,10 +181,191 @@ private void replaceOutputSmapForOuterJoin() { } } + /** + * Holds the source scan slots of a = join predicate. + * The underlying table and column on both sides have stats. + */ + public static final class EqJoinConjunctScanSlots { + private final Expr eqJoinConjunct; + private final SlotDescriptor lhs; + private final SlotDescriptor rhs; + + private EqJoinConjunctScanSlots(Expr eqJoinConjunct, SlotDescriptor lhs, + SlotDescriptor rhs) { + this.eqJoinConjunct = eqJoinConjunct; + this.lhs = lhs; + this.rhs = rhs; + } + + // Convenience functions. They return double to avoid excessive casts in callers. + public double lhsNdv() { + // return the estimated number of rows in this partition (-1 if unknown) + return Math.min(lhs.getStats().getNumDistinctValues(), lhsNumRows()); + } + + public double rhsNdv() { + return Math.min(rhs.getStats().getNumDistinctValues(), rhsNumRows()); + } + + public double lhsNumRows() { + Table table = lhs.getParent().getTable(); + Preconditions.checkState(table instanceof OlapTable); + return ((OlapTable) (table)).getRowCount(); + } + + public double rhsNumRows() { + Table table = rhs.getParent().getTable(); + Preconditions.checkState(table instanceof OlapTable); + return ((OlapTable) (table)).getRowCount(); + } + + public TupleId lhsTid() { + return lhs.getParent().getId(); + } + + public TupleId rhsTid() { + return rhs.getParent().getId(); + } + + /** + * Returns a new EqJoinConjunctScanSlots for the given equi-join conjunct or null if + * the given conjunct is not of the form = or if the underlying + * table/column of at least one side is missing stats. + */ + public static EqJoinConjunctScanSlots create(Expr eqJoinConjunct) { + if (!Expr.IS_EQ_BINARY_PREDICATE.apply(eqJoinConjunct)) return null; + SlotDescriptor lhsScanSlot = eqJoinConjunct.getChild(0).findSrcScanSlot(); + if (lhsScanSlot == null || !hasNumRowsAndNdvStats(lhsScanSlot)) return null; + SlotDescriptor rhsScanSlot = eqJoinConjunct.getChild(1).findSrcScanSlot(); + if (rhsScanSlot == null || !hasNumRowsAndNdvStats(rhsScanSlot)) return null; + return new EqJoinConjunctScanSlots(eqJoinConjunct, lhsScanSlot, rhsScanSlot); + } + + private static boolean hasNumRowsAndNdvStats(SlotDescriptor slotDesc) { + if (slotDesc.getColumn() == null) return false; + if (!slotDesc.getStats().hasNumDistinctValues()) return false; + return true; + } + + /** + * Groups the given EqJoinConjunctScanSlots by the lhs/rhs tuple combination + * and returns the result as a map. + */ + public static Map, List> + groupByJoinedTupleIds(List eqJoinConjunctSlots) { + Map, List> scanSlotsByJoinedTids = + new LinkedHashMap<>(); + for (EqJoinConjunctScanSlots slots : eqJoinConjunctSlots) { + Pair tids = Pair.create(slots.lhsTid(), slots.rhsTid()); + List scanSlots = scanSlotsByJoinedTids.get(tids); + if (scanSlots == null) { + scanSlots = new ArrayList<>(); + scanSlotsByJoinedTids.put(tids, scanSlots); + } + scanSlots.add(slots); + } + return scanSlotsByJoinedTids; + } + + @Override + public String toString() { + return eqJoinConjunct.toSql(); + } + } + + private long getJoinCardinality() { + Preconditions.checkState(joinOp.isInnerJoin() || joinOp.isOuterJoin()); + + long lhsCard = getChild(0).cardinality; + long rhsCard = getChild(1).cardinality; + if (lhsCard == -1 || rhsCard == -1) { + return lhsCard; + } + + // Collect join conjuncts that are eligible to participate in cardinality estimation. + List eqJoinConjunctSlots = new ArrayList<>(); + for (Expr eqJoinConjunct : eqJoinConjuncts) { + EqJoinConjunctScanSlots slots = EqJoinConjunctScanSlots.create(eqJoinConjunct); + if (slots != null) eqJoinConjunctSlots.add(slots); + } + + if (eqJoinConjunctSlots.isEmpty()) { + // There are no eligible equi-join conjuncts. + return lhsCard; + } + + return getGenericJoinCardinality(eqJoinConjunctSlots, lhsCard, rhsCard); + } + + /** + * Returns the estimated join cardinality of a generic N:M inner or outer join based + * on the given list of equi-join conjunct slots and the join input cardinalities. + * The returned result is >= 0. + * The list of join conjuncts must be non-empty and the cardinalities must be >= 0. + *

+ * Generic estimation: + * cardinality = |child(0)| * |child(1)| / max(NDV(L.c), NDV(R.d)) + * - case A: NDV(L.c) <= NDV(R.d) + * every row from child(0) joins with |child(1)| / NDV(R.d) rows + * - case B: NDV(L.c) > NDV(R.d) + * every row from child(1) joins with |child(0)| / NDV(L.c) rows + * - we adjust the NDVs from both sides to account for predicates that may + * might have reduce the cardinality and NDVs + */ + private long getGenericJoinCardinality(List eqJoinConjunctSlots, long lhsCard, long rhsCard) { + Preconditions.checkState(joinOp.isInnerJoin() || joinOp.isOuterJoin()); + Preconditions.checkState(!eqJoinConjunctSlots.isEmpty()); + Preconditions.checkState(lhsCard >= 0 && rhsCard >= 0); + + long result = -1; + for (EqJoinConjunctScanSlots slots : eqJoinConjunctSlots) { + // Adjust the NDVs on both sides to account for predicates. Intuitively, the NDVs + // should only decrease. We ignore adjustments that would lead to an increase. + double lhsAdjNdv = slots.lhsNdv(); + if (slots.lhsNumRows() > lhsCard) { + lhsAdjNdv *= lhsCard / slots.lhsNumRows(); + } + double rhsAdjNdv = slots.rhsNdv(); + if (slots.rhsNumRows() > rhsCard) { + rhsAdjNdv *= rhsCard / slots.rhsNumRows(); + } + // A lower limit of 1 on the max Adjusted Ndv ensures we don't estimate + // cardinality more than the max possible. + long joinCard = CheckedMath.checkedMultiply( + Math.round((lhsCard / Math.max(1, Math.max(lhsAdjNdv, rhsAdjNdv)))), rhsCard); + if (result == -1) { + result = joinCard; + } else { + result = Math.min(result, joinCard); + } + } + Preconditions.checkState(result >= 0); + return result; + } + + @Override public void computeStats(Analyzer analyzer) { super.computeStats(analyzer); + if (!analyzer.safeIsEnableJoinReorderBasedCost()) { + return; + } + if (joinOp.isSemiAntiJoin()) { + cardinality = getSemiJoinCardinality(); + } else if (joinOp.isInnerJoin() || joinOp.isOuterJoin()) { + cardinality = getJoinCardinality(); + } else { + Preconditions.checkState(false, "joinOp is not supported"); + } + capCardinalityAtLimit(); + if (LOG.isDebugEnabled()) { + LOG.debug("stats HashJoin:" + id + ", cardinality: " + cardinality); + } + } + + @Override + protected void computeOldCardinality() { // For a join between child(0) and child(1), we look for join conditions "L.c = R.d" // (with L being from child(0) and R from child(1)) and use as the cardinality // estimate the maximum of @@ -228,11 +411,11 @@ public void computeStats(Analyzer analyzer) { // TODO rownum //Table rhsTbl = slotDesc.getParent().getTableFamilyGroup().getBaseTable(); // if (rhsTbl != null && rhsTbl.getNumRows() != -1) { - // we can't have more distinct values than rows in the table, even though - // the metastore stats may think so - // LOG.info( - // "#distinct=" + numDistinct + " #rows=" + Long.toString(rhsTbl.getNumRows())); - // numDistinct = Math.min(numDistinct, rhsTbl.getNumRows()); + // we can't have more distinct values than rows in the table, even though + // the metastore stats may think so + // LOG.info( + // "#distinct=" + numDistinct + " #rows=" + Long.toString(rhsTbl.getNumRows())); + // numDistinct = Math.min(numDistinct, rhsTbl.getNumRows()); // } maxNumDistinct = Math.max(maxNumDistinct, numDistinct); LOG.debug("min slotref: {}, #distinct: {}", rhsSlotRef.toSql(), numDistinct); @@ -245,12 +428,112 @@ public void computeStats(Analyzer analyzer) { cardinality = getChild(0).cardinality; } else { cardinality = Math.round((double) getChild(0).cardinality * (double) getChild( - 1).cardinality / (double) maxNumDistinct); + 1).cardinality / (double) maxNumDistinct); LOG.debug("lhs card: {}, rhs card: {}", getChild(0).cardinality, getChild(1).cardinality); } LOG.debug("stats HashJoin: cardinality {}", cardinality); } + /** + * Unwraps the SlotRef in expr and returns the NDVs of it. + * Returns -1 if the NDVs are unknown or if expr is not a SlotRef. + */ + private long getNdv(Expr expr) { + SlotRef slotRef = expr.unwrapSlotRef(false); + if (slotRef == null) { + return -1; + } + SlotDescriptor slotDesc = slotRef.getDesc(); + if (slotDesc == null) { + return -1; + } + ColumnStats stats = slotDesc.getStats(); + if (!stats.hasNumDistinctValues()) { + return -1; + } + return stats.getNumDistinctValues(); + } + + /** + * Returns the estimated cardinality of a semi join node. + * For a left semi join between child(0) and child(1), we look for equality join + * conditions "L.c = R.d" (with L being from child(0) and R from child(1)) and use as + * the cardinality estimate the minimum of + * |child(0)| * Min(NDV(L.c), NDV(R.d)) / NDV(L.c) + * over all suitable join conditions. The reasoning is that: + * - each row in child(0) is returned at most once + * - the probability of a row in child(0) having a match in R is + * Min(NDV(L.c), NDV(R.d)) / NDV(L.c) + *

+ * For a left anti join we estimate the cardinality as the minimum of: + * |L| * Max(NDV(L.c) - NDV(R.d), NDV(L.c)) / NDV(L.c) + * over all suitable join conditions. The reasoning is that: + * - each row in child(0) is returned at most once + * - if NDV(L.c) > NDV(R.d) then the probability of row in L having a match + * in child(1) is (NDV(L.c) - NDV(R.d)) / NDV(L.c) + * - otherwise, we conservatively use |L| to avoid underestimation + *

+ * We analogously estimate the cardinality for right semi/anti joins, and treat the + * null-aware anti join like a regular anti join + */ + private long getSemiJoinCardinality() { + Preconditions.checkState(joinOp.isSemiJoin()); + + // Return -1 if the cardinality of the returned side is unknown. + long cardinality; + if (joinOp == JoinOperator.RIGHT_SEMI_JOIN + || joinOp == JoinOperator.RIGHT_ANTI_JOIN) { + if (getChild(1).cardinality == -1) { + return -1; + } + cardinality = getChild(1).cardinality; + } else { + if (getChild(0).cardinality == -1) { + return -1; + } + cardinality = getChild(0).cardinality; + } + double minSelectivity = 1.0; + for (Expr eqJoinPredicate : eqJoinConjuncts) { + long lhsNdv = getNdv(eqJoinPredicate.getChild(0)); + lhsNdv = Math.min(lhsNdv, getChild(0).cardinality); + long rhsNdv = getNdv(eqJoinPredicate.getChild(1)); + rhsNdv = Math.min(rhsNdv, getChild(1).cardinality); + + // Skip conjuncts with unknown NDV on either side. + if (lhsNdv == -1 || rhsNdv == -1) { + continue; + } + + double selectivity = 1.0; + switch (joinOp) { + case LEFT_SEMI_JOIN: { + selectivity = (double) Math.min(lhsNdv, rhsNdv) / (double) (lhsNdv); + break; + } + case RIGHT_SEMI_JOIN: { + selectivity = (double) Math.min(lhsNdv, rhsNdv) / (double) (rhsNdv); + break; + } + case LEFT_ANTI_JOIN: + case NULL_AWARE_LEFT_ANTI_JOIN: { + selectivity = (double) (lhsNdv > rhsNdv ? (lhsNdv - rhsNdv) : lhsNdv) / (double) lhsNdv; + break; + } + case RIGHT_ANTI_JOIN: { + selectivity = (double) (rhsNdv > lhsNdv ? (rhsNdv - lhsNdv) : rhsNdv) / (double) rhsNdv; + break; + } + default: + Preconditions.checkState(false); + } + minSelectivity = Math.min(minSelectivity, selectivity); + } + + Preconditions.checkState(cardinality != -1); + return Math.round(cardinality * minSelectivity); + } + @Override protected String debugString() { return MoreObjects.toStringHelper(this).add("eqJoinConjuncts", diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/LoadScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/LoadScanNode.java index 2029891639bc77..de896f980629ae 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/LoadScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/LoadScanNode.java @@ -186,7 +186,7 @@ protected void finalizeParams(Map slotDescByName, // LOG.info("brokerScanRange is {}", brokerScanRange); // Need re compute memory layout after set some slot descriptor to nullable - srcTupleDesc.computeMemLayout(); + srcTupleDesc.computeStatAndMemLayout(); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/MergeJoinNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/MergeJoinNode.java deleted file mode 100644 index a7758746a27448..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/MergeJoinNode.java +++ /dev/null @@ -1,170 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.planner; - -import org.apache.doris.analysis.Analyzer; -import org.apache.doris.analysis.Expr; -import org.apache.doris.analysis.SlotId; -import org.apache.doris.common.Pair; -import org.apache.doris.thrift.TEqJoinCondition; -import org.apache.doris.thrift.TExplainLevel; -import org.apache.doris.thrift.TMergeJoinNode; -import org.apache.doris.thrift.TPlanNode; -import org.apache.doris.thrift.TPlanNodeType; - -import com.google.common.base.MoreObjects; -import com.google.common.base.Preconditions; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import java.util.List; - -/** - * Merge join between left child and right child. - * The right child must be a leaf node, ie, can only materialize - * a single input tuple. - */ -public class MergeJoinNode extends PlanNode { - private final static Logger LOG = LogManager.getLogger(MergeJoinNode.class); - // conjuncts of the form " = ", recorded as Pair(, ) - private final List> cmpConjuncts; - // join conjuncts from the JOIN clause that aren't equi-join predicates - private final List otherJoinConjuncts; - private DistributionMode distrMode; - - public MergeJoinNode(PlanNodeId id, PlanNode outer, PlanNode inner, - List> cmpConjuncts, List otherJoinConjuncts) { - super(id, "MERGE JOIN"); - Preconditions.checkArgument(cmpConjuncts != null); - Preconditions.checkArgument(otherJoinConjuncts != null); - tupleIds.addAll(outer.getTupleIds()); - tupleIds.addAll(inner.getTupleIds()); - this.distrMode = DistributionMode.PARTITIONED; - this.cmpConjuncts = cmpConjuncts; - this.otherJoinConjuncts = otherJoinConjuncts; - children.add(outer); - children.add(inner); - - // Inherits all the nullable tuple from the children - // Mark tuples that form the "nullable" side of the outer join as nullable. - nullableTupleIds.addAll(inner.getNullableTupleIds()); - nullableTupleIds.addAll(outer.getNullableTupleIds()); - nullableTupleIds.addAll(outer.getTupleIds()); - nullableTupleIds.addAll(inner.getTupleIds()); - } - - public List> getCmpConjuncts() { - return cmpConjuncts; - } - - public DistributionMode getDistributionMode() { - return distrMode; - } - - @Override - public void computeStats(Analyzer analyzer) { - super.computeStats(analyzer); - } - - @Override - protected String debugString() { - return MoreObjects.toStringHelper(this).add("cmpConjuncts", cmpConjunctsDebugString()).addValue( - super.debugString()).toString(); - } - - private String cmpConjunctsDebugString() { - MoreObjects.ToStringHelper helper = MoreObjects.toStringHelper(this); - for (Pair entry : cmpConjuncts) { - helper.add("lhs", entry.first).add("rhs", entry.second); - } - return helper.toString(); - } - - @Override - public void getMaterializedIds(Analyzer analyzer, List ids) { - super.getMaterializedIds(analyzer, ids); - // we also need to materialize everything referenced by cmpConjuncts - // and otherJoinConjuncts - for (Pair p : cmpConjuncts) { - p.first.getIds(null, ids); - p.second.getIds(null, ids); - } - for (Expr e : otherJoinConjuncts) { - e.getIds(null, ids); - } - } - - @Override - protected void toThrift(TPlanNode msg) { - msg.node_type = TPlanNodeType.MERGE_JOIN_NODE; - msg.merge_join_node = new TMergeJoinNode(); - for (Pair entry : cmpConjuncts) { - TEqJoinCondition eqJoinCondition = - new TEqJoinCondition(entry.first.treeToThrift(), entry.second.treeToThrift()); - msg.merge_join_node.addToCmpConjuncts(eqJoinCondition); - } - for (Expr e : otherJoinConjuncts) { - msg.hash_join_node.addToOtherJoinConjuncts(e.treeToThrift()); - } - } - - @Override - public String getNodeExplainString(String detailPrefix, TExplainLevel detailLevel) { - String distrModeStr = - (distrMode != DistributionMode.NONE) ? (" (" + distrMode.toString() + ")") : ""; - StringBuilder output = new StringBuilder().append( - detailPrefix + "join op: MERGE JOIN" + distrModeStr + "\n").append( - detailPrefix + "hash predicates:\n"); - for (Pair entry : cmpConjuncts) { - output.append(detailPrefix + " " + - entry.first.toSql() + " = " + entry.second.toSql() + "\n"); - } - if (!otherJoinConjuncts.isEmpty()) { - output.append(detailPrefix + "other join predicates: ").append( - getExplainString(otherJoinConjuncts) + "\n"); - } - if (!conjuncts.isEmpty()) { - output.append(detailPrefix + "other predicates: ").append( - getExplainString(conjuncts) + "\n"); - } - return output.toString(); - } - - @Override - public int getNumInstances() { - return Math.max(children.get(0).getNumInstances(), children.get(1).getNumInstances()); - } - - enum DistributionMode { - NONE("NONE"), - BROADCAST("BROADCAST"), - PARTITIONED("PARTITIONED"); - - private final String description; - - private DistributionMode(String descr) { - this.description = descr; - } - - @Override - public String toString() { - return description; - } - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/MergeNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/MergeNode.java index 629cfa9cd88e9b..bb307021ef3228 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/MergeNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/MergeNode.java @@ -24,7 +24,6 @@ import org.apache.doris.analysis.TupleDescriptor; import org.apache.doris.analysis.TupleId; import org.apache.doris.common.UserException; - import org.apache.doris.thrift.TExplainLevel; import org.apache.doris.thrift.TExpr; import org.apache.doris.thrift.TMergeNode; @@ -62,19 +61,9 @@ public class MergeNode extends PlanNode { protected final TupleId tupleId; - private final boolean isIntermediateMerge; - - protected MergeNode(PlanNodeId id, TupleId tupleId) { - super(id, tupleId.asList(), "MERGE"); - // this.rowTupleIds.add(tupleId); - this.tupleId = tupleId; - this.isIntermediateMerge = false; - } - protected MergeNode(PlanNodeId id, MergeNode node) { super(id, node, "MERGE"); this.tupleId = node.tupleId; - this.isIntermediateMerge = true; } public void addConstExprList(List exprs) { @@ -137,6 +126,26 @@ public void init(Analyzer analyzer) throws UserException { @Override public void computeStats(Analyzer analyzer) { super.computeStats(analyzer); + if (!analyzer.safeIsEnableJoinReorderBasedCost()) { + return; + } + cardinality = constExprLists.size(); + for (PlanNode child : children) { + // ignore missing child cardinality info in the hope it won't matter enough + // to change the planning outcome + if (child.cardinality > 0) { + cardinality += child.cardinality; + } + } + applyConjunctsSelectivity(); + capCardinalityAtLimit(); + if (LOG.isDebugEnabled()) { + LOG.debug("stats Merge: cardinality={}", Long.toString(cardinality)); + } + } + + @Override + protected void computeOldCardinality() { cardinality = constExprLists.size(); for (PlanNode child : children) { // ignore missing child cardinality info in the hope it won't matter enough diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/MysqlScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/MysqlScanNode.java index 945542be1a3f0a..fcc22125c99117 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/MysqlScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/MysqlScanNode.java @@ -60,6 +60,12 @@ public MysqlScanNode(PlanNodeId id, TupleDescriptor desc, MysqlTable tbl) { tblName = "`" + tbl.getMysqlTableName() + "`"; } + @Override + public void init(Analyzer analyzer) throws UserException { + super.init(analyzer); + computeStats(analyzer); + } + @Override protected String debugString() { MoreObjects.ToStringHelper helper = MoreObjects.toStringHelper(this); @@ -71,7 +77,6 @@ public void finalize(Analyzer analyzer) throws UserException { // Convert predicates to MySQL columns and filters. createMySQLColumns(analyzer); createMySQLFilters(analyzer); - computeStats(analyzer); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/OdbcScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/OdbcScanNode.java index 918a56781857c5..90f989d35c97e6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/OdbcScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/OdbcScanNode.java @@ -79,6 +79,12 @@ public OdbcScanNode(PlanNodeId id, TupleDescriptor desc, OdbcTable tbl) { tblName = OdbcTable.databaseProperName(odbcType, tbl.getOdbcTableName()); } + @Override + public void init(Analyzer analyzer) throws UserException { + super.init(analyzer); + computeStats(analyzer); + } + @Override protected String debugString() { MoreObjects.ToStringHelper helper = MoreObjects.toStringHelper(this); @@ -90,7 +96,6 @@ public void finalize(Analyzer analyzer) throws UserException { // Convert predicates to Odbc columns and filters. createOdbcColumns(analyzer); createOdbcFilters(analyzer); - computeStats(analyzer); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java index 99e158eb1176a5..a69c08d5143210 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java @@ -132,13 +132,12 @@ public class OlapScanNode extends ScanNode { // List of tablets will be scanned by current olap_scan_node private ArrayList scanTabletIds = Lists.newArrayList(); - private boolean isFinalized = false; private HashSet scanBackendIds = new HashSet<>(); private Map tabletId2BucketSeq = Maps.newHashMap(); // a bucket seq may map to many tablets, and each tablet has a TScanRangeLocations. - public ArrayListMultimap bucketSeq2locations= ArrayListMultimap.create(); + public ArrayListMultimap bucketSeq2locations = ArrayListMultimap.create(); // Constructs node to scan given data files of table 'tbl'. public OlapScanNode(PlanNodeId id, TupleDescriptor desc, String planNodeName) { @@ -228,8 +227,8 @@ public void updateScanRangeInfoByNewMVSelector(long selectedIndexId, boolean isP SessionVariable sessionVariable = ConnectContext.get().getSessionVariable(); if (sessionVariable.getTestMaterializedView()) { throw new AnalysisException("The old scan range info is different from the new one when " - + "test_materialized_view is true. " - + scanRangeInfo); + + "test_materialized_view is true. " + + scanRangeInfo); } situation = "The key type of table is aggregated."; update = false; @@ -297,38 +296,93 @@ public void init(Analyzer analyzer) throws UserException { filterDeletedRows(analyzer); computePartitionInfo(); + computeTupleState(analyzer); + + /** + * Compute InAccurate stats before mv selector and tablet pruning. + * - Accurate statistical information relies on the selector of materialized views and bucket reduction. + * - However, Those both processes occur after the reorder algorithm is completed. + * - When Join reorder is turned on, the computeStats() must be completed before the reorder algorithm. + * - So only an inaccurate statistical information can be calculated here. + */ + if (analyzer.safeIsEnableJoinReorderBasedCost()) { + computeInaccurateStats(analyzer); + } } @Override public void finalize(Analyzer analyzer) throws UserException { - if (isFinalized) { - return; + LOG.debug("OlapScanNode get scan range locations. Tuple: {}", desc); + /** + * If JoinReorder is turned on, it will be calculated init(), and this value is not accurate. + * In the following logic, cardinality will be accurately calculated again. + * So here we need to reset the value of cardinality. + */ + if (analyzer.safeIsEnableJoinReorderBasedCost()) { + cardinality = 0; } - - LOG.debug("OlapScanNode finalize. Tuple: {}", desc); try { getScanRangeLocations(); } catch (AnalysisException e) { throw new UserException(e.getMessage()); } + if (!analyzer.safeIsEnableJoinReorderBasedCost()) { + computeOldRowSizeAndCardinality(); + } + computeNumNodes(); + } - computeStats(analyzer); - isFinalized = true; + public void computeTupleState(Analyzer analyzer) { + for (TupleId id : tupleIds) { + analyzer.getDescTbl().getTupleDesc(id).computeStat(); + } } - @Override - public void computeStats(Analyzer analyzer) { + public void computeOldRowSizeAndCardinality() { if (cardinality > 0) { avgRowSize = totalBytes / (float) cardinality; - if (hasLimit()) { - cardinality = Math.min(cardinality, limit); - } + capCardinalityAtLimit(); + } + // when node scan has no data, cardinality should be 0 instead of a invalid value after computeStats() + cardinality = cardinality == -1 ? 0 : cardinality; + } + + @Override + protected void computeNumNodes() { + if (cardinality > 0) { numNodes = scanBackendIds.size(); } // even current node scan has no data,at least on backend will be assigned when the fragment actually execute numNodes = numNodes <= 0 ? 1 : numNodes; - // when node scan has no data, cardinality should be 0 instead of a invalid value after computeStats() - cardinality = cardinality == -1 ? 0 : cardinality; + } + + /** + * Calculate inaccurate stats such as: cardinality. + * cardinality: the value of cardinality is the sum of rowcount which belongs to selectedPartitionIds + * The cardinality here is actually inaccurate, it will be greater than the actual value. + * There are two reasons + * 1. During the actual execution, not all tablets belonging to the selected partition will be scanned. + * Some tablets may have been pruned before execution. + * 2. The base index may eventually be replaced by mv index. + *

+ * There are three steps to calculate cardinality + * 1. Calculate how many rows were scanned + * 2. Apply conjunct + * 3. Apply limit + * + * @param analyzer + */ + private void computeInaccurateStats(Analyzer analyzer) { + super.computeStats(analyzer); + // step1: Calculate how many rows were scanned + cardinality = 0; + for (long selectedPartitionId : selectedPartitionIds) { + final Partition partition = olapTable.getPartition(selectedPartitionId); + final MaterializedIndex baseIndex = partition.getBaseIndex(); + cardinality += baseIndex.getRowCount(); + } + applyConjunctsSelectivity(); + capCardinalityAtLimit(); } private Collection partitionPrune(PartitionInfo partitionInfo, PartitionNames partitionNames) throws AnalysisException { @@ -360,13 +414,13 @@ private Collection distributionPrune( MaterializedIndex table, DistributionInfo distributionInfo) throws AnalysisException { DistributionPruner distributionPruner = null; - switch(distributionInfo.getType()) { + switch (distributionInfo.getType()) { case HASH: { HashDistributionInfo info = (HashDistributionInfo) distributionInfo; distributionPruner = new HashDistributionPruner(table.getTabletIdsInOrder(), - info.getDistributionColumns(), - columnFilters, - info.getBucketNum()); + info.getDistributionColumns(), + columnFilters, + info.getBucketNum()); return distributionPruner.prune(); } case RANDOM: { @@ -409,7 +463,7 @@ private void addScanRangeLocations(Partition partition, visibleVersion, visibleVersionHash, localBeId, schemaHash); if (allQueryableReplicas.isEmpty()) { LOG.error("no queryable replica found in tablet {}. visible version {}-{}", - tabletId, visibleVersion, visibleVersionHash); + tabletId, visibleVersion, visibleVersionHash); if (LOG.isDebugEnabled()) { for (Replica replica : tablet.getReplicas()) { LOG.debug("tablet {}, replica: {}", tabletId, replica.toString()); @@ -462,6 +516,12 @@ private void addScanRangeLocations(Partition partition, result.add(scanRangeLocations); } + // FIXME(dhc): we use cardinality here to simulate ndv + if (tablets.size() == 0) { + desc.setCardinality(0); + } else { + desc.setCardinality(cardinality); + } } private void computePartitionInfo() throws AnalysisException { @@ -512,6 +572,7 @@ public void selectBestRollupByRollupSelector(Analyzer analyzer) throws UserExcep private void getScanRangeLocations() throws UserException { if (selectedPartitionIds.size() == 0) { + desc.setCardinality(0); return; } Preconditions.checkState(selectedIndexId != -1); @@ -597,9 +658,9 @@ public String getNodeExplainString(String prefix, TExplainLevel detailLevel) { } output.append(prefix).append(String.format( - "partitions=%s/%s", - selectedPartitionNum, - olapTable.getPartitions().size())); + "partitions=%s/%s", + selectedPartitionNum, + olapTable.getPartitions().size())); String indexName = olapTable.getIndexNameById(selectedIndexId); output.append("\n").append(prefix).append(String.format("rollup: %s", indexName)); @@ -607,7 +668,7 @@ public String getNodeExplainString(String prefix, TExplainLevel detailLevel) { output.append("\n"); output.append(prefix).append(String.format( - "tabletRatio=%s/%s", selectedTabletsNum, totalTabletsNum)); + "tabletRatio=%s/%s", selectedTabletsNum, totalTabletsNum)); output.append("\n"); // We print up to 10 tablet, and we print "..." if the number is more than 10 @@ -673,7 +734,6 @@ public static OlapScanNode createOlapScanNodeByLocation( olapScanNode.selectedTabletsNum = 1; olapScanNode.totalTabletsNum = 1; olapScanNode.setIsPreAggregation(false, "Export job"); - olapScanNode.isFinalized = true; olapScanNode.result.addAll(locationsList); return olapScanNode; @@ -715,7 +775,7 @@ public void collectColumns(Analyzer analyzer, Set equivalenceColumns, Se } } - public TupleId getTupleId(){ + public TupleId getTupleId() { Preconditions.checkNotNull(desc); return desc.getId(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/PlanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/PlanNode.java index 1233f45edf1021..dc53c44a81cdf1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/PlanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/PlanNode.java @@ -20,6 +20,7 @@ import com.google.common.base.Joiner; import org.apache.doris.analysis.Analyzer; import org.apache.doris.analysis.Expr; +import org.apache.doris.analysis.ExprId; import org.apache.doris.analysis.ExprSubstitutionMap; import org.apache.doris.analysis.SlotId; import org.apache.doris.analysis.TupleDescriptor; @@ -35,13 +36,13 @@ import com.google.common.base.Predicates; import com.google.common.collect.Lists; import com.google.common.collect.Sets; -import com.google.common.math.LongMath; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; import java.util.List; import java.util.Set; @@ -65,9 +66,9 @@ abstract public class PlanNode extends TreeNode { protected String planNodeName; - protected PlanNodeId id; // unique w/in plan tree; assigned by planner + protected PlanNodeId id; // unique w/in plan tree; assigned by planner protected PlanFragmentId fragmentId; // assigned by planner after fragmentation step - protected long limit; // max. # of rows to be returned; 0: no limit + protected long limit; // max. # of rows to be returned; 0: no limit // ids materialized by the tree rooted at this node protected ArrayList tupleIds; @@ -115,13 +116,11 @@ abstract public class PlanNode extends TreeNode { protected boolean compactData; protected int numInstances; - public String getPlanNodeName() { - return planNodeName; - } - // Runtime filters assigned to this node. protected List runtimeFilters = new ArrayList<>(); + private boolean cardinalityIsDone = false; + protected PlanNode(PlanNodeId id, ArrayList tupleIds, String planNodeName) { this.id = id; this.limit = -1; @@ -159,6 +158,10 @@ protected PlanNode(PlanNodeId id, PlanNode node, String planNodeName) { this.numInstances = 1; } + public String getPlanNodeName() { + return planNodeName; + } + /** * Sets tblRefIds_, tupleIds_, and nullableTupleIds_. * The default implementation is a no-op. @@ -197,8 +200,14 @@ public void setFragmentId(PlanFragmentId id) { fragmentId = id; } - public void setFragment(PlanFragment fragment) { fragment_ = fragment; } - public PlanFragment getFragment() { return fragment_; } + public void setFragment(PlanFragment fragment) { + fragment_ = fragment; + } + + public PlanFragment getFragment() { + return fragment_; + } + public long getLimit() { return limit; } @@ -284,11 +293,12 @@ public void addConjuncts(List conjuncts) { this.conjuncts.addAll(conjuncts); } - public void addPreFilterConjuncts(List conjuncts) { - if (conjuncts == null) { - return; - } - this.preFilterConjuncts.addAll(conjuncts); + public void setAssignedConjuncts(Set conjuncts) { + assignedConjuncts = conjuncts; + } + + public Set getAssignedConjuncts() { + return assignedConjuncts; } public void transferConjuncts(PlanNode recipient) { @@ -296,29 +306,25 @@ public void transferConjuncts(PlanNode recipient) { conjuncts.clear(); } - /** - * Call computeMemLayout() for all materialized tuples. - */ - protected void computeMemLayout(Analyzer analyzer) { - for (TupleId id: tupleIds) { - analyzer.getDescTbl().getTupleDesc(id).computeMemLayout(); + public void addPreFilterConjuncts(List conjuncts) { + if (conjuncts == null) { + return; } + this.preFilterConjuncts.addAll(conjuncts); } - /** - * Computes and returns the sum of two cardinalities. If an overflow occurs, - * the maximum Long value is returned (Long.MAX_VALUE). + * Call computeStatAndMemLayout() for all materialized tuples. */ - public static long addCardinalities(long a, long b) { - try { - return LongMath.checkedAdd(a, b); - } catch (ArithmeticException e) { - LOG.warn("overflow when adding cardinalities: " + a + ", " + b); - return Long.MAX_VALUE; + protected void computeTupleStatAndMemLayout(Analyzer analyzer) { + for (TupleId id : tupleIds) { + analyzer.getDescTbl().getTupleDesc(id).computeStatAndMemLayout(); } } + + + public String getExplainString() { return getExplainString("", "", TExplainLevel.VERBOSE); } @@ -378,8 +384,8 @@ protected final String getExplainString(String rootPrefix, String prefix, TExpla String childDetailPrefix = prefix + "| "; for (int i = 1; i < children.size(); ++i) { expBuilder.append( - children.get(i).getExplainString(childHeadlinePrefix, childDetailPrefix, - detailLevel)); + children.get(i).getExplainString(childHeadlinePrefix, childDetailPrefix, + detailLevel)); expBuilder.append(childDetailPrefix + "\n"); } expBuilder.append(children.get(0).getExplainString(prefix, prefix, detailLevel)); @@ -428,7 +434,7 @@ private void treeToThriftHelper(TPlan container) { return; } else { msg.num_children = children.size(); - for (PlanNode child: children) { + for (PlanNode child : children) { child.treeToThriftHelper(container); } } @@ -443,11 +449,20 @@ public void finalize(Analyzer analyzer) throws UserException { for (PlanNode child : children) { child.finalize(analyzer); } - computeStats(analyzer); + computeNumNodes(); + if (!analyzer.safeIsEnableJoinReorderBasedCost()) { + computeOldCardinality(); + } + } + + protected void computeNumNodes() { + if (!children.isEmpty()) { + numNodes = getChild(0).numNodes; + } } /** - * Computes planner statistics: avgRowSize, numNodes, cardinality. + * Computes planner statistics: avgRowSize. * Subclasses need to override this. * Assumes that it has already been called on all children. * This is broken out of finalize() so that it can be called separately @@ -460,31 +475,39 @@ protected void computeStats(Analyzer analyzer) { TupleDescriptor desc = analyzer.getTupleDesc(tid); avgRowSize += desc.getAvgSerializedSize(); } - if (!children.isEmpty()) { - numNodes = getChild(0).numNodes; - } } /** - * Compute the product of the selectivity of all conjuncts. + * This function will calculate the cardinality when the old join reorder algorithm is enabled. + * This value is used to determine the distributed way(broadcast of shuffle) of join in the distributed planning. + * + * If the new join reorder and the old join reorder have the same cardinality calculation method, + * also the calculation is completed in the init(), + * there is no need to override this function. */ - protected double computeSelectivity() { - double prod = 1.0; - for (Expr e : conjuncts) { - if (e.getSelectivity() < 0) { - return -1.0; - } - prod *= e.getSelectivity(); + protected void computeOldCardinality() { + } + + protected void capCardinalityAtLimit() { + if (hasLimit()) { + cardinality = cardinality == -1 ? limit : Math.min(cardinality, limit); } - return prod; } protected ExprSubstitutionMap outputSmap; + + // global state of planning wrt conjunct assignment; used by planner as a shortcut + // to avoid having to pass assigned conjuncts back and forth + // (the planner uses this to save and reset the global state in between join tree + // alternatives) + protected Set assignedConjuncts; + protected ExprSubstitutionMap withoutTupleIsNullOutputSmap; public ExprSubstitutionMap getOutputSmap() { return outputSmap; } + public void setOutputSmap(ExprSubstitutionMap smap) { outputSmap = smap; } @@ -492,25 +515,13 @@ public void setOutputSmap(ExprSubstitutionMap smap) { public void setWithoutTupleIsNullOutputSmap(ExprSubstitutionMap smap) { withoutTupleIsNullOutputSmap = smap; } + public ExprSubstitutionMap getWithoutTupleIsNullOutputSmap() { return withoutTupleIsNullOutputSmap == null ? outputSmap : withoutTupleIsNullOutputSmap; } - /** - * Marks all slots referenced in exprs as materialized. - */ - protected void markSlotsMaterialized(Analyzer analyzer, List exprs) { - List refdIdList = Lists.newArrayList(); - - for (Expr expr: exprs) { - expr.getIds(null, refdIdList); - } - - analyzer.materializeSlots(exprs); - } public void init(Analyzer analyzer) throws UserException { assignConjuncts(analyzer); - computeStats(analyzer); createDefaultSmap(analyzer); } @@ -567,12 +578,13 @@ protected ExprSubstitutionMap getCombinedChildWithoutTupleIsNullSmap() { /** * Sets outputSmap_ to compose(existing smap, combined child smap). Also * substitutes conjuncts_ using the combined child smap. + * * @throws AnalysisException */ protected void createDefaultSmap(Analyzer analyzer) throws UserException { ExprSubstitutionMap combinedChildSmap = getCombinedChildSmap(); outputSmap = - ExprSubstitutionMap.compose(outputSmap, combinedChildSmap, analyzer); + ExprSubstitutionMap.compose(outputSmap, combinedChildSmap, analyzer); conjuncts = Expr.substituteList(conjuncts, outputSmap, analyzer, false); } @@ -646,13 +658,85 @@ public void appendTrace(StringBuilder sb) { } } - @Override - public String toString() { - StringBuilder sb = new StringBuilder(); - sb.append("[").append(getId().asInt()).append(": ").append(getPlanNodeName()).append("]"); - sb.append("\nFragment: ").append(getFragmentId().asInt()).append("]"); - sb.append("\n").append(getNodeExplainString("", TExplainLevel.BRIEF)); - return sb.toString(); + + /** + * Returns the estimated combined selectivity of all conjuncts. Uses heuristics to + * address the following estimation challenges: + * 1. The individual selectivities of conjuncts may be unknown. + * 2. Two selectivities, whether known or unknown, could be correlated. Assuming + * independence can lead to significant underestimation. + *

+ * The first issue is addressed by using a single default selectivity that is + * representative of all conjuncts with unknown selectivities. + * The second issue is addressed by an exponential backoff when multiplying each + * additional selectivity into the final result. + */ + static protected double computeCombinedSelectivity(List conjuncts) { + // Collect all estimated selectivities. + List selectivities = new ArrayList<>(); + for (Expr e : conjuncts) { + if (e.hasSelectivity()) selectivities.add(e.getSelectivity()); + } + if (selectivities.size() != conjuncts.size()) { + // Some conjuncts have no estimated selectivity. Use a single default + // representative selectivity for all those conjuncts. + selectivities.add(Expr.DEFAULT_SELECTIVITY); + } + // Sort the selectivities to get a consistent estimate, regardless of the original + // conjunct order. Sort in ascending order such that the most selective conjunct + // is fully applied. + Collections.sort(selectivities); + double result = 1.0; + // selectivity = 1 * (s1)^(1/1) * (s2)^(1/2) * ... * (sn-1)^(1/(n-1)) * (sn)^(1/n) + for (int i = 0; i < selectivities.size(); ++i) { + // Exponential backoff for each selectivity multiplied into the final result. + result *= Math.pow(selectivities.get(i), 1.0 / (double) (i + 1)); + } + // Bound result in [0, 1] + return Math.max(0.0, Math.min(1.0, result)); + } + + protected double computeSelectivity() { + for (Expr expr : conjuncts) { + expr.setSelectivity(); + } + return computeCombinedSelectivity(conjuncts); + } + + /** + * Compute the product of the selectivity of all conjuncts. + * This function is used for old cardinality in finalize() + */ + protected double computeOldSelectivity() { + double prod = 1.0; + for (Expr e : conjuncts) { + if (e.getSelectivity() < 0) { + return -1.0; + } + prod *= e.getSelectivity(); + } + return prod; + } + + // Compute the cardinality after applying conjuncts based on 'preConjunctCardinality'. + protected void applyConjunctsSelectivity() { + if (cardinality == -1) { + return; + } + applySelectivity(); + } + + // Compute the cardinality after applying conjuncts with 'selectivity', based on + // 'preConjunctCardinality'. + private void applySelectivity() { + double selectivity = computeSelectivity(); + Preconditions.checkState(cardinality >= 0); + long preConjunctCardinality = cardinality; + cardinality = Math.round(cardinality * selectivity); + // don't round cardinality down to zero for safety. + if (cardinality == 0 && preConjunctCardinality > 0) { + cardinality = 1; + } } public ScanNode getScanNodeInOneFragmentByTupleId(TupleId tupleId) { @@ -695,4 +779,13 @@ protected String getRuntimeFilterExplainString(boolean isBuildNode) { } return Joiner.on(", ").join(filtersStr) + "\n"; } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append("[").append(getId().asInt()).append(": ").append(getPlanNodeName()).append("]"); + sb.append("\nFragment: ").append(getFragmentId().asInt()).append("]"); + sb.append("\n").append(getNodeExplainString("", TExplainLevel.BRIEF)); + return sb.toString(); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/Planner.java b/fe/fe-core/src/main/java/org/apache/doris/planner/Planner.java index 16c0a371405648..6fd8d6e69f078d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/Planner.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/Planner.java @@ -183,10 +183,26 @@ public void createPlanFragments(StatementBase statement, Analyzer analyzer, TQue if (selectFailed) { throw new MVSelectFailedException("Failed to select materialize view"); } - // compute mem layout *before* finalize(); finalize() may reference - // TupleDescriptor.avgSerializedSize + + /** + * - Under normal circumstances, computeMemLayout() will be executed + * at the end of the init function of the plan node. + * Such as : + * OlapScanNode { + * init () { + * analyzer.materializeSlots(conjuncts); + * computeTupleStatAndMemLayout(analyzer); + * computeStat(); + * } + * } + * - However Doris is currently unable to determine + * whether it is possible to cut or increase the columns in the tuple after PlanNode.init(). + * - Therefore, for the time being, computeMemLayout() can only be placed + * after the completion of the entire single node planner. + */ analyzer.getDescTbl().computeMemLayout(); singleNodePlan.finalize(analyzer); + if (queryOptions.num_nodes == 1) { // single-node execution; we're almost done singleNodePlan = addUnassignedConjuncts(analyzer, singleNodePlan); diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/RepeatNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/RepeatNode.java index b70af9e8a9c949..cc8ab6f6bdeb83 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/RepeatNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/RepeatNode.java @@ -33,11 +33,13 @@ import org.apache.doris.thrift.TPlanNodeType; import org.apache.doris.thrift.TRepeatNode; -import org.apache.commons.collections.CollectionUtils; - import com.google.common.base.MoreObjects; import com.google.common.base.Preconditions; +import org.apache.commons.collections.CollectionUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + import java.util.ArrayList; import java.util.BitSet; import java.util.Collections; @@ -52,6 +54,8 @@ */ public class RepeatNode extends PlanNode { + private static final Logger LOG = LogManager.getLogger(RepeatNode.class); + private List> repeatSlotIdList; private Set allSlotId; private TupleDescriptor outputTupleDesc; @@ -96,8 +100,11 @@ private static List> buildIdSetList(List> repeatSlotIdL @Override public void computeStats(Analyzer analyzer) { avgRowSize = 0; - cardinality = 0; numNodes = 1; + cardinality = 0; + if (LOG.isDebugEnabled()) { + LOG.debug("stats Sort: cardinality=" + cardinality); + } } @Override @@ -128,7 +135,7 @@ public void init(Analyzer analyzer) throws UserException { ((SlotRef) slot).getDesc().setIsNullable(true); } } - outputTupleDesc.computeMemLayout(); + outputTupleDesc.computeStatAndMemLayout(); List> groupingIdList = new ArrayList<>(); List exprList = groupByClause.getGroupingExprs(); @@ -163,7 +170,7 @@ public void init(Analyzer analyzer) throws UserException { for (TupleId id : tupleIds) { analyzer.getTupleDesc(id).setIsMaterialized(true); } - computeMemLayout(analyzer); + computeTupleStatAndMemLayout(analyzer); computeStats(analyzer); createDefaultSmap(analyzer); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/ScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/ScanNode.java index e0398810f5d130..649d1c5f5347f5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/ScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/ScanNode.java @@ -17,6 +17,7 @@ package org.apache.doris.planner; +import org.apache.doris.analysis.Analyzer; import org.apache.doris.analysis.Expr; import org.apache.doris.analysis.SlotDescriptor; import org.apache.doris.analysis.TupleDescriptor; @@ -43,6 +44,13 @@ public ScanNode(PlanNodeId id, TupleDescriptor desc, String planNodeName) { this.desc = desc; } + @Override + public void init(Analyzer analyzer) throws UserException { + super.init(analyzer); + // materialize conjuncts in where + analyzer.materializeSlots(conjuncts); + } + /** * Helper function to parse a "host:port" address string into TNetworkAddress * This is called with ipaddress:port when doing scan range assigment. diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/SelectNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/SelectNode.java index c2d30bcb168f08..d8eacd21c384bc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/SelectNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/SelectNode.java @@ -18,14 +18,15 @@ package org.apache.doris.planner; import org.apache.doris.analysis.Analyzer; -import org.apache.doris.common.UserException; import org.apache.doris.analysis.Expr; - +import org.apache.doris.common.UserException; import org.apache.doris.thrift.TExplainLevel; import org.apache.doris.thrift.TPlanNode; import org.apache.doris.thrift.TPlanNodeType; -import org.apache.logging.log4j.Logger; + import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + import java.util.List; /** @@ -54,16 +55,29 @@ protected void toThrift(TPlanNode msg) { @Override public void init(Analyzer analyzer) throws UserException { - analyzer.markConjunctsAssigned(conjuncts); - computeStats(analyzer); - createDefaultSmap(analyzer); + super.init(analyzer); + analyzer.markConjunctsAssigned(conjuncts); + computeStats(analyzer); } @Override public void computeStats(Analyzer analyzer) { super.computeStats(analyzer); + if (!analyzer.safeIsEnableJoinReorderBasedCost()) { + return; + } + cardinality = getChild(0).cardinality; + applyConjunctsSelectivity(); + capCardinalityAtLimit(); + if (LOG.isDebugEnabled()) { + LOG.debug("stats Select: cardinality={}", this.cardinality); + } + } + + @Override + protected void computeOldCardinality() { long cardinality = getChild(0).cardinality; - double selectivity = computeSelectivity(); + double selectivity = computeOldSelectivity(); if (cardinality < 0 || selectivity < 0) { this.cardinality = -1; } else { diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/SetOperationNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/SetOperationNode.java index 502d4ec72db5db..fbd3a4824935a2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/SetOperationNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/SetOperationNode.java @@ -17,17 +17,13 @@ package org.apache.doris.planner; -import com.google.common.base.Joiner; -import com.google.common.base.Preconditions; -import com.google.common.collect.Lists; - -import org.apache.commons.collections.CollectionUtils; import org.apache.doris.analysis.Analyzer; import org.apache.doris.analysis.Expr; import org.apache.doris.analysis.SlotDescriptor; import org.apache.doris.analysis.SlotRef; import org.apache.doris.analysis.TupleDescriptor; import org.apache.doris.analysis.TupleId; +import org.apache.doris.common.CheckedMath; import org.apache.doris.thrift.TExceptNode; import org.apache.doris.thrift.TExplainLevel; import org.apache.doris.thrift.TExpr; @@ -36,18 +32,19 @@ import org.apache.doris.thrift.TPlanNodeType; import org.apache.doris.thrift.TUnionNode; -import org.apache.commons.collections.CollectionUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import com.google.common.base.Joiner; import com.google.common.base.Preconditions; import com.google.common.collect.Lists; +import org.apache.commons.collections.CollectionUtils; + import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + /** * Node that merges the results of its child plans, Normally, this is done by * materializing the corresponding result exprs into a new tuple. However, if @@ -131,12 +128,24 @@ public List> getMaterializedConstExprLists_() { @Override public void computeStats(Analyzer analyzer) { super.computeStats(analyzer); + if (!analyzer.safeIsEnableJoinReorderBasedCost()) { + return; + } + computeCardinality(); + } + + @Override + protected void computeOldCardinality() { + computeCardinality(); + } + + private void computeCardinality() { cardinality = constExprLists_.size(); for (PlanNode child : children) { // ignore missing child cardinality info in the hope it won't matter enough // to change the planning outcome if (child.cardinality > 0) { - cardinality = addCardinalities(cardinality, child.cardinality); + cardinality = CheckedMath.checkedAdd(cardinality, child.cardinality); } } // The number of nodes of a set operation node is -1 (invalid) if all the referenced tables @@ -145,31 +154,12 @@ public void computeStats(Analyzer analyzer) { if (numNodes == -1) { numNodes = 1; } - cardinality = capAtLimit(cardinality); - if (LOG.isTraceEnabled()) { - LOG.trace("stats Union: cardinality=" + Long.toString(cardinality)); + capCardinalityAtLimit(); + if (LOG.isDebugEnabled()) { + LOG.trace("stats Union: cardinality=" + cardinality); } } - protected long capAtLimit(long cardinality) { - if (hasLimit()) { - if (cardinality == -1) { - return limit; - } else { - return Math.min(cardinality, limit); - } - } - return cardinality; - } - - /* - @Override - public void computeResourceProfile(TQueryOptions queryOptions) { - // TODO: add an estimate - resourceProfile_ = new ResourceProfile(0, 0); - } - */ - /** * Returns true if rows from the child with 'childTupleIds' and 'childResultExprs' can * be returned directly by the set operation node (without materialization into a new tuple). @@ -270,7 +260,7 @@ void computePassthrough(Analyzer analyzer) { @Override public void init(Analyzer analyzer) { Preconditions.checkState(conjuncts.isEmpty()); - computeMemLayout(analyzer); + computeTupleStatAndMemLayout(analyzer); computeStats(analyzer); // except Node must not reorder the child if (!(this instanceof ExceptNode)) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/SingleNodePlanner.java b/fe/fe-core/src/main/java/org/apache/doris/planner/SingleNodePlanner.java index 9dcadd7dccbe80..f11fbf7ef88198 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/SingleNodePlanner.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/SingleNodePlanner.java @@ -56,6 +56,7 @@ import org.apache.doris.catalog.Table; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.FeConstants; +import org.apache.doris.common.Pair; import org.apache.doris.common.Reference; import org.apache.doris.common.UserException; @@ -64,15 +65,20 @@ import com.google.common.collect.Iterables; import com.google.common.collect.Lists; import com.google.common.collect.Maps; +import com.google.common.collect.Sets; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.HashSet; import java.util.Set; +import java.util.stream.Collectors; /** * Constructs a non-executable single-node plan from an analyzed parse tree. @@ -81,7 +87,7 @@ * The single-node plan needs to be wrapped in a plan fragment for it to be executable. */ public class SingleNodePlanner { - private final static Logger LOG = LogManager.getLogger(DistributedPlanner.class); + private final static Logger LOG = LogManager.getLogger(SingleNodePlanner.class); private final PlannerContext ctx_; private final ArrayList scanNodes = Lists.newArrayList(); @@ -91,6 +97,10 @@ public SingleNodePlanner(PlannerContext ctx) { ctx_ = ctx; } + public PlannerContext getPlannerContext() { + return ctx_; + } + public ArrayList getScanNodes() { return scanNodes; } @@ -652,6 +662,270 @@ private void turnOffPreAgg(AggregateInfo aggInfo, SelectStmt selectStmt, Analyze } } + /** + * Return the cheapest plan that materializes the joins of all TableRefs in + * refPlans and the subplans of all applicable TableRefs in subplanRefs. + * Assumes that refPlans are in the order as they originally appeared in + * the query. + * For this plan: + * - the plan is executable, ie, all non-cross joins have equi-join predicates + * - the leftmost scan is over the largest of the inputs for which we can still + * construct an executable plan + * - from bottom to top, all rhs's are in increasing order of selectivity (percentage + * of surviving rows) + * - outer/cross/semi joins: rhs serialized size is < lhs serialized size; + * enforced via join inversion, if necessary + * - SubplanNodes are placed as low as possible in the plan tree - as soon as the + * required tuple ids of one or more TableRefs in subplanRefs are materialized + * Returns null if we can't create an executable plan. + */ + private PlanNode createCheapestJoinPlan(Analyzer analyzer, List> refPlans) throws UserException { + if (refPlans.size() == 1) { + return refPlans.get(0).second; + } + + // collect eligible candidates for the leftmost input; list contains + // (plan, materialized size) + List> candidates = new ArrayList<>(); + for (Pair entry : refPlans) { + TableRef ref = entry.first; + JoinOperator joinOp = ref.getJoinOp(); + + // Avoid reordering outer/semi joins which is generally incorrect. + // consideration of the joinOps that result from such a re-ordering (IMPALA-1281). + // TODO: Allow the rhs of any cross join as the leftmost table. This needs careful + if (joinOp.isOuterJoin() || joinOp.isSemiJoin() || joinOp.isCrossJoin()) { + continue; + } + + PlanNode plan = entry.second; + if (plan.getCardinality() == -1) { + // use 0 for the size to avoid it becoming the leftmost input + // TODO: Consider raw size of scanned partitions in the absence of stats. + candidates.add(new Pair<>(ref, new Long(0))); + LOG.debug("The candidate of " + ref.getUniqueAlias() + ": -1. " + + "Using 0 instead of -1 to avoid error"); + continue; + } + Preconditions.checkState(ref.isAnalyzed()); + long materializedSize = plan.getCardinality(); + candidates.add(new Pair<>(ref, new Long(materializedSize))); + LOG.debug("The candidate of " + ref.getUniqueAlias() + ": " + materializedSize); + } + // (ML): 这里感觉是不可能运行到的，因为起码第一个节点是inner join + if (candidates.isEmpty()) return null; + + // order candidates by descending materialized size; we want to minimize the memory + // consumption of the materialized hash tables required for the join sequence + Collections.sort(candidates, + new Comparator>() { + @Override + public int compare(Pair a, Pair b) { + long diff = b.second - a.second; + return (diff < 0 ? -1 : (diff > 0 ? 1 : 0)); + } + }); + + for (Pair candidate : candidates) { + PlanNode result = createJoinPlan(analyzer, candidate.first, refPlans); + if (result != null) return result; + } + return null; + } + + boolean candidateCardinalityIsSmaller(PlanNode candidate, long candidateInnerNodeCardinality, + PlanNode newRoot, long newRootInnerNodeCardinality) { + if (candidate.getCardinality() < newRoot.getCardinality()) { + return true; + } else if (candidate.getCardinality() == newRoot.getCardinality()) { + if (((candidate instanceof HashJoinNode) && ((HashJoinNode) candidate).getJoinOp().isInnerJoin()) + && ((newRoot instanceof HashJoinNode) && ((HashJoinNode) newRoot).getJoinOp().isInnerJoin())) { + if (candidateInnerNodeCardinality < newRootInnerNodeCardinality) { + return true; + } + } + } + return false; + } + + /** + * Returns a plan with leftmostRef's plan as its leftmost input; the joins + * are in decreasing order of selectiveness (percentage of rows they eliminate). + * Creates and adds subplan nodes as soon as the tuple ids required by at least one + * subplan ref are materialized by a join node added during plan generation. + */ + // (ML): change the function name + private PlanNode createJoinPlan(Analyzer analyzer, + TableRef leftmostRef, List> refPlans) + throws UserException { + LOG.debug("Try to create a query plan starting with " + leftmostRef.getUniqueAlias()); + + // the refs that have yet to be joined + List> remainingRefs = new ArrayList<>(); + PlanNode root = null; // root of accumulated join plan + for (Pair entry : refPlans) { + if (entry.first == leftmostRef) { + root = entry.second; + } else { + remainingRefs.add(entry); + } + } + Preconditions.checkNotNull(root); + + // Maps from a TableRef in refPlans with an outer/semi join op to the set of + // TableRefs that precede it refPlans (i.e., in FROM-clause order). + // The map is used to place outer/semi joins at a fixed position in the plan tree + // (IMPALA-860), s.t. all the tables appearing to the left/right of an outer/semi + // join in the original query still remain to the left/right after join ordering. + // This prevents join re-ordering across outer/semi joins which is generally wrong. + + /** + * Key of precedingRefs: the right table ref of outer or semi join + * Value of precedingRefs: the preceding refs of this key + * For example: + * select * from t1, t2, t3 left join t4, t5, t6 right semi join t7, t8, t9 + * Map: + * { t4: [t1, t2, t3], + * t7: [t1, t2, t3, t4, t5, t6] + * } + */ + Map> precedingRefs = new HashMap<>(); + List tmpTblRefs = new ArrayList<>(); + for (Pair entry : refPlans) { + TableRef tblRef = entry.first; + if (tblRef.getJoinOp().isOuterJoin() || tblRef.getJoinOp().isSemiJoin()) { + precedingRefs.put(tblRef, Sets.newHashSet(tmpTblRefs)); + } + tmpTblRefs.add(tblRef); + } + + // Refs that have been joined. The union of joinedRefs and the refs in remainingRefs + // are the set of all table refs. + Set joinedRefs = Sets.newHashSet(leftmostRef); + // two statistical value + long numOps = 0; + // A total of several rounds of successful selection + int successfulSelectionTimes = 0; + while (!remainingRefs.isEmpty()) { + // We minimize the resulting cardinality at each step in the join chain, + // which minimizes the total number of hash table lookups. + PlanNode newRoot = null; + Pair minEntry = null; + long newRootRightChildCardinality = 0; + for (Pair tblRefToPlanNodeOfCandidate : remainingRefs) { + TableRef tblRefOfCandidate = tblRefToPlanNodeOfCandidate.first; + long cardinalityOfCandidate = tblRefToPlanNodeOfCandidate.second.getCardinality(); + PlanNode rootPlanNodeOfCandidate = tblRefToPlanNodeOfCandidate.second; + JoinOperator joinOp = tblRefOfCandidate.getJoinOp(); + + // Place outer/semi joins at a fixed position in the plan tree. + Set requiredRefs = precedingRefs.get(tblRefOfCandidate); + if (requiredRefs != null) { + Preconditions.checkState(joinOp.isOuterJoin() + || joinOp.isSemiJoin()); + /** + * The semi and outer join nodes are similar to the stop nodes in each round of the algorithm. + * If the stop node is encountered during the current round of optimal selection, + * it means that the following nodes do not need to be referred to. + * This round has been completed. + * There are two situation in here. + * Situation 1: required table refs have not been placed yet + * t1, t2, t3 left join t4, t5 + * Round 1: t3, t1(new root) meets t4(stop) + * stop this round and begin next round + * Situation 2: the remaining table refs to prevent incorrect re-ordering of tables across outer/semi joins + * Round 1: t5, t1, t2, t3(root) meets t4(stop) + * stop this round while the new root is null + * planning failed and return null + */ + if (!requiredRefs.equals(joinedRefs)) { + break; + } + } + // reset assigned conjuncts of analyzer in every compare + analyzer.setAssignedConjuncts(root.getAssignedConjuncts()); + PlanNode candidate = createJoinNode(analyzer, root, rootPlanNodeOfCandidate, tblRefOfCandidate); + // (ML): 这里还需要吗？应该不会返回null吧 + if (candidate == null) { + continue; + } + // Have the build side of a join copy data to a compact representation + // in the tuple buffer. + candidate.getChildren().get(1).setCompactData(true); + + if (LOG.isDebugEnabled()) { + StringBuilder stringBuilder = new StringBuilder(); + stringBuilder.append("The " + tblRefOfCandidate.getUniqueAlias() + " is right child of join node."); + stringBuilder.append("The join cardinality is " + candidate.getCardinality() + "."); + stringBuilder.append("In round " + successfulSelectionTimes); + LOG.debug(stringBuilder.toString()); + } + + // Use 'candidate' as the new root; don't consider any other table refs at this + // position in the plan. + if (joinOp.isOuterJoin() || joinOp.isSemiJoin()) { + newRoot = candidate; + minEntry = tblRefToPlanNodeOfCandidate; + break; + } + + // Always prefer Hash Join over Nested-Loop Join due to limited costing + // infrastructure. + /** + * The following three conditions are met while the candidate is better. + * 1. The first candidate + * 2. The candidate is better than new root: [t3, t2] pk [t3, t1] => [t3, t1] + * 3. The hash join is better than cross join: [t3 cross t1] pk [t3 hash t2] => t3 hash t2 + */ + if (newRoot == null + || ((candidate.getClass().equals(newRoot.getClass()) && candidateCardinalityIsSmaller(candidate, tblRefToPlanNodeOfCandidate.second.getCardinality(), newRoot, newRootRightChildCardinality))) + || (candidate instanceof HashJoinNode && newRoot instanceof CrossJoinNode)) { + newRoot = candidate; + minEntry = tblRefToPlanNodeOfCandidate; + newRootRightChildCardinality = cardinalityOfCandidate; + } + } + + /** + * The table after the outer or semi join is wrongly planned to the front, + * causing the first tblRefToPlanNodeOfCandidate (outer or semi tbl ref) in this round of loop to fail and exit the loop. + * This means that the current leftmost node must be wrong, and the correct result cannot be planned. + * + * For example: + * Query: t1 left join t2 inner join t3 + * Input params: t3(left most tbl ref), [t1,t2] (remaining refs) + * Round 1: t3, t1 (joined refs) t2 (remaining refs) + * Round 2: requiredRefs.equals(joinedRefs) is false and break, the newRoot is null + * Result: null + * The t3 should not appear before t2 so planning is fail + */ + if (newRoot == null) { + // Could not generate a valid plan. + // for example: the biggest table is the last table + return null; + } + + // we need to insert every rhs row into the hash table and then look up + // every lhs row + long lhsCardinality = root.getCardinality(); + long rhsCardinality = minEntry.second.getCardinality(); + numOps += lhsCardinality + rhsCardinality; + if (LOG.isDebugEnabled()) { + LOG.debug("Round " + successfulSelectionTimes + " chose " + minEntry.first.getUniqueAlias() + + " #lhs=" + lhsCardinality + " #rhs=" + rhsCardinality + " #ops=" + numOps); + } + remainingRefs.remove(minEntry); + joinedRefs.add(minEntry.first); + root = newRoot; + analyzer.setAssignedConjuncts(root.getAssignedConjuncts()); + ++successfulSelectionTimes; + } + + LOG.debug("The final join sequence is " + + joinedRefs.stream().map(TableRef::getUniqueAlias).collect(Collectors.joining(","))); + return root; + } + /** * Create tree of PlanNodes that implements the Select/Project/Join/Group by/Having * of the selectStmt query block. @@ -691,35 +965,66 @@ private PlanNode createSelectPlan(SelectStmt selectStmt, Analyzer analyzer, long return createAggregationPlan(selectStmt, analyzer, emptySetNode); } - // create left-deep sequence of binary hash joins; assign node ids as we go along - TableRef tblRef = selectStmt.getTableRefs().get(0); - materializeTableResultForCrossJoinOrCountStar(tblRef, analyzer); - PlanNode root = createTableRefNode(analyzer, tblRef, selectStmt); - // to change the inner contains analytic function - // selectStmt.seondSubstituteInlineViewExprs(analyzer.getChangeResSmap()); - - // add aggregate node here + PlanNode root = null; AggregateInfo aggInfo = selectStmt.getAggInfo(); - turnOffPreAgg(aggInfo, selectStmt, analyzer, root); + if (analyzer.safeIsEnableJoinReorderBasedCost()) { + LOG.debug("Using new join reorder strategy when enable_join_reorder_based_cost is true"); + // create plans for our table refs; use a list here instead of a map to + // maintain a deterministic order of traversing the TableRefs during join + // plan generation (helps with tests) + List> refPlans = Lists.newArrayList(); + for (TableRef ref : selectStmt.getTableRefs()) { + materializeTableResultForCrossJoinOrCountStar(ref, analyzer); + PlanNode plan = createTableRefNode(analyzer, ref, selectStmt); + turnOffPreAgg(aggInfo, selectStmt, analyzer, plan); + + if (plan instanceof OlapScanNode) { + OlapScanNode olapNode = (OlapScanNode) plan; + // this olap scan node has been turn off pre-aggregation, should not be turned on again. + // e.g. select sum(v1) from (select v1 from test_table); + if (!olapNode.isPreAggregation()) { + olapNode.setCanTurnOnPreAggr(false); + } + } - if (root instanceof OlapScanNode) { - OlapScanNode olapNode = (OlapScanNode) root; - // this olap scan node has been turn off pre-aggregation, should not be turned on again. - // e.g. select sum(v1) from (select v1 from test_table); - if (!olapNode.isPreAggregation()) { - olapNode.setCanTurnOnPreAggr(false); + Preconditions.checkState(plan != null); + refPlans.add(new Pair(ref, plan)); + } + // save state of conjunct assignment; needed for join plan generation + for (Pair entry : refPlans) { + entry.second.setAssignedConjuncts(analyzer.getAssignedConjuncts()); + } + root = createCheapestJoinPlan(analyzer, refPlans); + Preconditions.checkState(root != null); + } else { + // create left-deep sequence of binary hash joins; assign node ids as we go along + TableRef tblRef = selectStmt.getTableRefs().get(0); + materializeTableResultForCrossJoinOrCountStar(tblRef, analyzer); + root = createTableRefNode(analyzer, tblRef, selectStmt); + // to change the inner contains analytic function + // selectStmt.seondSubstituteInlineViewExprs(analyzer.getChangeResSmap()); + + turnOffPreAgg(aggInfo, selectStmt, analyzer, root); + + if (root instanceof OlapScanNode) { + OlapScanNode olapNode = (OlapScanNode) root; + // this olap scan node has been turn off pre-aggregation, should not be turned on again. + // e .g. select sum(v1) from (select v1 from test_table); + if (!olapNode.isPreAggregation()) { + olapNode.setCanTurnOnPreAggr(false); + } } - } - for (int i = 1; i < selectStmt.getTableRefs().size(); ++i) { - TableRef outerRef = selectStmt.getTableRefs().get(i - 1); - TableRef innerRef = selectStmt.getTableRefs().get(i); - root = createJoinNode(analyzer, root, outerRef, innerRef, selectStmt); - // Have the build side of a join copy data to a compact representation - // in the tuple buffer. - root.getChildren().get(1).setCompactData(true); - root.assignConjuncts(analyzer); + for (int i = 1; i < selectStmt.getTableRefs().size(); ++i) { + TableRef outerRef = selectStmt.getTableRefs().get(i - 1); + TableRef innerRef = selectStmt.getTableRefs().get(i); + root = createJoinNode(analyzer, root, innerRef, selectStmt); + // Have the build side of a join copy data to a compact representation + // in the tuple buffer. + root.getChildren().get(1).setCompactData(true); + root.assignConjuncts(analyzer); + } } if (selectStmt.getSortInfo() != null && selectStmt.getLimit() == -1 @@ -902,7 +1207,7 @@ private TupleDescriptor createResultTupleDescriptor(SelectStmt selectStmt, // slotDesc.setStats(ColumnStats.fromExpr(resultExpr)); slotDesc.setIsMaterialized(true); } - tupleDesc.computeMemLayout(); + tupleDesc.computeStatAndMemLayout(); return tupleDesc; } @@ -1451,15 +1756,14 @@ private PlanNode createScanNode(Analyzer analyzer, TableRef tblRef, SelectStmt s scanNode.setSortColumn(tblRef.getSortColumn()); scanNode.addConjuncts(pushDownConjuncts); } - // assignConjuncts(scanNode, analyzer); - scanNode.init(analyzer); - // TODO chenhao add - // materialize conjuncts in where - analyzer.materializeSlots(scanNode.getConjuncts()); scanNodes.add(scanNode); + // now we put the selectStmtToScanNodes's init before the scanNode.init List scanNodeList = selectStmtToScanNodes.computeIfAbsent(selectStmt.getAnalyzer(), k -> Lists.newArrayList()); scanNodeList.add(scanNode); + + scanNode.init(analyzer); + return scanNode; } @@ -1549,18 +1853,9 @@ private void getHashLookupJoinConjuncts(Analyzer analyzer, PlanNode left, PlanNo } } - /** - * Creates a new node to join outer with inner. Collects and assigns join conjunct - * as well as regular conjuncts. Calls init() on the new join node. - * Throws if the JoinNode.init() fails. - */ - private PlanNode createJoinNode(Analyzer analyzer, PlanNode outer, TableRef outerRef, TableRef innerRef, - SelectStmt selectStmt) - throws UserException, AnalysisException { + private PlanNode createJoinNodeBase(Analyzer analyzer, PlanNode outer, PlanNode inner, TableRef innerRef) + throws UserException { materializeTableResultForCrossJoinOrCountStar(innerRef, analyzer); - // the rows coming from the build node only need to have space for the tuple - // materialized by that node - PlanNode inner = createTableRefNode(analyzer, innerRef, selectStmt); List eqJoinConjuncts = Lists.newArrayList(); Reference errMsg = new Reference(); @@ -1577,8 +1872,8 @@ private PlanNode createJoinNode(Analyzer analyzer, PlanNode outer, TableRef oute } // construct cross join node - LOG.debug("Join between {} and {} requires at least one conjunctive equality predicate between the two tables", - outerRef.getAliasAsName(), innerRef.getAliasAsName()); + // LOG.debug("Join between {} and {} requires at least one conjunctive equality predicate between the two tables", + // outerRef.getAliasAsName(), innerRef.getAliasAsName()); // TODO If there are eq join predicates then we should construct a hash join CrossJoinNode result = new CrossJoinNode(ctx_.getNextNodeId(), outer, inner, innerRef); @@ -1606,6 +1901,28 @@ private PlanNode createJoinNode(Analyzer analyzer, PlanNode outer, TableRef oute return result; } + /* + for joinreorder + */ + public PlanNode createJoinNode(Analyzer analyzer, PlanNode outer, PlanNode inner, TableRef innerRef) + throws UserException { + return createJoinNodeBase(analyzer, outer, inner, innerRef); + } + + /** + * Creates a new node to join outer with inner. Collects and assigns join conjunct + * as well as regular conjuncts. Calls init() on the new join node. + * Throws if the JoinNode.init() fails. + */ + private PlanNode createJoinNode(Analyzer analyzer, PlanNode outer, TableRef innerRef, + SelectStmt selectStmt) throws UserException { + // the rows coming from the build node only need to have space for the tuple + // materialized by that node + PlanNode inner = createTableRefNode(analyzer, innerRef, selectStmt); + + return createJoinNodeBase(analyzer, outer, inner, innerRef); + } + /** * Create a tree of PlanNodes for the given tblRef, which can be a BaseTableRef, * CollectionTableRef or an InlineViewRef. diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/SortNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/SortNode.java index 93c139e43e9408..c0450b97b99de2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/SortNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/SortNode.java @@ -124,6 +124,19 @@ public void setCompactData(boolean on) { @Override protected void computeStats(Analyzer analyzer) { super.computeStats(analyzer); + if (!analyzer.safeIsEnableJoinReorderBasedCost()) { + return; + } + cardinality = getChild(0).cardinality; + applyConjunctsSelectivity(); + capCardinalityAtLimit(); + if (LOG.isDebugEnabled()) { + LOG.debug("stats Sort: cardinality=" + cardinality); + } + } + + @Override + protected void computeOldCardinality() { cardinality = getChild(0).cardinality; if (hasLimit()) { if (cardinality == -1) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/StreamLoadPlanner.java b/fe/fe-core/src/main/java/org/apache/doris/planner/StreamLoadPlanner.java index 39c0f5d35c0494..dcd8afbd0c286f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/StreamLoadPlanner.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/StreamLoadPlanner.java @@ -137,7 +137,7 @@ public TExecPlanFragmentParams plan(TUniqueId loadId) throws UserException { // create scan node scanNode = new StreamLoadScanNode(loadId, new PlanNodeId(0), tupleDesc, destTable, taskInfo); scanNode.init(analyzer); - descTable.computeMemLayout(); + descTable.computeStatAndMemLayout(); scanNode.finalize(analyzer); // create dest sink diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/StreamLoadScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/StreamLoadScanNode.java index 76058ff3224e33..200e232e2e9021 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/StreamLoadScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/StreamLoadScanNode.java @@ -143,7 +143,6 @@ public void init(Analyzer analyzer) throws UserException { initAndSetPrecedingFilter(taskInfo.getPrecedingFilter(), this.srcTupleDesc, analyzer); initAndSetWhereExpr(taskInfo.getWhereExpr(), this.desc, analyzer); - computeStats(analyzer); createDefaultSmap(analyzer); if (taskInfo.getColumnSeparator() != null) { @@ -169,6 +168,7 @@ public void init(Analyzer analyzer) throws UserException { brokerScanRange.setParams(params); brokerScanRange.setBrokerAddresses(Lists.newArrayList()); + computeStats(analyzer); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java index 0dcae49efc0f89..19bedaacfdc5f9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java @@ -84,6 +84,8 @@ public class SessionVariable implements Serializable, Writable { public static final String ENABLE_SQL_CACHE = "enable_sql_cache"; public static final String ENABLE_PARTITION_CACHE = "enable_partition_cache"; + public static final String ENABLE_COST_BASED_JOIN_REORDER = "enable_cost_based_join_reorder"; + public static final int MIN_EXEC_INSTANCE_NUM = 1; public static final int MAX_EXEC_INSTANCE_NUM = 32; // if set to true, some of stmt will be forwarded to master FE to get result @@ -288,6 +290,9 @@ public class SessionVariable implements Serializable, Writable { @VariableMgr.VarAttr(name = ENABLE_PARTITION_CACHE) public boolean enablePartitionCache = false; + @VariableMgr.VarAttr(name = ENABLE_COST_BASED_JOIN_REORDER) + private boolean enableJoinReorderBasedCost = false; + @VariableMgr.VarAttr(name = FORWARD_TO_MASTER) public boolean forwardToMaster = false; @@ -374,6 +379,8 @@ public void setSqlMode(long sqlMode) { this.sqlMode = sqlMode; } + public boolean isEnableJoinReorderBasedCost() { return enableJoinReorderBasedCost; } + public boolean isAutoCommit() { return autoCommit; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java index 14e5c315c32d9b..7ec62223c000b1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java @@ -356,7 +356,7 @@ public void execute(TUniqueId queryId) throws Exception { throw e; } catch (UserException e) { // analysis exception only print message, not print the stack - LOG.warn("execute Exception. {}", e.getMessage()); + LOG.warn("execute Exception. {}", e); context.getState().setError(e.getMessage()); context.getState().setErrType(QueryState.ErrType.ANALYSIS_ERR); } catch (Exception e) { diff --git a/fe/fe-core/src/test/java/org/apache/doris/planner/QueryPlanTest.java b/fe/fe-core/src/test/java/org/apache/doris/planner/QueryPlanTest.java index 5aac2dbf0062f5..d6535a38b4fdcf 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/planner/QueryPlanTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/planner/QueryPlanTest.java @@ -1456,6 +1456,8 @@ public void testAggregateSatisfyOlapTableDistribution() throws Exception { Assert.assertTrue(explainString.contains("AGGREGATE (update serialize)")); } + + @Test public void testLeadAndLagFunction() throws Exception { connectContext.setDatabase("default_cluster:test"); diff --git a/fe/fe-core/src/test/java/org/apache/doris/planner/SingleNodePlannerTest.java b/fe/fe-core/src/test/java/org/apache/doris/planner/SingleNodePlannerTest.java index 528d91fcb3bff9..8b77b485b0d868 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/planner/SingleNodePlannerTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/planner/SingleNodePlannerTest.java @@ -20,22 +20,36 @@ package org.apache.doris.planner; +import mockit.Mock; +import mockit.MockUp; +import mockit.Tested; import org.apache.doris.analysis.Analyzer; import org.apache.doris.analysis.BaseTableRef; +import org.apache.doris.analysis.BinaryPredicate; +import org.apache.doris.analysis.Expr; +import org.apache.doris.analysis.ExprSubstitutionMap; +import org.apache.doris.analysis.JoinOperator; import org.apache.doris.analysis.SlotDescriptor; import org.apache.doris.analysis.SlotId; +import org.apache.doris.analysis.SlotRef; import org.apache.doris.analysis.TableName; import org.apache.doris.analysis.TableRef; import org.apache.doris.analysis.TupleDescriptor; import org.apache.doris.analysis.TupleId; import org.apache.doris.catalog.Column; import org.apache.doris.catalog.Table; +import org.apache.doris.common.Pair; +import org.apache.doris.common.UserException; import org.apache.doris.common.jmockit.Deencapsulation; import com.google.common.collect.Lists; +import org.apache.log4j.Level; +import org.apache.log4j.LogManager; +import org.junit.Assert; import org.junit.Test; +import java.util.ArrayList; import java.util.List; import mockit.Expectations; @@ -69,6 +83,1348 @@ public void testMaterializeBaseTableRefResultForCrossJoinOrCountStar(@Injectable }; SingleNodePlanner singleNodePlanner = new SingleNodePlanner(plannerContext); Deencapsulation.invoke(singleNodePlanner, "materializeSlotForEmptyMaterializedTableRef", - baseTableRef, analyzer); + baseTableRef, analyzer); + } + + /* + Assumptions: + 1. The order of materialized size from smallest to largest is t1, t2 ... tn + 2. The predicates are orthogonal to each other and don't affect each other. + */ + + /* + Query: select * from t1 inner join t2 on t1.k1=t2.k1 + Original Query: select * from test1 inner join test2 on test1.k1=test2.k2 + Expect: without changed + */ + @Test + public void testJoinReorderWithTwoTuple1(@Injectable PlannerContext context, + @Injectable Analyzer analyzer, + @Injectable BaseTableRef tableRef1, + @Injectable OlapScanNode scanNode1, + @Injectable BaseTableRef tableRef2, + @Injectable OlapScanNode scanNode2, + @Injectable TupleDescriptor tupleDescriptor1, + @Injectable SlotDescriptor slotDescriptor1, + @Injectable SlotDescriptor slotDescriptor2, + @Injectable BinaryPredicate eqBinaryPredicate, + @Injectable SlotRef eqSlot1, + @Injectable SlotRef eqSlot2, + @Tested ExprSubstitutionMap exprSubstitutionMap) { + List slotDescriptors1 = Lists.newArrayList(); + slotDescriptors1.add(slotDescriptor1); + List slotDescriptors2 = Lists.newArrayList(); + slotDescriptors2.add(slotDescriptor2); + tableRef1.setJoinOp(JoinOperator.INNER_JOIN); + tableRef2.setJoinOp(JoinOperator.INNER_JOIN); + List eqConjuncts = Lists.newArrayList(); + eqConjuncts.add(eqBinaryPredicate); + new Expectations() { + { + tableRef1.isAnalyzed(); + result = true; + tableRef2.isAnalyzed(); + result = true; + scanNode1.getCardinality(); + result = 1; + scanNode2.getCardinality(); + result = 2; + tableRef1.getDesc(); + result = tupleDescriptor1; + tupleDescriptor1.getMaterializedSlots(); + result = slotDescriptors1; + analyzer.getEqJoinConjuncts(new ArrayList<>(), new ArrayList<>()); + result = eqConjuncts; + scanNode1.getTblRefIds(); + result = Lists.newArrayList(); + scanNode2.getTblRefIds(); + result = Lists.newArrayList(); + eqBinaryPredicate.getChild(0); + result = eqSlot1; + eqSlot1.isBoundByTupleIds(new ArrayList<>()); + result = true; + eqBinaryPredicate.getChild(1); + result = eqSlot2; + eqSlot2.isBoundByTupleIds(new ArrayList<>()); + result = true; + scanNode1.getTupleIds(); + result = Lists.newArrayList(); + scanNode2.getTupleIds(); + result = Lists.newArrayList(); + scanNode1.getOutputSmap(); + result = null; + scanNode2.getOutputSmap(); + result = null; + tableRef1.getUniqueAlias(); + result = "t1"; + tableRef2.getUniqueAlias(); + result = "t2"; + } + }; + new MockUp() { + @Mock + public ExprSubstitutionMap compose(ExprSubstitutionMap f, ExprSubstitutionMap g, + Analyzer analyzer) { + return exprSubstitutionMap; + } + + @Mock + public ExprSubstitutionMap combine(ExprSubstitutionMap f, ExprSubstitutionMap g) { + return exprSubstitutionMap; + } + }; + SingleNodePlanner singleNodePlanner = new SingleNodePlanner(context); + Pair pair1 = new Pair<>(tableRef1, scanNode1); + Pair pair2 = new Pair<>(tableRef2, scanNode2); + List> refPlans = Lists.newArrayList(); + refPlans.add(pair1); + refPlans.add(pair2); + + PlanNode cheapestJoinNode = + Deencapsulation.invoke(singleNodePlanner, "createCheapestJoinPlan", analyzer, refPlans); + Assert.assertEquals(2, cheapestJoinNode.getChildren().size()); + Assert.assertEquals(scanNode2, cheapestJoinNode.getChild(0)); + Assert.assertEquals(scanNode1, cheapestJoinNode.getChild(1)); + } + + /* + Query: select * from t1 left join t2 on t1.k1=t2.k1 + Original Query: select * from test1 left join test2 on test1.k1=test2.k2 + Expect: without changed + */ + @Test + public void testJoinReorderWithTwoTuple2(@Injectable PlannerContext context, + @Injectable Analyzer analyzer, + @Injectable BaseTableRef tableRef1, + @Injectable OlapScanNode scanNode1, + @Injectable BaseTableRef tableRef2, + @Injectable OlapScanNode scanNode2, + @Injectable TupleDescriptor tupleDescriptor2, + @Injectable SlotDescriptor slotDescriptor1, + @Injectable SlotDescriptor slotDescriptor2, + @Injectable BinaryPredicate eqBinaryPredicate, + @Injectable SlotRef eqSlot1, + @Injectable SlotRef eqSlot2, + @Tested ExprSubstitutionMap exprSubstitutionMap) { + Pair pair1 = new Pair<>(tableRef1, scanNode1); + Pair pair2 = new Pair<>(tableRef2, scanNode2); + List> refPlans = Lists.newArrayList(); + refPlans.add(pair1); + refPlans.add(pair2); + tableRef1.setJoinOp(JoinOperator.INNER_JOIN); + tableRef2.setJoinOp(JoinOperator.LEFT_OUTER_JOIN); + + List slotDescriptors1 = Lists.newArrayList(); + slotDescriptors1.add(slotDescriptor1); + List slotDescriptors2 = Lists.newArrayList(); + slotDescriptors2.add(slotDescriptor2); + List eqConjuncts = Lists.newArrayList(); + eqConjuncts.add(eqBinaryPredicate); + + new Expectations() { + { + tableRef1.isAnalyzed(); + result = true; + scanNode1.getCardinality(); + result = 1; + scanNode2.getCardinality(); + result = 2; + tableRef2.getDesc(); + result = tupleDescriptor2; + tupleDescriptor2.getMaterializedSlots(); + result = slotDescriptors2; + analyzer.getEqJoinConjuncts(new ArrayList<>(), new ArrayList<>()); + result = eqConjuncts; + scanNode1.getTblRefIds(); + result = Lists.newArrayList(); + scanNode2.getTblRefIds(); + result = Lists.newArrayList(); + eqBinaryPredicate.getChild(0); + result = eqSlot1; + eqSlot1.isBoundByTupleIds(new ArrayList<>()); + result = true; + eqBinaryPredicate.getChild(1); + result = eqSlot2; + eqSlot2.isBoundByTupleIds(new ArrayList<>()); + result = true; + scanNode1.getTupleIds(); + result = Lists.newArrayList(); + scanNode2.getTupleIds(); + result = Lists.newArrayList(); + scanNode1.getOutputSmap(); + result = null; + scanNode2.getOutputSmap(); + result = null; + tableRef1.getUniqueAlias(); + result = "t1"; + tableRef2.getUniqueAlias(); + result = "t2"; + tableRef1.getJoinOp(); + result = JoinOperator.INNER_JOIN; + tableRef2.getJoinOp(); + result = JoinOperator.LEFT_OUTER_JOIN; + } + }; + new MockUp() { + @Mock + public ExprSubstitutionMap compose(ExprSubstitutionMap f, ExprSubstitutionMap g, + Analyzer analyzer) { + return exprSubstitutionMap; + } + + @Mock + public ExprSubstitutionMap combine(ExprSubstitutionMap f, ExprSubstitutionMap g) { + return exprSubstitutionMap; + } + }; + + SingleNodePlanner singleNodePlanner = new SingleNodePlanner(context); + PlanNode cheapestJoinNode = Deencapsulation.invoke(singleNodePlanner, "createCheapestJoinPlan", analyzer, refPlans); + Assert.assertEquals(2, cheapestJoinNode.getChildren().size()); + Assert.assertEquals(true, cheapestJoinNode instanceof HashJoinNode); + Assert.assertEquals(JoinOperator.LEFT_OUTER_JOIN, ((HashJoinNode) cheapestJoinNode).getJoinOp()); + Assert.assertEquals(scanNode1, cheapestJoinNode.getChild(0)); + Assert.assertEquals(scanNode2, cheapestJoinNode.getChild(1)); + } + + /* + Query: select * from t1 right join t2 on t1.k1=t2.k1 + Original Query: select * from test1 right join test2 on test1.k1=test2.k2 + Expect: without changed + */ + @Test + public void testJoinReorderWithTwoTuple3(@Injectable PlannerContext context, + @Injectable Analyzer analyzer, + @Injectable BaseTableRef tableRef1, + @Injectable OlapScanNode scanNode1, + @Injectable BaseTableRef tableRef2, + @Injectable OlapScanNode scanNode2, + @Injectable TupleDescriptor tupleDescriptor2, + @Injectable SlotDescriptor slotDescriptor1, + @Injectable SlotDescriptor slotDescriptor2, + @Injectable BinaryPredicate eqBinaryPredicate, + @Injectable SlotRef eqSlot1, + @Injectable SlotRef eqSlot2, + @Tested ExprSubstitutionMap exprSubstitutionMap) { + Pair pair1 = new Pair<>(tableRef1, scanNode1); + Pair pair2 = new Pair<>(tableRef2, scanNode2); + List> refPlans = Lists.newArrayList(); + refPlans.add(pair1); + refPlans.add(pair2); + + List slotDescriptors1 = Lists.newArrayList(); + slotDescriptors1.add(slotDescriptor1); + List slotDescriptors2 = Lists.newArrayList(); + slotDescriptors2.add(slotDescriptor2); + List eqConjuncts = Lists.newArrayList(); + eqConjuncts.add(eqBinaryPredicate); + + new Expectations() { + { + tableRef1.isAnalyzed(); + result = true; + scanNode1.getCardinality(); + result = 1; + scanNode2.getCardinality(); + result = 2; + tableRef2.getDesc(); + result = tupleDescriptor2; + tupleDescriptor2.getMaterializedSlots(); + result = slotDescriptors2; + analyzer.getEqJoinConjuncts(new ArrayList<>(), new ArrayList<>()); + result = eqConjuncts; + scanNode1.getTblRefIds(); + result = Lists.newArrayList(); + scanNode2.getTblRefIds(); + result = Lists.newArrayList(); + eqBinaryPredicate.getChild(0); + result = eqSlot1; + eqSlot1.isBoundByTupleIds(new ArrayList<>()); + result = true; + eqBinaryPredicate.getChild(1); + result = eqSlot2; + eqSlot2.isBoundByTupleIds(new ArrayList<>()); + result = true; + scanNode1.getTupleIds(); + result = Lists.newArrayList(); + scanNode2.getTupleIds(); + result = Lists.newArrayList(); + scanNode1.getOutputSmap(); + result = null; + scanNode2.getOutputSmap(); + result = null; + tableRef1.getUniqueAlias(); + result = "t1"; + tableRef2.getUniqueAlias(); + result = "t2"; + tableRef1.getJoinOp(); + result = JoinOperator.INNER_JOIN; + tableRef2.getJoinOp(); + result = JoinOperator.RIGHT_OUTER_JOIN; + } + }; + new MockUp() { + @Mock + public ExprSubstitutionMap compose(ExprSubstitutionMap f, ExprSubstitutionMap g, + Analyzer analyzer) { + return exprSubstitutionMap; + } + + @Mock + public ExprSubstitutionMap combine(ExprSubstitutionMap f, ExprSubstitutionMap g) { + return exprSubstitutionMap; + } + }; + + SingleNodePlanner singleNodePlanner = new SingleNodePlanner(context); + PlanNode cheapestJoinNode = Deencapsulation.invoke(singleNodePlanner, "createCheapestJoinPlan", analyzer, refPlans); + Assert.assertEquals(2, cheapestJoinNode.getChildren().size()); + Assert.assertEquals(true, cheapestJoinNode instanceof HashJoinNode); + Assert.assertEquals(JoinOperator.RIGHT_OUTER_JOIN, ((HashJoinNode) cheapestJoinNode).getJoinOp()); + Assert.assertEquals(scanNode1, cheapestJoinNode.getChild(0)); + Assert.assertEquals(scanNode2, cheapestJoinNode.getChild(1)); + } + + /* + Query: select * from t1 left join t2 on t1.k1=t2.k1 inner join t3 on xxx + Original Query: select * from test1 left join test2 on test1.k1=test2.k1 inner join test3 where test2.k1=test3.k1; + Expect: without changed + */ + @Test + public void testKeepRightTableRefOnLeftJoin(@Injectable PlannerContext context, + @Injectable Analyzer analyzer, + @Injectable BaseTableRef tableRef1, + @Injectable OlapScanNode scanNode1, + @Injectable BaseTableRef tableRef2, + @Injectable OlapScanNode scanNode2, + @Injectable BaseTableRef tableRef3, + @Injectable OlapScanNode scanNode3, + @Injectable TupleDescriptor tupleDescriptor1, + @Injectable TupleDescriptor tupleDescriptor2, + @Injectable TupleDescriptor tupleDescriptor3, + @Injectable SlotDescriptor slotDescriptor1, + @Injectable SlotDescriptor slotDescriptor2, + @Injectable SlotDescriptor slotDescriptor3, + @Injectable BinaryPredicate eqBinaryPredicate1, + @Injectable BinaryPredicate eqBinaryPredicate2, + @Injectable BinaryPredicate eqBinaryPredicate3, + @Injectable SlotRef eqT1Slot1, + @Injectable SlotRef eqT2Slot2, + @Injectable SlotRef eqT3Slot3, + @Tested ExprSubstitutionMap exprSubstitutionMap) { + Pair pair1 = new Pair<>(tableRef1, scanNode1); + Pair pair2 = new Pair<>(tableRef2, scanNode2); + Pair pair3 = new Pair<>(tableRef3, scanNode3); + List> refPlans = Lists.newArrayList(); + refPlans.add(pair1); + refPlans.add(pair2); + refPlans.add(pair3); + + TupleId tupleId1 = new TupleId(1); + TupleId tupleId2 = new TupleId(2); + TupleId tupleId3 = new TupleId(3); + List tupleIds1 = Lists.newArrayList(tupleId1); + List tupleIds2 = Lists.newArrayList(tupleId2); + List tupleIds3 = Lists.newArrayList(tupleId3); + + List slotDescriptors1 = Lists.newArrayList(); + slotDescriptors1.add(slotDescriptor1); + List slotDescriptors2 = Lists.newArrayList(); + slotDescriptors2.add(slotDescriptor2); + List eqConjuncts1 = Lists.newArrayList(); + eqConjuncts1.add(eqBinaryPredicate1); + List eqConjuncts2 = Lists.newArrayList(); + eqConjuncts2.add(eqBinaryPredicate2); + List eqConjuncts3 = Lists.newArrayList(); + eqConjuncts3.add(eqBinaryPredicate3); + + + new Expectations() { + { + tableRef1.isAnalyzed(); + result = true; + tableRef3.isAnalyzed(); + result = true; + scanNode1.getCardinality(); + result = 1; + scanNode2.getCardinality(); + result = 2; + scanNode3.getCardinality(); + result = 3; + tableRef1.getDesc(); + result = tupleDescriptor1; + tupleDescriptor1.getMaterializedSlots(); + result = slotDescriptors1; + tableRef2.getDesc(); + result = tupleDescriptor2; + tupleDescriptor2.getMaterializedSlots(); + result = slotDescriptors2; + tableRef3.getDesc(); + result = tupleDescriptor3; + tupleDescriptor3.getMaterializedSlots(); + result = slotDescriptor3; + analyzer.getEqJoinConjuncts(tupleIds1, tupleIds2); + result = eqConjuncts1; + analyzer.getEqJoinConjuncts(Lists.newArrayList(tupleId1, tupleId2), tupleIds3); + result = eqConjuncts2; + analyzer.getEqJoinConjuncts(tupleIds3, tupleIds1); + result = eqConjuncts3; + scanNode1.getTblRefIds(); + result = Lists.newArrayList(tupleIds1); + scanNode2.getTblRefIds(); + result = Lists.newArrayList(tupleIds2); + scanNode3.getTblRefIds(); + result = Lists.newArrayList(tupleIds3); + eqBinaryPredicate1.getChild(0); + result = eqT1Slot1; + eqT1Slot1.isBoundByTupleIds(tupleIds1); + result = true; + eqBinaryPredicate1.getChild(1); + result = eqT2Slot2; + eqT2Slot2.isBoundByTupleIds(tupleIds2); + result = true; + eqT2Slot2.isBoundByTupleIds(Lists.newArrayList(tupleId1, tupleId2)); + result = true; + eqBinaryPredicate2.getChild(0); + result = eqT2Slot2; + eqBinaryPredicate2.getChild(1); + result = eqT3Slot3; + eqT3Slot3.isBoundByTupleIds(tupleIds3); + result = true; + eqBinaryPredicate3.getChild(0); + result = eqT1Slot1; + eqBinaryPredicate3.getChild(1); + result = eqT3Slot3; + scanNode1.getTupleIds(); + result = tupleIds1; + scanNode2.getTupleIds(); + result = tupleIds2; + scanNode3.getTupleIds(); + result = tupleId3; + scanNode1.getOutputSmap(); + result = null; + scanNode2.getOutputSmap(); + result = null; + tableRef1.getUniqueAlias(); + result = "t1"; + tableRef2.getUniqueAlias(); + result = "t2"; + tableRef3.getUniqueAlias(); + result = "t3"; + tableRef1.getJoinOp(); + result = JoinOperator.INNER_JOIN; + tableRef2.getJoinOp(); + result = JoinOperator.LEFT_OUTER_JOIN; + tableRef3.getJoinOp(); + result = JoinOperator.INNER_JOIN; + } + }; + new MockUp() { + @Mock + public ExprSubstitutionMap compose(ExprSubstitutionMap f, ExprSubstitutionMap g, + Analyzer analyzer) { + return exprSubstitutionMap; + } + + @Mock + public ExprSubstitutionMap combine(ExprSubstitutionMap f, ExprSubstitutionMap g) { + return exprSubstitutionMap; + } + }; + + SingleNodePlanner singleNodePlanner = new SingleNodePlanner(context); + PlanNode cheapestJoinNode = Deencapsulation.invoke(singleNodePlanner, "createCheapestJoinPlan", analyzer, refPlans); + Assert.assertEquals(2, cheapestJoinNode.getChildren().size()); + Assert.assertEquals(true, cheapestJoinNode instanceof HashJoinNode); + Assert.assertTrue(((HashJoinNode) cheapestJoinNode).getJoinOp().isInnerJoin()); + Assert.assertEquals(true, cheapestJoinNode.getChild(0) instanceof HashJoinNode); + HashJoinNode child0 = (HashJoinNode) cheapestJoinNode.getChild(0); + Assert.assertTrue(child0.getJoinOp().isOuterJoin()); + Assert.assertEquals(2, child0.getChildren().size()); + Assert.assertEquals(scanNode1, child0.getChild(0)); + Assert.assertEquals(scanNode2, child0.getChild(1)); + Assert.assertEquals(scanNode3, cheapestJoinNode.getChild(1)); + + } + + /* + Query: select * from t1 right join t2 on t1.k1=t2.k1 inner join t3 on xxx + Original Query: select * from test1 right join test2 on test1.k1=test2.k1 inner join test3 where test2.k1=test3.k1 + Expect: without changed + */ + @Test + public void testKeepRightTableRefOnRightJoin(@Injectable PlannerContext context, + @Injectable Analyzer analyzer, + @Injectable BaseTableRef tableRef1, + @Injectable OlapScanNode scanNode1, + @Injectable BaseTableRef tableRef2, + @Injectable OlapScanNode scanNode2, + @Injectable BaseTableRef tableRef3, + @Injectable OlapScanNode scanNode3, + @Injectable TupleDescriptor tupleDescriptor1, + @Injectable TupleDescriptor tupleDescriptor2, + @Injectable TupleDescriptor tupleDescriptor3, + @Injectable SlotDescriptor slotDescriptor1, + @Injectable SlotDescriptor slotDescriptor2, + @Injectable SlotDescriptor slotDescriptor3, + @Injectable BinaryPredicate eqBinaryPredicate1, + @Injectable BinaryPredicate eqBinaryPredicate2, + @Injectable BinaryPredicate eqBinaryPredicate3, + @Injectable SlotRef eqT1Slot1, + @Injectable SlotRef eqT2Slot2, + @Injectable SlotRef eqT3Slot3, + @Tested ExprSubstitutionMap exprSubstitutionMap) { + Pair pair1 = new Pair<>(tableRef1, scanNode1); + Pair pair2 = new Pair<>(tableRef2, scanNode2); + Pair pair3 = new Pair<>(tableRef3, scanNode3); + List> refPlans = Lists.newArrayList(); + refPlans.add(pair1); + refPlans.add(pair2); + refPlans.add(pair3); + + TupleId tupleId1 = new TupleId(1); + TupleId tupleId2 = new TupleId(2); + TupleId tupleId3 = new TupleId(3); + List tupleIds1 = Lists.newArrayList(tupleId1); + List tupleIds2 = Lists.newArrayList(tupleId2); + List tupleIds3 = Lists.newArrayList(tupleId3); + + List slotDescriptors1 = Lists.newArrayList(); + slotDescriptors1.add(slotDescriptor1); + List slotDescriptors2 = Lists.newArrayList(); + slotDescriptors2.add(slotDescriptor2); + List eqConjuncts1 = Lists.newArrayList(); + eqConjuncts1.add(eqBinaryPredicate1); + List eqConjuncts2 = Lists.newArrayList(); + eqConjuncts2.add(eqBinaryPredicate2); + List eqConjuncts3 = Lists.newArrayList(); + eqConjuncts3.add(eqBinaryPredicate3); + + + new Expectations() { + { + tableRef1.isAnalyzed(); + result = true; + tableRef3.isAnalyzed(); + result = true; + scanNode1.getCardinality(); + result = 1; + scanNode2.getCardinality(); + result = 2; + scanNode3.getCardinality(); + result = 3; + tableRef1.getDesc(); + result = tupleDescriptor1; + tupleDescriptor1.getMaterializedSlots(); + result = slotDescriptors1; + tableRef2.getDesc(); + result = tupleDescriptor2; + tupleDescriptor2.getMaterializedSlots(); + result = slotDescriptors2; + tableRef3.getDesc(); + result = tupleDescriptor3; + tupleDescriptor3.getMaterializedSlots(); + result = Lists.newArrayList(slotDescriptor3); + analyzer.getEqJoinConjuncts(tupleIds1, tupleIds2); + result = eqConjuncts1; + analyzer.getEqJoinConjuncts(Lists.newArrayList(tupleId1, tupleId2), tupleIds3); + result = eqConjuncts2; + analyzer.getEqJoinConjuncts(tupleIds3, tupleIds1); + result = eqConjuncts3; + scanNode1.getTblRefIds(); + result = Lists.newArrayList(tupleIds1); + scanNode2.getTblRefIds(); + result = Lists.newArrayList(tupleIds2); + scanNode3.getTblRefIds(); + result = Lists.newArrayList(tupleIds3); + eqBinaryPredicate1.getChild(0); + result = eqT1Slot1; + eqT1Slot1.isBoundByTupleIds(tupleIds1); + result = true; + eqBinaryPredicate1.getChild(1); + result = eqT2Slot2; + eqT2Slot2.isBoundByTupleIds(tupleIds2); + result = true; + eqT2Slot2.isBoundByTupleIds(Lists.newArrayList(tupleId1, tupleId2)); + result = true; + eqBinaryPredicate2.getChild(0); + result = eqT2Slot2; + eqBinaryPredicate2.getChild(1); + result = eqT3Slot3; + eqT3Slot3.isBoundByTupleIds(tupleIds3); + result = true; + eqBinaryPredicate3.getChild(0); + result = eqT1Slot1; + eqBinaryPredicate3.getChild(1); + result = eqT3Slot3; + scanNode1.getTupleIds(); + result = tupleIds1; + scanNode2.getTupleIds(); + result = tupleIds2; + scanNode3.getTupleIds(); + result = tupleId3; + scanNode1.getOutputSmap(); + result = null; + scanNode2.getOutputSmap(); + result = null; + tableRef1.getUniqueAlias(); + result = "t1"; + tableRef2.getUniqueAlias(); + result = "t2"; + tableRef3.getUniqueAlias(); + result = "t3"; + tableRef1.getJoinOp(); + result = JoinOperator.INNER_JOIN; + tableRef2.getJoinOp(); + result = JoinOperator.RIGHT_OUTER_JOIN; + tableRef3.getJoinOp(); + result = JoinOperator.INNER_JOIN; + } + }; + new MockUp() { + @Mock + public ExprSubstitutionMap compose(ExprSubstitutionMap f, ExprSubstitutionMap g, + Analyzer analyzer) { + return exprSubstitutionMap; + } + + @Mock + public ExprSubstitutionMap combine(ExprSubstitutionMap f, ExprSubstitutionMap g) { + return exprSubstitutionMap; + } + }; + + SingleNodePlanner singleNodePlanner = new SingleNodePlanner(context); + PlanNode cheapestJoinNode = Deencapsulation.invoke(singleNodePlanner, "createCheapestJoinPlan", analyzer, refPlans); + Assert.assertEquals(2, cheapestJoinNode.getChildren().size()); + Assert.assertEquals(true, cheapestJoinNode instanceof HashJoinNode); + Assert.assertTrue(((HashJoinNode) cheapestJoinNode).getJoinOp().isInnerJoin()); + Assert.assertEquals(true, cheapestJoinNode.getChild(0) instanceof HashJoinNode); + HashJoinNode child0 = (HashJoinNode) cheapestJoinNode.getChild(0); + Assert.assertTrue(child0.getJoinOp().isOuterJoin()); + Assert.assertEquals(2, child0.getChildren().size()); + Assert.assertEquals(scanNode1, child0.getChild(0)); + Assert.assertEquals(scanNode2, child0.getChild(1)); + Assert.assertEquals(scanNode3, cheapestJoinNode.getChild(1)); + } + + /* + Query: select * from t1,t2 right join t3,t5,t4 left join t6,t7 + Expect: keep t3, t6 position + t2, t1 right join t3, t4,t5 left join t6,t7 + */ + @Test + public void testKeepMultiOuterJoin(@Injectable PlannerContext context, + @Injectable Analyzer analyzer, + @Injectable BaseTableRef tableRef1, @Injectable OlapScanNode scanNode1, + @Injectable BaseTableRef tableRef2, @Injectable OlapScanNode scanNode2, + @Injectable BaseTableRef tableRef3, @Injectable OlapScanNode scanNode3, + @Injectable BaseTableRef tableRef4, @Injectable OlapScanNode scanNode4, + @Injectable BaseTableRef tableRef5, @Injectable OlapScanNode scanNode5, + @Injectable BaseTableRef tableRef6, @Injectable OlapScanNode scanNode6, + @Injectable BaseTableRef tableRef7, @Injectable OlapScanNode scanNode7, + @Injectable TupleDescriptor tupleDescriptor1, + @Injectable TupleDescriptor tupleDescriptor2, + @Injectable TupleDescriptor tupleDescriptor3, + @Injectable TupleDescriptor tupleDescriptor4, + @Injectable TupleDescriptor tupleDescriptor5, + @Injectable TupleDescriptor tupleDescriptor6, + @Injectable TupleDescriptor tupleDescriptor7, + @Injectable SlotDescriptor slotDescriptor1, + @Injectable SlotDescriptor slotDescriptor2, + @Injectable SlotDescriptor slotDescriptor3, + @Injectable SlotDescriptor slotDescriptor4, + @Injectable SlotDescriptor slotDescriptor5, + @Injectable SlotDescriptor slotDescriptor6, + @Injectable SlotDescriptor slotDescriptor7, + @Injectable BinaryPredicate eqBinaryPredicate1, + @Injectable BinaryPredicate eqBinaryPredicate2, + @Injectable BinaryPredicate eqBinaryPredicate3, + @Injectable BinaryPredicate eqBinaryPredicate4, + @Injectable BinaryPredicate eqBinaryPredicate5, + @Injectable BinaryPredicate eqBinaryPredicate6, + @Injectable BinaryPredicate eqBinaryPredicate7, + @Injectable SlotRef eqT1Slot1, + @Injectable SlotRef eqT2Slot2, + @Injectable SlotRef eqT3Slot3, + @Injectable SlotRef eqT4Slot4, + @Injectable SlotRef eqT5Slot5, + @Injectable SlotRef eqT6Slot6, + @Injectable SlotRef eqT7Slot7, + @Tested ExprSubstitutionMap exprSubstitutionMap) { + Pair pair1 = new Pair<>(tableRef1, scanNode1); + Pair pair2 = new Pair<>(tableRef2, scanNode2); + Pair pair3 = new Pair<>(tableRef3, scanNode3); + Pair pair4 = new Pair<>(tableRef4, scanNode4); + Pair pair5 = new Pair<>(tableRef5, scanNode5); + Pair pair6 = new Pair<>(tableRef6, scanNode6); + Pair pair7 = new Pair<>(tableRef7, scanNode7); + List> refPlans = Lists.newArrayList(); + refPlans.add(pair1); + refPlans.add(pair2); + refPlans.add(pair3); + refPlans.add(pair5); + refPlans.add(pair4); + refPlans.add(pair6); + refPlans.add(pair7); + + TupleId tupleId1 = new TupleId(1); + TupleId tupleId2 = new TupleId(2); + TupleId tupleId3 = new TupleId(3); + TupleId tupleId4 = new TupleId(4); + TupleId tupleId5 = new TupleId(5); + TupleId tupleId6 = new TupleId(6); + TupleId tupleId7 = new TupleId(7); + List tupleIds1 = Lists.newArrayList(tupleId1); + List tupleIds2 = Lists.newArrayList(tupleId2); + List tupleIds3 = Lists.newArrayList(tupleId3); + List tupleIds4 = Lists.newArrayList(tupleId4); + List tupleIds5 = Lists.newArrayList(tupleId5); + List tupleIds6 = Lists.newArrayList(tupleId6); + List tupleIds7 = Lists.newArrayList(tupleId7); + List tupleIds213 = Lists.newArrayList(tupleId2, tupleId1, tupleId3); + List tupleIds21345 = new ArrayList<>(); + tupleIds21345.addAll(tupleIds213); + tupleIds21345.add(tupleId4); + tupleIds21345.add(tupleId5); + List tupleIds213456 = Lists.newArrayList(tupleId2, tupleId1, tupleId3, tupleId4, tupleId5, tupleId6); + + List slotDescriptors1 = Lists.newArrayList(); + slotDescriptors1.add(slotDescriptor1); + List slotDescriptors2 = Lists.newArrayList(); + slotDescriptors2.add(slotDescriptor2); + List eqConjuncts1 = Lists.newArrayList(); + eqConjuncts1.add(eqBinaryPredicate1); + List eqConjuncts2 = Lists.newArrayList(); + eqConjuncts2.add(eqBinaryPredicate2); + List eqConjuncts3 = Lists.newArrayList(); + eqConjuncts3.add(eqBinaryPredicate3); + + + new Expectations() { + { + tableRef1.isAnalyzed(); + result = true; + tableRef2.isAnalyzed(); + result = true; + tableRef4.isAnalyzed(); + result = true; + tableRef5.isAnalyzed(); + result = true; + tableRef7.isAnalyzed(); + result = true; + scanNode1.getCardinality(); + result = 1; + scanNode2.getCardinality(); + result = 2; + scanNode3.getCardinality(); + result = 3; + scanNode4.getCardinality(); + result = 4; + scanNode5.getCardinality(); + result = 5; + scanNode6.getCardinality(); + result = 6; + scanNode7.getCardinality(); + result = 7; + tableRef1.getDesc(); + result = tupleDescriptor1; + tupleDescriptor1.getMaterializedSlots(); + result = slotDescriptors1; + tableRef2.getDesc(); + result = tupleDescriptor2; + tupleDescriptor2.getMaterializedSlots(); + result = slotDescriptors2; + tableRef3.getDesc(); + result = tupleDescriptor3; + tupleDescriptor3.getMaterializedSlots(); + result = Lists.newArrayList(slotDescriptor3); + tableRef4.getDesc(); + result = tupleDescriptor4; + tupleDescriptor4.getMaterializedSlots(); + result = Lists.newArrayList(slotDescriptor4); + tableRef5.getDesc(); + result = tupleDescriptor5; + tupleDescriptor5.getMaterializedSlots(); + result = Lists.newArrayList(slotDescriptor5); + tableRef6.getDesc(); + result = tupleDescriptor6; + tupleDescriptor6.getMaterializedSlots(); + result = Lists.newArrayList(slotDescriptor6); + tableRef7.getDesc(); + result = tupleDescriptor7; + tupleDescriptor7.getMaterializedSlots(); + result = Lists.newArrayList(slotDescriptor7); + analyzer.getEqJoinConjuncts(tupleIds7, tupleIds1); + result = eqConjuncts1; + eqBinaryPredicate1.getChild(0); + result = eqT7Slot7; + eqT7Slot7.isBoundByTupleIds(tupleIds7); + result = true; + eqBinaryPredicate1.getChild(1); + result = eqT1Slot1; + eqT1Slot1.isBoundByTupleIds(tupleIds1); + result = true; + analyzer.getEqJoinConjuncts(Lists.newArrayList(tupleId2, tupleId1), tupleIds3); + result = eqConjuncts2; + analyzer.getEqJoinConjuncts(tupleIds213, tupleIds5); + result = Lists.newArrayList(eqBinaryPredicate4); + analyzer.getEqJoinConjuncts(tupleIds213, tupleIds4); + result = Lists.newArrayList(eqBinaryPredicate5); + analyzer.getEqJoinConjuncts(tupleIds21345, tupleIds6); + result = Lists.newArrayList(eqBinaryPredicate6); + eqBinaryPredicate6.getChild(0); + result = eqT5Slot5; + eqBinaryPredicate6.getChild(1); + result = eqT6Slot6; + eqT5Slot5.isBoundByTupleIds(tupleIds21345); + result = true; + eqT6Slot6.isBoundByTupleIds(tupleIds6); + result = true; + analyzer.getEqJoinConjuncts(tupleIds213456, tupleIds7); + result = Lists.newArrayList(eqBinaryPredicate7); + eqBinaryPredicate7.getChild(0); + result = eqT6Slot6; + eqBinaryPredicate7.getChild(1); + result = eqT7Slot7; + eqT6Slot6.isBoundByTupleIds(tupleIds213456); + result = true; + eqT7Slot7.isBoundByTupleIds(tupleIds7); + result = true; + scanNode1.getTblRefIds(); + result = Lists.newArrayList(tupleIds1); + scanNode2.getTblRefIds(); + result = Lists.newArrayList(tupleIds2); + scanNode3.getTblRefIds(); + result = Lists.newArrayList(tupleIds3); + scanNode4.getTblRefIds(); + result = Lists.newArrayList(tupleId4); + scanNode5.getTblRefIds(); + result = Lists.newArrayList(tupleId5); + scanNode6.getTblRefIds(); + result = Lists.newArrayList(tupleId6); + scanNode7.getTblRefIds(); + result = Lists.newArrayList(tupleId7); + + eqT2Slot2.isBoundByTupleIds(Lists.newArrayList(tupleId2, tupleId1)); + result = true; + eqBinaryPredicate2.getChild(0); + result = eqT2Slot2; + eqBinaryPredicate2.getChild(1); + result = eqT3Slot3; + eqT3Slot3.isBoundByTupleIds(tupleIds3); + result = true; + eqBinaryPredicate4.getChild(0); + result = eqT3Slot3; + eqBinaryPredicate4.getChild(1); + result = eqT5Slot5; + eqT3Slot3.isBoundByTupleIds(tupleIds213); + result = true; + eqT5Slot5.isBoundByTupleIds(Lists.newArrayList(tupleId5)); + result = true; + eqBinaryPredicate5.getChild(0); + result = eqT3Slot3; + eqBinaryPredicate5.getChild(1); + result = eqT4Slot4; + eqT4Slot4.isBoundByTupleIds(Lists.newArrayList(tupleId4)); + result = true; + scanNode1.getTupleIds(); + result = tupleIds1; + scanNode2.getTupleIds(); + result = tupleIds2; + scanNode3.getTupleIds(); + result = tupleIds3; + scanNode4.getTupleIds(); + result = tupleIds4; + scanNode5.getTupleIds(); + result = tupleIds5; + scanNode6.getTupleIds(); + result = tupleIds6; + scanNode7.getTupleIds(); + result = tupleIds7; + scanNode1.getOutputSmap(); + result = null; + scanNode2.getOutputSmap(); + result = null; + scanNode3.getOutputSmap(); + result = null; + scanNode4.getOutputSmap(); + result = null; + scanNode5.getOutputSmap(); + result = null; + scanNode6.getOutputSmap(); + result = null; + scanNode7.getOutputSmap(); + result = null; + tableRef1.getUniqueAlias(); + result = "t1"; + tableRef2.getUniqueAlias(); + result = "t2"; + tableRef3.getUniqueAlias(); + result = "t3"; + tableRef4.getUniqueAlias(); + result = "t4"; + tableRef5.getUniqueAlias(); + result = "t5"; + tableRef6.getUniqueAlias(); + result = "t6"; + tableRef7.getUniqueAlias(); + result = "t7"; + tableRef1.getJoinOp(); + result = JoinOperator.INNER_JOIN; + tableRef2.getJoinOp(); + result = JoinOperator.INNER_JOIN; + tableRef3.getJoinOp(); + result = JoinOperator.RIGHT_OUTER_JOIN; + tableRef4.getJoinOp(); + result = JoinOperator.INNER_JOIN; + tableRef5.getJoinOp(); + result = JoinOperator.INNER_JOIN; + tableRef6.getJoinOp(); + result = JoinOperator.LEFT_OUTER_JOIN; + tableRef7.getJoinOp(); + result = JoinOperator.INNER_JOIN; + } + }; + new MockUp() { + @Mock + public ExprSubstitutionMap compose(ExprSubstitutionMap f, ExprSubstitutionMap g, + Analyzer analyzer) { + return exprSubstitutionMap; + } + + @Mock + public ExprSubstitutionMap combine(ExprSubstitutionMap f, ExprSubstitutionMap g) { + return exprSubstitutionMap; + } + }; + + SingleNodePlanner singleNodePlanner = new SingleNodePlanner(context); + PlanNode cheapestJoinNode = Deencapsulation.invoke(singleNodePlanner, "createCheapestJoinPlan", analyzer, refPlans); + Assert.assertEquals(2, cheapestJoinNode.getChildren().size()); + Assert.assertEquals(Lists.newArrayList(tupleId2, tupleId1, tupleId3, tupleId4, tupleId5, tupleId6, tupleId7), + cheapestJoinNode.getTupleIds()); + } + + /* + Query: select * from t1, t3, t2, t4 where (all of inner join condition) + Original Query: select * from test1, test3, test2, test4 + where test1.k1=test3.k1 and test3.k2=test2.k2 and test2.k3=test4.k3; + Expect: t4(the largest), t2, t3, t1 (without cross join) + Round1: (t4 cross t1) pk (t4 cross t3) pk (t4 inner t2) => t4, t2 + Round2: ([t4,t2] cross t1) pk ([t4,t2] inner t3) => t4, t2, t3 + Round3: t4, t2, t3, t1 without pk + */ + @Test + public void testMultiInnerJoinReorderAvoidCrossJoin(@Injectable PlannerContext context, + @Injectable Analyzer analyzer, + @Injectable BaseTableRef tableRef1, @Injectable OlapScanNode scanNode1, + @Injectable BaseTableRef tableRef2, @Injectable OlapScanNode scanNode2, + @Injectable BaseTableRef tableRef3, @Injectable OlapScanNode scanNode3, + @Injectable BaseTableRef tableRef4, @Injectable OlapScanNode scanNode4, + @Injectable TupleDescriptor tupleDescriptor1, + @Injectable TupleDescriptor tupleDescriptor2, + @Injectable TupleDescriptor tupleDescriptor3, + @Injectable SlotDescriptor slotDescriptor1, + @Injectable SlotDescriptor slotDescriptor2, + @Injectable SlotDescriptor slotDescriptor3, + @Injectable BinaryPredicate eqBinaryPredicate3, + @Injectable BinaryPredicate eqBinaryPredicate5, + @Injectable BinaryPredicate eqBinaryPredicate6, + @Injectable SlotRef eqT1Slot1, + @Injectable SlotRef eqT2Slot2, + @Injectable SlotRef eqT3Slot3, + @Injectable SlotRef eqT4Slot4, + @Tested ExprSubstitutionMap exprSubstitutionMap) { + Pair pair1 = new Pair<>(tableRef1, scanNode1); + Pair pair2 = new Pair<>(tableRef2, scanNode2); + Pair pair3 = new Pair<>(tableRef3, scanNode3); + Pair pair4 = new Pair<>(tableRef4, scanNode4); + List> refPlans = Lists.newArrayList(); + refPlans.add(pair3); + refPlans.add(pair2); + refPlans.add(pair1); + refPlans.add(pair4); + + TupleId tupleId1 = new TupleId(1); + TupleId tupleId2 = new TupleId(2); + TupleId tupleId3 = new TupleId(3); + TupleId tupleId4 = new TupleId(4); + List tupleIds1 = Lists.newArrayList(tupleId1); + List tupleIds2 = Lists.newArrayList(tupleId2); + List tupleIds3 = Lists.newArrayList(tupleId3); + List tupleIds4 = Lists.newArrayList(tupleId4); + List tupleIds41 = Lists.newArrayList(tupleId4, tupleId1); + List tupleIds412 = Lists.newArrayList(tupleId4, tupleId1, tupleId2); + + new Expectations() { + { + tableRef1.isAnalyzed(); + result = true; + tableRef2.isAnalyzed(); + result = true; + tableRef3.isAnalyzed(); + result = true; + tableRef4.isAnalyzed(); + result = true; + scanNode1.getCardinality(); + result = 1; + scanNode2.getCardinality(); + result = 2; + scanNode3.getCardinality(); + result = 3; + scanNode4.getCardinality(); + result = 4; + tableRef1.getDesc(); + result = tupleDescriptor1; + tupleDescriptor1.getMaterializedSlots(); + result = Lists.newArrayList(slotDescriptor1); + tableRef2.getDesc(); + result = tupleDescriptor2; + tupleDescriptor2.getMaterializedSlots(); + result = Lists.newArrayList(slotDescriptor2); + tableRef3.getDesc(); + result = tupleDescriptor3; + tupleDescriptor3.getMaterializedSlots(); + result = Lists.newArrayList(slotDescriptor3); + + // where t4.k1=t1.k1 + analyzer.getEqJoinConjuncts(tupleIds4, tupleIds1); + result = Lists.newArrayList(eqBinaryPredicate3); + eqBinaryPredicate3.getChild(0); + result = eqT4Slot4; + eqT4Slot4.isBoundByTupleIds(tupleIds4); + result = true; + eqBinaryPredicate3.getChild(1); + result = eqT1Slot1; + eqT1Slot1.isBoundByTupleIds(tupleIds1); + result = true; + // where t1.k1=t2.k1 + analyzer.getEqJoinConjuncts(tupleIds41, tupleIds2); + result = Lists.newArrayList(eqBinaryPredicate5); + eqBinaryPredicate5.getChild(0); + result = eqT1Slot1; + eqT1Slot1.isBoundByTupleIds(tupleIds41); + result = true; + eqBinaryPredicate5.getChild(1); + result = eqT2Slot2; + eqT2Slot2.isBoundByTupleIds(tupleIds2); + result = true; + // where t2.k1 = t3.k1 + analyzer.getEqJoinConjuncts(tupleIds412, tupleIds3); + result = Lists.newArrayList(eqBinaryPredicate6); + eqBinaryPredicate6.getChild(0); + result = eqT2Slot2; + eqT2Slot2.isBoundByTupleIds(tupleIds412); + result = true; + eqBinaryPredicate6.getChild(1); + result = eqT3Slot3; + eqT3Slot3.isBoundByTupleIds(tupleIds3); + result = true; + + scanNode1.getTblRefIds(); + result = tupleIds1; + scanNode2.getTblRefIds(); + result = tupleIds2; + scanNode3.getTblRefIds(); + result = tupleIds3; + scanNode4.getTblRefIds(); + result = tupleIds4; + + scanNode1.getTupleIds(); + result = tupleIds1; + scanNode2.getTupleIds(); + result = tupleIds2; + scanNode3.getTupleIds(); + result = tupleIds3; + scanNode4.getTupleIds(); + result = tupleIds4; + scanNode1.getOutputSmap(); + result = null; + scanNode2.getOutputSmap(); + result = null; + scanNode3.getOutputSmap(); + result = null; + scanNode4.getOutputSmap(); + result = null; + tableRef1.getUniqueAlias(); + result = "t1"; + tableRef2.getUniqueAlias(); + result = "t2"; + tableRef3.getUniqueAlias(); + result = "t3"; + tableRef4.getUniqueAlias(); + result = "t4"; + tableRef1.getJoinOp(); + result = JoinOperator.INNER_JOIN; + tableRef2.getJoinOp(); + result = JoinOperator.INNER_JOIN; + tableRef3.getJoinOp(); + result = JoinOperator.INNER_JOIN; + tableRef4.getJoinOp(); + result = JoinOperator.INNER_JOIN; + } + }; + new MockUp() { + @Mock + public ExprSubstitutionMap compose(ExprSubstitutionMap f, ExprSubstitutionMap g, + Analyzer analyzer) { + return exprSubstitutionMap; + } + + @Mock + public ExprSubstitutionMap combine(ExprSubstitutionMap f, ExprSubstitutionMap g) { + return exprSubstitutionMap; + } + }; + + SingleNodePlanner singleNodePlanner = new SingleNodePlanner(context); + PlanNode cheapestJoinNode = Deencapsulation.invoke(singleNodePlanner, "createCheapestJoinPlan", analyzer, refPlans); + Assert.assertEquals(2, cheapestJoinNode.getChildren().size()); + Assert.assertEquals(Lists.newArrayList(tupleId4, tupleId1, tupleId2, tupleId3), + cheapestJoinNode.getTupleIds()); + } + + /* + Query: select * from t3, t2, t1, t4 where (multi inner join condition) + Original Query: select * from test3, test2, test1, test4 + where test3.k1=test2.k1 and test2.k1=test1.k1 and test1.k1=test4.k1 and test4.k2=test2.k2 + and test4.k2=test3.k2 and test3.k3=test1.k3; + Expect: same as above + */ + @Test + public void testMultiInnerJoinMultiJoinPredicateReorder(@Injectable PlannerContext context, + @Injectable Analyzer analyzer, + @Injectable BaseTableRef tableRef1, @Injectable OlapScanNode scanNode1, + @Injectable BaseTableRef tableRef2, @Injectable OlapScanNode scanNode2, + @Injectable BaseTableRef tableRef3, @Injectable OlapScanNode scanNode3, + @Injectable BaseTableRef tableRef4, @Injectable OlapScanNode scanNode4, + @Injectable TupleDescriptor tupleDescriptor1, + @Injectable TupleDescriptor tupleDescriptor2, + @Injectable TupleDescriptor tupleDescriptor3, + @Injectable TupleDescriptor tupleDescriptor4, + @Injectable SlotDescriptor slotDescriptor1, + @Injectable SlotDescriptor slotDescriptor2, + @Injectable SlotDescriptor slotDescriptor3, + @Injectable SlotDescriptor slotDescriptor4, + @Injectable BinaryPredicate eqBinaryPredicate1, + @Injectable BinaryPredicate eqBinaryPredicate2, + @Injectable BinaryPredicate eqBinaryPredicate3, + @Injectable BinaryPredicate eqBinaryPredicate4, + @Injectable BinaryPredicate eqBinaryPredicate5, + @Injectable BinaryPredicate eqBinaryPredicate6, + @Injectable SlotRef eqT1Slot1, + @Injectable SlotRef eqT2Slot2, + @Injectable SlotRef eqT3Slot3, + @Injectable SlotRef eqT4Slot4, + @Tested ExprSubstitutionMap exprSubstitutionMap) { + Pair pair1 = new Pair<>(tableRef1, scanNode1); + Pair pair2 = new Pair<>(tableRef2, scanNode2); + Pair pair3 = new Pair<>(tableRef3, scanNode3); + Pair pair4 = new Pair<>(tableRef4, scanNode4); + List> refPlans = Lists.newArrayList(); + refPlans.add(pair3); + refPlans.add(pair2); + refPlans.add(pair1); + refPlans.add(pair4); + + TupleId tupleId1 = new TupleId(1); + TupleId tupleId2 = new TupleId(2); + TupleId tupleId3 = new TupleId(3); + TupleId tupleId4 = new TupleId(4); + List tupleIds1 = Lists.newArrayList(tupleId1); + List tupleIds2 = Lists.newArrayList(tupleId2); + List tupleIds3 = Lists.newArrayList(tupleId3); + List tupleIds4 = Lists.newArrayList(tupleId4); + List tupleIds41 = Lists.newArrayList(tupleId4, tupleId1); + List tupleIds412 = Lists.newArrayList(tupleId4, tupleId1, tupleId2); + + new Expectations() { + { + tableRef1.isAnalyzed(); + result = true; + tableRef2.isAnalyzed(); + result = true; + tableRef3.isAnalyzed(); + result = true; + tableRef4.isAnalyzed(); + result = true; + scanNode1.getCardinality(); + result = 1; + scanNode2.getCardinality(); + result = 2; + scanNode3.getCardinality(); + result = 3; + scanNode4.getCardinality(); + result = 4; + tableRef1.getDesc(); + result = tupleDescriptor1; + tupleDescriptor1.getMaterializedSlots(); + result = Lists.newArrayList(slotDescriptor1); + tableRef2.getDesc(); + result = tupleDescriptor2; + tupleDescriptor2.getMaterializedSlots(); + result = Lists.newArrayList(slotDescriptor2); + tableRef3.getDesc(); + result = tupleDescriptor3; + tupleDescriptor3.getMaterializedSlots(); + result = Lists.newArrayList(slotDescriptor3); + + // where t4.k1=t3.k1 + analyzer.getEqJoinConjuncts(tupleIds4, tupleIds3); + result = Lists.newArrayList(eqBinaryPredicate1); + eqBinaryPredicate1.getChild(0); + result = eqT4Slot4; + eqT4Slot4.isBoundByTupleIds(tupleIds4); + result = true; + eqBinaryPredicate1.getChild(1); + result = eqT3Slot3; + eqT3Slot3.isBoundByTupleIds(tupleIds3); + result = true; + // where t4.k1 = t2.k1 + analyzer.getEqJoinConjuncts(tupleIds4, tupleIds2); + result = Lists.newArrayList(eqBinaryPredicate2); + eqBinaryPredicate2.getChild(0); + result = eqT4Slot4; + eqBinaryPredicate2.getChild(1); + result = eqT2Slot2; + eqT2Slot2.isBoundByTupleIds(tupleIds2); + result = true; + // where t4.k1=t1.k1 + analyzer.getEqJoinConjuncts(tupleIds4, tupleIds1); + result = Lists.newArrayList(eqBinaryPredicate3); + eqBinaryPredicate3.getChild(0); + result = eqT4Slot4; + eqBinaryPredicate3.getChild(1); + result = eqT1Slot1; + eqT1Slot1.isBoundByTupleIds(tupleIds1); + result = true; + // where t1.k1=t3.k1 + analyzer.getEqJoinConjuncts(tupleIds41, tupleIds3); + result = Lists.newArrayList(eqBinaryPredicate4); + eqBinaryPredicate4.getChild(0); + result = eqT1Slot1; + eqT1Slot1.isBoundByTupleIds(tupleIds41); + result = true; + eqBinaryPredicate4.getChild(1); + result = eqT3Slot3; + // where t1.k1=t2.k1 + analyzer.getEqJoinConjuncts(tupleIds41, tupleIds2); + result = Lists.newArrayList(eqBinaryPredicate5); + eqBinaryPredicate5.getChild(0); + result = eqT1Slot1; + eqT1Slot1.isBoundByTupleIds(tupleIds41); + result = true; + eqBinaryPredicate5.getChild(1); + result = eqT2Slot2; + eqT2Slot2.isBoundByTupleIds(tupleIds2); + result = true; + // where t2.k1 = t3.k1 + analyzer.getEqJoinConjuncts(tupleIds412, tupleIds3); + result = Lists.newArrayList(eqBinaryPredicate6); + eqBinaryPredicate6.getChild(0); + result = eqT2Slot2; + eqT2Slot2.isBoundByTupleIds(tupleIds412); + result = true; + eqBinaryPredicate6.getChild(1); + result = eqT3Slot3; + eqT3Slot3.isBoundByTupleIds(tupleIds3); + result = true; + + scanNode1.getTblRefIds(); + result = tupleIds1; + scanNode2.getTblRefIds(); + result = tupleIds2; + scanNode3.getTblRefIds(); + result = tupleIds3; + scanNode4.getTblRefIds(); + result = tupleIds4; + + scanNode1.getTupleIds(); + result = tupleIds1; + scanNode2.getTupleIds(); + result = tupleIds2; + scanNode3.getTupleIds(); + result = tupleIds3; + scanNode4.getTupleIds(); + result = tupleIds4; + scanNode1.getOutputSmap(); + result = null; + scanNode2.getOutputSmap(); + result = null; + scanNode3.getOutputSmap(); + result = null; + scanNode4.getOutputSmap(); + result = null; + tableRef1.getUniqueAlias(); + result = "t1"; + tableRef2.getUniqueAlias(); + result = "t2"; + tableRef3.getUniqueAlias(); + result = "t3"; + tableRef4.getUniqueAlias(); + result = "t4"; + tableRef1.getJoinOp(); + result = JoinOperator.INNER_JOIN; + tableRef2.getJoinOp(); + result = JoinOperator.INNER_JOIN; + tableRef3.getJoinOp(); + result = JoinOperator.INNER_JOIN; + tableRef4.getJoinOp(); + result = JoinOperator.INNER_JOIN; + } + }; + new MockUp() { + @Mock + public ExprSubstitutionMap compose(ExprSubstitutionMap f, ExprSubstitutionMap g, + Analyzer analyzer) { + return exprSubstitutionMap; + } + + @Mock + public ExprSubstitutionMap combine(ExprSubstitutionMap f, ExprSubstitutionMap g) { + return exprSubstitutionMap; + } + }; + + SingleNodePlanner singleNodePlanner = new SingleNodePlanner(context); + PlanNode cheapestJoinNode = Deencapsulation.invoke(singleNodePlanner, "createCheapestJoinPlan", analyzer, refPlans); + Assert.assertEquals(2, cheapestJoinNode.getChildren().size()); + Assert.assertEquals(Lists.newArrayList(tupleId4, tupleId1, tupleId2, tupleId3), + cheapestJoinNode.getTupleIds()); + } + + /* + Query: select * from t3, t2, t1, t4 where (there is no predicate related to t2) + Expect: t4(the largest), t1, t3, t2 + Round1: (t4,t3) pk (t4,t2) pk (t4,t1) => t4,t1 + Round2: ([t4,t1],t3) pk ([t4,t1],t2) => t4,t1,t3 + Round3: t4,t1,t3,t2 without pk + */ + @Test + public void testInnerPriorToCrossJoinReorder() { + + } + + /* + Query: select * from t3, t2, t1, t4 + Original Query: select * from test3, test1, test2, test4; + Expect: t4(the largest), t1, t2, t3 (from the smallest to the second largest) + */ + @Test + public void testMultiCrossJoinReorder() { + + } + + /* + Test explicit cross join + Query: select * from t3, t2, t1, t4 where ('>', '<' etc predicates) + Original Query: select * from test3,test2,test1,test4 + where test3.k1>test2.k1 and test2.k2=test1.k3; + Expect: need same as implicit cross join + */ + @Test + public void testExplicitCrossJoinReorder() { + } } diff --git a/fe/fe-core/src/test/resources/log4j2.xml b/fe/fe-core/src/test/resources/log4j2.xml index 7e0957b1a926d0..d0e6cbaced4356 100644 --- a/fe/fe-core/src/test/resources/log4j2.xml +++ b/fe/fe-core/src/test/resources/log4j2.xml @@ -16,7 +16,7 @@ - +