[JoinReorder] Implement a better join reorder algorithm

fe/fe-core/src/main/cup/sql_parser.cup

@@ -2733,6 +2733,10 @@ opt_alter_type ::=
     {:
         RESULT = ShowAlterStmt.AlterType.ROLLUP;
     :}
+    | KW_MATERIALIZED KW_VIEW
+    {:
+        RESULT =  ShowAlterStmt.AlterType.ROLLUP;
+    :}
     | KW_COLUMN
     {:
         RESULT = ShowAlterStmt.AlterType.COLUMN;

fe/fe-core/src/main/java/org/apache/doris/analysis/Analyzer.java

@@ -152,7 +152,7 @@ public class Analyzer {
 
     // The runtime filter that is expected to be used
     private final List<RuntimeFilter> assignedRuntimeFilters = new ArrayList<>();
-    
+
     public void setIsSubquery() {
         isSubquery = true;
         globalState.containsSubquery = true;
@@ -206,8 +206,8 @@ private static class GlobalState {
         private final Map<TupleId, List<ExprId>> eqJoinConjuncts = Maps.newHashMap();
 
         // set of conjuncts that have been assigned to some PlanNode
-        private final Set<ExprId> assignedConjuncts =
-            Collections.newSetFromMap(new IdentityHashMap<ExprId, Boolean>());
+        private Set<ExprId> assignedConjuncts =
+                Collections.newSetFromMap(new IdentityHashMap<ExprId, Boolean>());
 
         // map from outer-joined tuple id, ie, one that is nullable in this select block,
         // to the last Join clause (represented by its rhs table ref) that outer-joined it
@@ -849,6 +849,15 @@ private void registerConstantConjunct(TupleId id, Expr e) {
         }
     }
 
+    /**
+     * register expr id
+     *
+     * @param expr
+     */
+    void registerExprId(Expr expr) {
+        expr.setId(globalState.conjunctIdGenerator.getNextId());
+    }
+
     /**
      * Register individual conjunct with all tuple and slot ids it references
      * and with the global conjunct list.
@@ -945,14 +954,24 @@ public void createAuxEquivPredicate(Expr lhs, Expr rhs) {
         registerConjunct(p);
     }
 
+    public Set<ExprId> getAssignedConjuncts() {
+        return Sets.newHashSet(globalState.assignedConjuncts);
+    }
+
+    public void setAssignedConjuncts(Set<ExprId> assigned) {
+        if (assigned != null) {
+            globalState.assignedConjuncts = Sets.newHashSet(assigned);
+        }
+    }
+
     /**
      * Return all unassigned registered conjuncts that are fully bound by the given
      * (logical) tuple ids, can be evaluated by 'tupleIds' and are not tied to an
      * Outer Join clause.
      */
     public List<Expr> getUnassignedConjuncts(List<TupleId> tupleIds) {
         List<Expr> result = Lists.newArrayList();
-        for (Expr e: getUnassignedConjuncts(tupleIds, true)) {
+        for (Expr e : getUnassignedConjuncts(tupleIds, true)) {
             if (canEvalPredicate(tupleIds, e)) result.add(e);
         }
         return result;
@@ -1256,7 +1275,7 @@ private void markConstantConjunct(Expr conjunct, boolean fromHavingClause)
                 }
                 final Expr newConjunct = conjunct.getResultValue();
                 if (newConjunct instanceof BoolLiteral) {
-                    final BoolLiteral value = (BoolLiteral)newConjunct;
+                    final BoolLiteral value = (BoolLiteral) newConjunct;
                     if (!value.getValue()) {
                         if (fromHavingClause) {
                             hasEmptyResultSet_ = true;
@@ -1597,6 +1616,17 @@ public ExprSubstitutionMap getChangeResSmap() {
     public void setChangeResSmap(ExprSubstitutionMap changeResSmap) {
         this.changeResSmap = changeResSmap;
     }
+
+    // Load plan and query plan are the same framework
+    // Some Load method in doris access through http protocol, which will cause the session may be empty.
+    // In order to avoid the occurrence of null pointer exceptions, a check will be added here
+    public boolean safeIsEnableJoinReorderBasedCost() {
+        if (globalState.context == null) {
+            return false;
+        }
+        return globalState.context.getSessionVariable().isEnableJoinReorderBasedCost();
+    }
+
     /**
      * Returns true if predicate 'e' can be correctly evaluated by a tree materializing
      * 'tupleIds', otherwise false:
@@ -1783,7 +1813,7 @@ public boolean containSubquery() {
      * materialization decision be cost-based?
      */
     public void markRefdSlots(Analyzer analyzer, PlanNode planRoot,
-                               List<Expr> outputExprs, AnalyticInfo analyticInfo) {
+                              List<Expr> outputExprs, AnalyticInfo analyticInfo) {
         if (planRoot == null) {
             return;
         }
@@ -1824,7 +1854,7 @@ public void markRefdSlots(Analyzer analyzer, PlanNode planRoot,
 
     /**
      * Column conduction, can slot a value-transfer to slot b
-     *
+     * <p>
      * TODO(zxy) Use value-transfer graph to check
      */
     public boolean hasValueTransfer(SlotId a, SlotId b) {
@@ -1834,7 +1864,7 @@ public boolean hasValueTransfer(SlotId a, SlotId b) {
     /**
      * Returns sorted slot IDs with value transfers from 'srcSid'.
      * Time complexity: O(V) where V = number of slots
-     *
+     * <p>
      * TODO(zxy) Use value-transfer graph to check
      */
     public List<SlotId> getValueTransferTargets(SlotId srcSid) {
@@ -1848,8 +1878,8 @@ public List<SlotId> getValueTransferTargets(SlotId srcSid) {
      * to an outer-joined tuple.
      */
     public boolean hasOuterJoinedValueTransferTarget(List<SlotId> sids) {
-        for (SlotId srcSid: sids) {
-            for (SlotId dstSid: getValueTransferTargets(srcSid)) {
+        for (SlotId srcSid : sids) {
+            for (SlotId dstSid : getValueTransferTargets(srcSid)) {
                 if (isOuterJoined(getTupleId(dstSid))) return true;
             }
         }

fe/fe-core/src/main/java/org/apache/doris/analysis/BinaryPredicate.java

@@ -622,6 +622,30 @@ private Expr compareLiteral(LiteralExpr first, LiteralExpr second) throws Analys
         return this;
     }
 
+    @Override
+    public void setSelectivity() {
+        switch(op) {
+            case EQ:
+            case EQ_FOR_NULL: {
+                Reference<SlotRef> slotRefRef = new Reference<SlotRef>();
+                boolean singlePredicate = isSingleColumnPredicate(slotRefRef, null);
+                if (singlePredicate) {
+                    long distinctValues = slotRefRef.getRef().getNumDistinctValues();
+                    if (distinctValues != -1) {
+                        selectivity = 1.0 / distinctValues;
+                    }
+                }
+                break;
+            } default: {
+                // Reference hive
+                selectivity = 1.0 / 3.0;
+                break;
+            }
+        }
+
+        return;
+    }
+
     @Override
     public int hashCode() {
         return 31 * super.hashCode() + Objects.hashCode(op);

fe/fe-core/src/main/java/org/apache/doris/analysis/DescriptorTable.java

@@ -83,7 +83,7 @@ public TupleDescriptor copyTupleDescriptor(TupleId srcId, String debugName) {
         for (SlotDescriptor slot: src.getSlots()) {
             copySlotDescriptor(d, slot);
         }
-        d.computeMemLayout();
+        d.computeStatAndMemLayout();
         return d;
     }
 
@@ -122,14 +122,21 @@ public void markSlotsMaterialized(List<SlotId> ids) {
         }
     }
 
-    // Computes physical layout parameters of all descriptors.
-    // Call this only after the last descriptor was added.
+    @Deprecated
     public void computeMemLayout() {
         for (TupleDescriptor d : tupleDescs.values()) {
             d.computeMemLayout();
         }
     }
 
+    // Computes physical layout parameters of all descriptors and calculate the statistics of the tuple.
+    // Call this only after the last descriptor was added.
+    public void computeStatAndMemLayout() {
+        for (TupleDescriptor d : tupleDescs.values()) {
+            d.computeStatAndMemLayout();
+        }
+    }
+
     public TDescriptorTable toThrift() {
         TDescriptorTable result = new TDescriptorTable();
         HashSet<Table> referencedTbls = Sets.newHashSet();

fe/fe-core/src/main/java/org/apache/doris/analysis/Expr.java

@@ -63,7 +63,7 @@ abstract public class Expr extends TreeNode<Expr> implements ParseNode, Cloneabl
     private static final String NEGATE_FN = "negate";
 
     // to be used where we can't come up with a better estimate
-    protected static final double DEFAULT_SELECTIVITY = 0.1;
+    public static final double DEFAULT_SELECTIVITY = 0.1;
 
     public final static float FUNCTION_CALL_COST = 10;
 
@@ -180,6 +180,10 @@ public boolean apply(Expr arg) {
                 public boolean apply(Expr arg) { return arg instanceof NullLiteral; }
             };
 
+    public void setSelectivity() {
+        selectivity = -1;
+    }
+
     /* TODO(zc)
     public final static com.google.common.base.Predicate<Expr>
             IS_NONDETERMINISTIC_BUILTIN_FN_PREDICATE =
@@ -299,6 +303,8 @@ public double getSelectivity() {
         return selectivity;
     }
 
+    public boolean hasSelectivity() { return selectivity >= 0; }
+
     public long getNumDistinctValues() {
         return numDistinctValues;
     }
@@ -374,6 +380,9 @@ public final void analyze(Analyzer analyzer) throws AnalysisException {
 
         // Do all the analysis for the expr subclass before marking the Expr analyzed.
         analyzeImpl(analyzer);
+        if (analyzer.safeIsEnableJoinReorderBasedCost()) {
+            setSelectivity();
+        }
         analysisDone();
     }
 
@@ -1420,6 +1429,29 @@ public Expr unwrapExpr(boolean implicitOnly) {
         return this;
     }
 
+    /**
+     * Returns the descriptor of the scan slot that directly or indirectly produces
+     * the values of 'this' SlotRef. Traverses the source exprs of intermediate slot
+     * descriptors to resolve materialization points (e.g., aggregations).
+     * Returns null if 'e' or any source expr of 'e' is not a SlotRef or cast SlotRef.
+     */
+    public SlotDescriptor findSrcScanSlot() {
+        SlotRef slotRef = unwrapSlotRef(false);
+        if (slotRef == null) {
+            return null;
+        }
+        SlotDescriptor slotDesc = slotRef.getDesc();
+        if (slotDesc.isScanSlot()) {
+            return slotDesc;
+        }
+        if (slotDesc.getSourceExprs().size() == 1) {
+            return slotDesc.getSourceExprs().get(0).findSrcScanSlot();
+        }
+        // No known source expr, or there are several source exprs meaning the slot is
+        // has no single source table.
+        return null;
+    }
+
     public static double getConstFromExpr(Expr e) throws AnalysisException{
         Preconditions.checkState(e.isConstant());
         double value = 0;

fe/fe-core/src/main/java/org/apache/doris/analysis/FromClause.java

@@ -17,11 +17,6 @@
 
 package org.apache.doris.analysis;
 
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.Iterator;
-import java.util.List;
 
 import org.apache.doris.catalog.Database;
 import org.apache.doris.catalog.Table;
@@ -31,10 +26,16 @@
 import org.apache.doris.common.ErrorReport;
 import org.apache.doris.common.UserException;
 
-import com.google.common.base.Strings;
 import com.google.common.base.Preconditions;
+import com.google.common.base.Strings;
 import com.google.common.collect.Lists;
 
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.Iterator;
+import java.util.List;
+
 /**
  * Wraps a list of TableRef instances that form a FROM clause, allowing them to be
  * analyzed independently of the statement using them. To increase the flexibility of
@@ -61,23 +62,6 @@ public void setNeedToSql(boolean needToSql) {
         this.needToSql = needToSql;
     }
 
-    private void sortTableRefForSubquery(Analyzer analyzer) {
-        Collections.sort(this.tableRefs_, new Comparator<TableRef>() {
-            @Override
-            public int compare(TableRef tableref1, TableRef tableref2) {
-                int i1 = 0;
-                int i2 = 0;
-                if (tableref1.getOnClause() != null) {
-                    i1 = 1;
-                }
-                if (tableref2.getOnClause() != null) {
-                    i2 = 1;
-                }
-                return i1 - i2;
-            }
-        });
-    }
-
     private void checkFromHiveTable(Analyzer analyzer) throws AnalysisException {
         for (TableRef tblRef : tableRefs_) {
             if (!(tblRef instanceof BaseTableRef)) {
@@ -111,6 +95,33 @@ private void checkFromHiveTable(Analyzer analyzer) throws AnalysisException {
         }
     }
 
+    /**
+     * In some cases, the reorder method of select stmt will incorrectly sort the tableRef with on clause.
+     * The meaning of this function is to reset those tableRefs with on clauses.
+     * For example:
+     * Origin stmt: select * from t1 inner join t2 on t1.k1=t2.k1
+     * After analyze: select * from t2 on t1.k1=t2.k1 inner join t1
+     *
+     * If this statement just needs to be reanalyze (query rewriter), an error will be reported
+     * because the table t1 in the on clause cannot be recognized.
+     */
+    private void sortTableRefKeepSequenceOfOnClause() {
+        Collections.sort(this.tableRefs_, new Comparator<TableRef>() {
+            @Override
+            public int compare(TableRef tableref1, TableRef tableref2) {
+                int i1 = 0;
+                int i2 = 0;
+                if (tableref1.getOnClause() != null) {
+                    i1 = 1;
+                }
+                if (tableref2.getOnClause() != null) {
+                    i2 = 1;
+                }
+                return i1 - i2;
+            }
+        });
+    }
+
     @Override
     public void analyze(Analyzer analyzer) throws AnalysisException, UserException {
         if (analyzed_) return;
@@ -120,7 +131,14 @@ public void analyze(Analyzer analyzer) throws AnalysisException, UserException {
             return;
         }
 
-        sortTableRefForSubquery(analyzer);
+        // The order of the tables may have changed during the previous analyzer process.
+        // For example, a join b on xxx is changed to b on xxx join a.
+        // This change will cause the predicate in on clause be adjusted to the front of the association table,
+        // causing semantic analysis to fail. Unknown column 'column1' in 'table1'
+        // So we need to readjust the order of the tables here.
+        if (!analyzer.safeIsEnableJoinReorderBasedCost()) {
+            sortTableRefKeepSequenceOfOnClause();
+        }
 
         // Start out with table refs to establish aliases.
         TableRef leftTblRef = null;  // the one to the left of tblRef

fe/fe-core/src/main/java/org/apache/doris/analysis/SelectStmt.java

@@ -515,7 +515,10 @@ public void analyze(Analyzer analyzer) throws AnalysisException, UserException {
         if (needToSql) {
             sqlString_ = toSql();
         }
-        reorderTable(analyzer);
+        if (!analyzer.safeIsEnableJoinReorderBasedCost()) {
+            LOG.debug("use old reorder logical in select stmt");
+            reorderTable(analyzer);
+        }
 
         resolveInlineViewRefs(analyzer);