[Feat](Nereids) support use cbo rule hint #35925 #39715 #40167#40958

fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java

@@ -51,6 +51,8 @@
 import org.apache.doris.mtmv.MTMVRelatedTableIf;
 import org.apache.doris.mtmv.MTMVSnapshotIf;
 import org.apache.doris.mtmv.MTMVVersionSnapshot;
+import org.apache.doris.nereids.hint.Hint;
+import org.apache.doris.nereids.hint.UseMvHint;
 import org.apache.doris.qe.ConnectContext;
 import org.apache.doris.qe.OriginStatement;
 import org.apache.doris.resource.Tag;
@@ -513,6 +515,99 @@ public Map<Long, MaterializedIndexMeta> getVisibleIndexIdToMeta() {
         return visibleMVs;
     }
 
+    public Long getBestMvIdWithHint(List<Long> orderedMvs) {
+        Optional<UseMvHint> useMvHint = getUseMvHint("USE_MV");
+        Optional<UseMvHint> noUseMvHint = getUseMvHint("NO_USE_MV");
+        if (useMvHint.isPresent() && noUseMvHint.isPresent()) {
+            if (noUseMvHint.get().getNoUseMVName(this.name).contains(useMvHint.get().getUseMvName(this.name))) {
+                String errorMsg = "conflict mv exist in use_mv and no_use_mv in the same time"
+                        + useMvHint.get().getUseMvName(this.name);
+                useMvHint.get().setStatus(Hint.HintStatus.SYNTAX_ERROR);
+                useMvHint.get().setErrorMessage(errorMsg);
+                noUseMvHint.get().setStatus(Hint.HintStatus.SYNTAX_ERROR);
+                noUseMvHint.get().setErrorMessage(errorMsg);
+            }
+            return getMvIdWithUseMvHint(useMvHint.get(), orderedMvs);
+        } else if (useMvHint.isPresent()) {
+            return getMvIdWithUseMvHint(useMvHint.get(), orderedMvs);
+        } else if (noUseMvHint.isPresent()) {
+            return getMvIdWithNoUseMvHint(noUseMvHint.get(), orderedMvs);
+        }
+        return orderedMvs.get(0);
+    }
+
+    private Long getMvIdWithUseMvHint(UseMvHint useMvHint, List<Long> orderedMvs) {
+        if (useMvHint.isAllMv()) {
+            useMvHint.setStatus(Hint.HintStatus.SYNTAX_ERROR);
+            useMvHint.setErrorMessage("use_mv hint should only have one mv in one table: "
+                    + this.name);
+            return orderedMvs.get(0);
+        } else {
+            String mvName = useMvHint.getUseMvName(this.name);
+            if (mvName != null) {
+                if (mvName.equals("`*`")) {
+                    useMvHint.setStatus(Hint.HintStatus.SYNTAX_ERROR);
+                    useMvHint.setErrorMessage("use_mv hint should only have one mv in one table: "
+                            + this.name);
+                    return orderedMvs.get(0);
+                }
+                Long choosedIndexId = indexNameToId.get(mvName);
+                if (orderedMvs.contains(choosedIndexId)) {
+                    useMvHint.setStatus(Hint.HintStatus.SUCCESS);
+                    return choosedIndexId;
+                } else {
+                    useMvHint.setStatus(Hint.HintStatus.SYNTAX_ERROR);
+                    useMvHint.setErrorMessage("do not have mv: " + mvName + " in table: " + this.name);
+                }
+            }
+        }
+        return orderedMvs.get(0);
+    }
+
+    private Long getMvIdWithNoUseMvHint(UseMvHint noUseMvHint, List<Long> orderedMvs) {
+        if (noUseMvHint.isAllMv()) {
+            noUseMvHint.setStatus(Hint.HintStatus.SUCCESS);
+            return getBaseIndex().getId();
+        } else {
+            List<String> mvNames = noUseMvHint.getNoUseMVName(this.name);
+            Set<Long> forbiddenIndexIds = Sets.newHashSet();
+            for (int i = 0; i < mvNames.size(); i++) {
+                if (mvNames.get(i).equals("`*`")) {
+                    noUseMvHint.setStatus(Hint.HintStatus.SUCCESS);
+                    return getBaseIndex().getId();
+                }
+                if (hasMaterializedIndex(mvNames.get(i))) {
+                    Long forbiddenIndexId = indexNameToId.get(mvNames.get(i));
+                    forbiddenIndexIds.add(forbiddenIndexId);
+                } else {
+                    noUseMvHint.setStatus(Hint.HintStatus.SYNTAX_ERROR);
+                    noUseMvHint.setErrorMessage("do not have mv: " + mvNames.get(i) + " in table: " + this.name);
+                    break;
+                }
+            }
+            for (int i = 0; i < orderedMvs.size(); i++) {
+                if (forbiddenIndexIds.contains(orderedMvs.get(i))) {
+                    noUseMvHint.setStatus(Hint.HintStatus.SUCCESS);
+                } else {
+                    return orderedMvs.get(i);
+                }
+            }
+        }
+        return orderedMvs.get(0);
+    }
+
+    private Optional<UseMvHint> getUseMvHint(String useMvName) {
+        for (Hint hint : ConnectContext.get().getStatementContext().getHints()) {
+            if (hint.isSyntaxError()) {
+                continue;
+            }
+            if (hint.getHintName().equalsIgnoreCase(useMvName)) {
+                return Optional.of((UseMvHint) hint);
+            }
+        }
+        return Optional.empty();
+    }
+
     public List<MaterializedIndex> getVisibleIndex() {
         Optional<Partition> partition = idToPartition.values().stream().findFirst();
         if (!partition.isPresent()) {

fe/fe-core/src/main/java/org/apache/doris/nereids/CascadesContext.java

@@ -148,7 +148,7 @@ public class CascadesContext implements ScheduleContext {
      */
     private CascadesContext(Optional<CascadesContext> parent, Optional<CTEId> currentTree,
             StatementContext statementContext, Plan plan, Memo memo,
-            CTEContext cteContext, PhysicalProperties requireProperties) {
+            CTEContext cteContext, PhysicalProperties requireProperties, boolean isLeadingDisableJoinReorder) {
         this.parent = Objects.requireNonNull(parent, "parent should not null");
         this.currentTree = Objects.requireNonNull(currentTree, "currentTree should not null");
         this.statementContext = Objects.requireNonNull(statementContext, "statementContext should not null");
@@ -169,6 +169,10 @@ private CascadesContext(Optional<CascadesContext> parent, Optional<CTEId> curren
         } else {
             this.isEnableExprTrace = false;
         }
+        if (parent.isPresent()) {
+            this.tables = parent.get().tables;
+        }
+        this.isLeadingDisableJoinReorder = isLeadingDisableJoinReorder;
     }
 
     /**
@@ -177,7 +181,7 @@ private CascadesContext(Optional<CascadesContext> parent, Optional<CTEId> curren
     public static CascadesContext initContext(StatementContext statementContext,
             Plan initPlan, PhysicalProperties requireProperties) {
         return newContext(Optional.empty(), Optional.empty(), statementContext,
-                initPlan, new CTEContext(), requireProperties);
+                initPlan, new CTEContext(), requireProperties, false);
     }
 
     /**
@@ -186,14 +190,15 @@ public static CascadesContext initContext(StatementContext statementContext,
     public static CascadesContext newContextWithCteContext(CascadesContext cascadesContext,
             Plan initPlan, CTEContext cteContext) {
         return newContext(Optional.of(cascadesContext), Optional.empty(),
-                cascadesContext.getStatementContext(), initPlan, cteContext, PhysicalProperties.ANY
+                cascadesContext.getStatementContext(), initPlan, cteContext, PhysicalProperties.ANY,
+                cascadesContext.isLeadingDisableJoinReorder
         );
     }
 
     public static CascadesContext newCurrentTreeContext(CascadesContext context) {
         return CascadesContext.newContext(context.getParent(), context.getCurrentTree(), context.getStatementContext(),
                 context.getRewritePlan(), context.getCteContext(),
-                context.getCurrentJobContext().getRequiredProperties());
+                context.getCurrentJobContext().getRequiredProperties(), context.isLeadingDisableJoinReorder);
     }
 
     /**
@@ -202,14 +207,14 @@ public static CascadesContext newCurrentTreeContext(CascadesContext context) {
     public static CascadesContext newSubtreeContext(Optional<CTEId> subtree, CascadesContext context,
             Plan plan, PhysicalProperties requireProperties) {
         return CascadesContext.newContext(Optional.of(context), subtree, context.getStatementContext(),
-                plan, context.getCteContext(), requireProperties);
+                plan, context.getCteContext(), requireProperties, context.isLeadingDisableJoinReorder);
     }
 
     private static CascadesContext newContext(Optional<CascadesContext> parent, Optional<CTEId> subtree,
             StatementContext statementContext, Plan initPlan, CTEContext cteContext,
-            PhysicalProperties requireProperties) {
+            PhysicalProperties requireProperties, boolean isLeadingDisableJoinReorder) {
         return new CascadesContext(parent, subtree, statementContext, initPlan, null,
-            cteContext, requireProperties);
+            cteContext, requireProperties, isLeadingDisableJoinReorder);
     }
 
     public CascadesContext getRoot() {

fe/fe-core/src/main/java/org/apache/doris/nereids/hint/UseCboRuleHint.java

@@ -0,0 +1,47 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.hint;
+
+/**
+ * rule hint.
+ */
+public class UseCboRuleHint extends Hint {
+
+    private final boolean isNotUseCboRule;
+
+    public UseCboRuleHint(String hintName, boolean isNotUseCboRule) {
+        super(hintName);
+        this.isNotUseCboRule = isNotUseCboRule;
+    }
+
+    public boolean isNotUseCboRule() {
+        return isNotUseCboRule;
+    }
+
+    @Override
+    public String getExplainString() {
+        StringBuilder out = new StringBuilder();
+        if (isNotUseCboRule) {
+            out.append("no_use_");
+        } else {
+            out.append("use_");
+        }
+        out.append(getHintName());
+        return out.toString();
+    }
+}

fe/fe-core/src/main/java/org/apache/doris/nereids/hint/UseMvHint.java

@@ -0,0 +1,144 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.hint;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * rule hint.
+ */
+public class UseMvHint extends Hint {
+
+    private final boolean isUseMv;
+
+    private final boolean isAllMv;
+
+    private final List<String> parameters;
+
+    private final Map<String, String> useMvTableColumnMap;
+
+    private final Map<String, List<String>> noUseMvTableColumnMap;
+
+    /**
+     * constructor of use mv hint
+     * @param hintName use mv
+     * @param parameters original parameters
+     * @param isUseMv use_mv hint or no_use_mv hint
+     * @param isAllMv should all mv be controlled
+     */
+    public UseMvHint(String hintName, List<String> parameters, boolean isUseMv, boolean isAllMv) {
+        super(hintName);
+        this.isUseMv = isUseMv;
+        this.isAllMv = isAllMv;
+        this.parameters = parameters;
+        this.useMvTableColumnMap = initUseMvTableColumnMap(parameters);
+        this.noUseMvTableColumnMap = initNoUseMvTableColumnMap(parameters);
+    }
+
+    private Map<String, String> initUseMvTableColumnMap(List<String> parameters) {
+        Map<String, String> tempUseMvTableColumnMap = new HashMap<>();
+        if (!isUseMv) {
+            return tempUseMvTableColumnMap;
+        }
+        if (parameters.size() % 2 == 1) {
+            this.setStatus(HintStatus.SYNTAX_ERROR);
+            this.setErrorMessage("parameter of use_mv hint must be in pairs");
+            return tempUseMvTableColumnMap;
+        }
+        for (int i = 0; i < parameters.size(); i += 2) {
+            String tableName = parameters.get(i);
+            String columnName = parameters.get(i + 1);
+            if (tempUseMvTableColumnMap.containsKey(tableName)) {
+                this.setStatus(HintStatus.SYNTAX_ERROR);
+                this.setErrorMessage("use_mv hint should only have one mv in one table: "
+                        + tableName + "." + columnName);
+                break;
+            }
+            tempUseMvTableColumnMap.put(tableName, columnName);
+        }
+        return tempUseMvTableColumnMap;
+    }
+
+    private Map<String, List<String>> initNoUseMvTableColumnMap(List<String> parameters) {
+        Map<String, List<String>> tempNoUseMvTableColumnMap = new HashMap<>();
+        if (isUseMv) {
+            return tempNoUseMvTableColumnMap;
+        }
+        if (parameters.size() % 2 == 1) {
+            this.setStatus(HintStatus.SYNTAX_ERROR);
+            this.setErrorMessage("parameter of no_use_mv hint must be in pairs");
+            return tempNoUseMvTableColumnMap;
+        }
+        for (int i = 0; i < parameters.size(); i += 2) {
+            String tableName = parameters.get(i);
+            String columnName = parameters.get(i + 1);
+            if (tempNoUseMvTableColumnMap.containsKey(tableName)) {
+                tempNoUseMvTableColumnMap.get(tableName).add(columnName);
+            } else {
+                List<String> list = new ArrayList<>();
+                list.add(columnName);
+                tempNoUseMvTableColumnMap.put(tableName, list);
+            }
+        }
+        return tempNoUseMvTableColumnMap;
+    }
+
+    public boolean isUseMv() {
+        return isUseMv;
+    }
+
+    public boolean isAllMv() {
+        return isAllMv;
+    }
+
+    public String getUseMvName(String tableName) {
+        return useMvTableColumnMap.get(tableName);
+    }
+
+    public List<String> getNoUseMVName(String tableName) {
+        return noUseMvTableColumnMap.get(tableName);
+    }
+
+    @Override
+    public String getExplainString() {
+        StringBuilder out = new StringBuilder();
+        if (isUseMv) {
+            out.append("use_mv");
+        } else {
+            out.append("no_use_mv");
+        }
+        if (!parameters.isEmpty()) {
+            out.append("(");
+            for (int i = 0; i < parameters.size(); i++) {
+                if (i % 2 == 0) {
+                    out.append(parameters.get(i));
+                } else {
+                    out.append(".");
+                    out.append(parameters.get(i));
+                    out.append(" ");
+                }
+            }
+            out.append(")");
+        }
+
+        return out.toString();
+    }
+}

fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java

@@ -328,11 +328,11 @@ public class Rewriter extends AbstractBatchJobExecutor {
                 ),
 
                 topic("Eager aggregation",
-                        topDown(
+                        costBased(topDown(
                                 new PushDownAggThroughJoinOneSide(),
                                 new PushDownAggThroughJoin()
-                        ),
-                        custom(RuleType.PUSH_DOWN_DISTINCT_THROUGH_JOIN, PushDownDistinctThroughJoin::new)
+                        )),
+                        costBased(custom(RuleType.PUSH_DOWN_DISTINCT_THROUGH_JOIN, PushDownDistinctThroughJoin::new))
                 ),
 
                 // this rule should invoke after infer predicate and push down distinct, and before push down limit