diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/HyperGraphComparator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/HyperGraphComparator.java index 956a974982ccec..0c9dc9c33deece 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/HyperGraphComparator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/HyperGraphComparator.java @@ -238,7 +238,7 @@ private boolean compareNodeWithExpr(StructInfoNode query, StructInfoNode view) { int size = queryExprSetList.size(); for (int i = 0; i < size; i++) { Set mappingQueryExprSet = queryExprSetList.get(i).stream() - .map(e -> logicalCompatibilityContext.getQueryToViewEdgeExpressionMapping().get(e)) + .map(e -> logicalCompatibilityContext.getQueryToViewAllExpressionMapping().get(e)) .collect(Collectors.toSet()); if (!mappingQueryExprSet.equals(viewExprSetList.get(i))) { return false; @@ -391,7 +391,7 @@ private List getViewEdges() { } private Map getQueryToViewExprMap() { - return logicalCompatibilityContext.getQueryToViewEdgeExpressionMapping(); + return logicalCompatibilityContext.getQueryToViewAllExpressionMapping(); } private Map getQueryToViewNodeIdMap() { @@ -414,7 +414,7 @@ private Map constructQueryToViewMapWithExpr() { } private Expression getViewExprFromQueryExpr(Expression query) { - return logicalCompatibilityContext.getQueryToViewEdgeExpressionMapping().get(query); + return logicalCompatibilityContext.getQueryToViewAllExpressionMapping().get(query); } private void refreshViewEdges() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/LogicalCompatibilityContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/LogicalCompatibilityContext.java index 7f92735da640b3..b4ed509f300bf7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/LogicalCompatibilityContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/LogicalCompatibilityContext.java @@ -19,6 +19,7 @@ import org.apache.doris.nereids.jobs.joinorder.hypergraph.node.StructInfoNode; import org.apache.doris.nereids.memo.GroupExpression; +import org.apache.doris.nereids.rules.exploration.mv.StructInfo.ExpressionPosition; import org.apache.doris.nereids.rules.exploration.mv.mapping.Mapping.MappedRelation; import org.apache.doris.nereids.rules.exploration.mv.mapping.RelationMapping; import org.apache.doris.nereids.rules.exploration.mv.mapping.SlotMapping; @@ -32,33 +33,55 @@ import org.apache.doris.nereids.util.ExpressionUtils; import org.apache.doris.nereids.util.Utils; +import com.google.common.base.Suppliers; import com.google.common.collect.BiMap; import com.google.common.collect.HashBiMap; import java.util.HashMap; import java.util.Map; +import java.util.function.Supplier; /** * For outer join we should check the outer join compatibility between query and view */ public class LogicalCompatibilityContext { private final BiMap queryToViewNodeMapping; - private final BiMap queryToViewEdgeExpressionMapping; private final BiMap queryToViewNodeIDMapping; private final ObjectId planNodeId; + private final Supplier> queryToViewJoinEdgeExpressionMappingSupplier; + private final Supplier> queryToViewNodeExpressionMappingSupplier; + private final Supplier> queryToViewFilterEdgeExpressionMappingSupplier; + @Deprecated + private BiMap queryToViewAllExpressionMapping; /** * LogicalCompatibilityContext */ - public LogicalCompatibilityContext(BiMap queryToViewNodeMapping, - BiMap queryToViewEdgeExpressionMapping, - StructInfo queryStructInfo) { + private LogicalCompatibilityContext(BiMap queryToViewNodeMapping, + Map viewToQuerySlotMapping, StructInfo queryStructInfo, + StructInfo viewStructInfo) { + + this.queryToViewJoinEdgeExpressionMappingSupplier = + Suppliers.memoize(() -> generateExpressionMapping(viewToQuerySlotMapping, + queryStructInfo.getShuttledExpressionsToExpressionsMap().get(ExpressionPosition.JOIN_EDGE), + viewStructInfo.getShuttledExpressionsToExpressionsMap().get(ExpressionPosition.JOIN_EDGE))); + + this.queryToViewNodeExpressionMappingSupplier = + Suppliers.memoize(() -> generateExpressionMapping(viewToQuerySlotMapping, + queryStructInfo.getShuttledExpressionsToExpressionsMap().get(ExpressionPosition.NODE), + viewStructInfo.getShuttledExpressionsToExpressionsMap().get(ExpressionPosition.NODE))); + + this.queryToViewFilterEdgeExpressionMappingSupplier = + Suppliers.memoize(() -> generateExpressionMapping(viewToQuerySlotMapping, + queryStructInfo.getShuttledExpressionsToExpressionsMap().get(ExpressionPosition.FILTER_EDGE), + viewStructInfo.getShuttledExpressionsToExpressionsMap().get(ExpressionPosition.FILTER_EDGE))); + this.queryToViewNodeMapping = queryToViewNodeMapping; - this.queryToViewEdgeExpressionMapping = queryToViewEdgeExpressionMapping; this.queryToViewNodeIDMapping = HashBiMap.create(); + queryToViewNodeMapping.forEach((k, v) -> queryToViewNodeIDMapping.put(k.getIndex(), v.getIndex())); + this.planNodeId = queryStructInfo.getTopPlan().getGroupExpression() .map(GroupExpression::getId).orElseGet(() -> new ObjectId(-1)); - queryToViewNodeMapping.forEach((k, v) -> queryToViewNodeIDMapping.put(k.getIndex(), v.getIndex())); } public BiMap getQueryToViewNodeMapping() { @@ -69,8 +92,31 @@ public BiMap getQueryToViewNodeIDMapping() { return queryToViewNodeIDMapping; } - public BiMap getQueryToViewEdgeExpressionMapping() { - return queryToViewEdgeExpressionMapping; + /** + * Get all expression mapping in query to view + */ + @Deprecated + public BiMap getQueryToViewAllExpressionMapping() { + if (queryToViewAllExpressionMapping != null) { + return queryToViewAllExpressionMapping; + } + queryToViewAllExpressionMapping = HashBiMap.create(); + queryToViewAllExpressionMapping.putAll(getQueryToViewJoinEdgeExpressionMapping()); + queryToViewAllExpressionMapping.putAll(getQueryToViewNodeExpressionMapping()); + queryToViewAllExpressionMapping.putAll(getQueryToViewFilterEdgeExpressionMapping()); + return queryToViewAllExpressionMapping; + } + + public BiMap getQueryToViewJoinEdgeExpressionMapping() { + return queryToViewJoinEdgeExpressionMappingSupplier.get(); + } + + public BiMap getQueryToViewNodeExpressionMapping() { + return queryToViewNodeExpressionMappingSupplier.get(); + } + + public BiMap getQueryToViewFilterEdgeExpressionMapping() { + return queryToViewFilterEdgeExpressionMappingSupplier.get(); } public ObjectId getPlanNodeId() { @@ -104,24 +150,33 @@ public static LogicalCompatibilityContext from(RelationMapping relationMapping, // init expression mapping Map viewToQuerySlotMapping = queryToViewSlotMapping.inverse() .toSlotReferenceMap(); - Map queryShuttledExprToExprMap = - queryStructInfo.getShuttledHashConjunctsToConjunctsMap(); - Map viewShuttledExprToExprMap = - viewStructInfo.getShuttledHashConjunctsToConjunctsMap(); + return new LogicalCompatibilityContext(queryToViewNodeMapping, + viewToQuerySlotMapping, + queryStructInfo, + viewStructInfo); + } + + private static BiMap generateExpressionMapping( + Map viewToQuerySlotMapping, + Map queryShuttledExprToExprMap, + Map viewShuttledExprToExprMap) { final Map viewEdgeToConjunctsMapQueryBased = new HashMap<>(); + BiMap queryToViewEdgeMapping = HashBiMap.create(); + if (queryShuttledExprToExprMap == null || viewShuttledExprToExprMap == null + || queryShuttledExprToExprMap.isEmpty() || viewShuttledExprToExprMap.isEmpty()) { + return queryToViewEdgeMapping; + } viewShuttledExprToExprMap.forEach((shuttledExpr, expr) -> { viewEdgeToConjunctsMapQueryBased.put( - orderSlotAsc(ExpressionUtils.replace(shuttledExpr, viewToQuerySlotMapping)), - expr); + orderSlotAsc(ExpressionUtils.replace(shuttledExpr, viewToQuerySlotMapping)), expr); }); - BiMap queryToViewEdgeMapping = HashBiMap.create(); queryShuttledExprToExprMap.forEach((exprSet, edge) -> { Expression viewExpr = viewEdgeToConjunctsMapQueryBased.get(orderSlotAsc(exprSet)); if (viewExpr != null) { queryToViewEdgeMapping.put(edge, viewExpr); } }); - return new LogicalCompatibilityContext(queryToViewNodeMapping, queryToViewEdgeMapping, queryStructInfo); + return queryToViewEdgeMapping; } private static Expression orderSlotAsc(Expression expression) { @@ -151,6 +206,14 @@ public Expression visitEqualTo(EqualTo equalTo, Void context) { public String toString() { return Utils.toSqlString("LogicalCompatibilityContext", "queryToViewNodeMapping", queryToViewNodeMapping.toString(), - "queryToViewEdgeExpressionMapping", queryToViewEdgeExpressionMapping.toString()); + "queryToViewJoinEdgeExpressionMapping", + queryToViewJoinEdgeExpressionMappingSupplier.get() == null + ? "" : queryToViewJoinEdgeExpressionMappingSupplier.get().toString(), + "queryToViewNodeExpressionMapping", + queryToViewNodeExpressionMappingSupplier.get() == null + ? "" : queryToViewNodeExpressionMappingSupplier.get().toString(), + "queryToViewFilterEdgeExpressionMapping", + queryToViewFilterEdgeExpressionMappingSupplier.get() == null + ? "" : queryToViewFilterEdgeExpressionMappingSupplier.get().toString()); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java index 1870be9212bebc..b836a03af0cb8b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java @@ -107,7 +107,7 @@ public class StructInfo { private final EquivalenceClass equivalenceClass; // Key is the expression shuttled and the value is the origin expression // this is for building LogicalCompatibilityContext later. - private final Map shuttledHashConjunctsToConjunctsMap; + private final Map> shuttledExpressionsToExpressionsMap; // Record the exprId and the corresponding expr map, this is used by expression shuttled private final Map namedExprIdAndExprMapping; @@ -118,7 +118,7 @@ public StructInfo(Plan originalPlan, ObjectId originalPlanId, HyperGraph hyperGr Plan bottomPlan, List relations, Map relationIdStructInfoNodeMap, @Nullable Predicates predicates, - Map shuttledHashConjunctsToConjunctsMap, + Map> shuttledExpressionsToExpressionsMap, Map namedExprIdAndExprMapping) { this.originalPlan = originalPlan; this.originalPlanId = originalPlanId; @@ -140,7 +140,7 @@ public StructInfo(Plan originalPlan, ObjectId originalPlanId, HyperGraph hyperGr predicatesDerive(this.predicates, topPlan, tableBitSet); this.splitPredicate = derivedPredicates.key(); this.equivalenceClass = derivedPredicates.value(); - this.shuttledHashConjunctsToConjunctsMap = shuttledHashConjunctsToConjunctsMap; + this.shuttledExpressionsToExpressionsMap = shuttledExpressionsToExpressionsMap; this.namedExprIdAndExprMapping = namedExprIdAndExprMapping; } @@ -150,12 +150,12 @@ public StructInfo(Plan originalPlan, ObjectId originalPlanId, HyperGraph hyperGr public StructInfo withPredicates(Predicates predicates) { return new StructInfo(this.originalPlan, this.originalPlanId, this.hyperGraph, this.valid, this.topPlan, this.bottomPlan, this.relations, this.relationIdStructInfoNodeMap, predicates, - this.shuttledHashConjunctsToConjunctsMap, this.namedExprIdAndExprMapping); + this.shuttledExpressionsToExpressionsMap, this.namedExprIdAndExprMapping); } private static boolean collectStructInfoFromGraph(HyperGraph hyperGraph, Plan topPlan, - Map shuttledHashConjunctsToConjunctsMap, + Map> shuttledExpressionsToExpressionsMap, Map namedExprIdAndExprMapping, List relations, Map relationIdStructInfoNodeMap) { @@ -189,7 +189,8 @@ private static boolean collectStructInfoFromGraph(HyperGraph hyperGraph, topPlan.accept(ExpressionLineageReplacer.INSTANCE, replaceContext); // Replace expressions by expression map List replacedExpressions = replaceContext.getReplacedExpressions(); - shuttledHashConjunctsToConjunctsMap.put(replacedExpressions.get(0), conjunctExpr); + putShuttledExpressionsToExpressionsMap(shuttledExpressionsToExpressionsMap, + ExpressionPosition.JOIN_EDGE, replacedExpressions.get(0), conjunctExpr); // Record this, will be used in top level expression shuttle later, see the method // ExpressionLineageReplacer#visitGroupPlan namedExprIdAndExprMapping.putAll(replaceContext.getExprIdExpressionMap()); @@ -213,7 +214,8 @@ private static boolean collectStructInfoFromGraph(HyperGraph hyperGraph, structInfoNode.getPlan().accept(ExpressionLineageReplacer.INSTANCE, replaceContext); // Replace expressions by expression map List replacedExpressions = replaceContext.getReplacedExpressions(); - shuttledHashConjunctsToConjunctsMap.put(replacedExpressions.get(0), expression); + putShuttledExpressionsToExpressionsMap(shuttledExpressionsToExpressionsMap, + ExpressionPosition.NODE, replacedExpressions.get(0), expression); // Record this, will be used in top level expression shuttle later, see the method // ExpressionLineageReplacer#visitGroupPlan namedExprIdAndExprMapping.putAll(replaceContext.getExprIdExpressionMap()); @@ -226,8 +228,10 @@ private static boolean collectStructInfoFromGraph(HyperGraph hyperGraph, filterExpressions.forEach(predicate -> { // this is used for LogicalCompatibilityContext ExpressionUtils.extractConjunction(predicate).forEach(expr -> - shuttledHashConjunctsToConjunctsMap.put(ExpressionUtils.shuttleExpressionWithLineage( - predicate, topPlan, hyperTableBitSet), predicate)); + putShuttledExpressionsToExpressionsMap(shuttledExpressionsToExpressionsMap, + ExpressionPosition.FILTER_EDGE, + ExpressionUtils.shuttleExpressionWithLineage(predicate, topPlan, hyperTableBitSet), + predicate)); }); }); return true; @@ -299,7 +303,8 @@ public static StructInfo of(Plan originalPlan, @Nullable Plan topPlan, @Nullable // collect struct info fromGraph List relationList = new ArrayList<>(); Map relationIdStructInfoNodeMap = new LinkedHashMap<>(); - Map shuttledHashConjunctsToConjunctsMap = new LinkedHashMap<>(); + Map> shuttledHashConjunctsToConjunctsMap = + new LinkedHashMap<>(); Map namedExprIdAndExprMapping = new LinkedHashMap<>(); boolean valid = collectStructInfoFromGraph(hyperGraph, topPlan, shuttledHashConjunctsToConjunctsMap, namedExprIdAndExprMapping, @@ -359,8 +364,21 @@ public Map getRelationIdStructInfoNodeMap() { return relationIdStructInfoNodeMap; } - public Map getShuttledHashConjunctsToConjunctsMap() { - return shuttledHashConjunctsToConjunctsMap; + public Map> getShuttledExpressionsToExpressionsMap() { + return shuttledExpressionsToExpressionsMap; + } + + private static void putShuttledExpressionsToExpressionsMap( + Map> shuttledExpressionsToExpressionsMap, + ExpressionPosition expressionPosition, + Expression key, Expression value) { + Map expressionExpressionMap = shuttledExpressionsToExpressionsMap.get( + expressionPosition); + if (expressionExpressionMap == null) { + expressionExpressionMap = new LinkedHashMap<>(); + shuttledExpressionsToExpressionsMap.put(expressionPosition, expressionExpressionMap); + } + expressionExpressionMap.put(key, value); } public List getExpressions() { @@ -444,7 +462,9 @@ public Void visit(Plan plan, PlanSplitContext context) { } } - /** Judge if source contains all target */ + /** + * Judge if source contains all target + */ public static boolean containsAll(BitSet source, BitSet target) { if (source.size() < target.size()) { return false; @@ -657,7 +677,9 @@ public Plan visitLogicalOlapScan(LogicalOlapScan olapScan, Pair> } } - /**Collect partitions which scan used according to given table */ + /** + * Collect partitions which scan used according to given table + */ public static class QueryScanPartitionsCollector extends DefaultPlanVisitor>> { @Override public Plan visitLogicalCatalogRelation(LogicalCatalogRelation catalogRelation, @@ -681,7 +703,9 @@ public Plan visitLogicalCatalogRelation(LogicalCatalogRelation catalogRelation, } } - /**Add filter on table scan according to table filter map */ + /** + * Add filter on table scan according to table filter map + */ public static Plan addFilterOnTableScan(Plan queryPlan, Map> filterOnOriginPlan, CascadesContext parentCascadesContext) { // Firstly, construct filter form invalid partition, this filter should be added on origin plan @@ -696,4 +720,14 @@ public static Plan addFilterOnTableScan(Plan queryPlan, Map