Skip to content

[improvement](mtmv) Split the expression mapping in LogicalCompatibilityContext for performance #34646

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
morrySnow merged 1 commit into from
May 11, 2024
Merged

[improvement](mtmv) Split the expression mapping in LogicalCompatibilityContext for performance #34646

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions ...ore/src/main/java/org/apache/doris/nereids/rules/exploration/mv/HyperGraphComparator.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -238,7 +238,7 @@ private boolean compareNodeWithExpr(StructInfoNode query, StructInfoNode view) {
int size = queryExprSetList.size();
for (int i = 0; i < size; i++) {
Set<Expression> mappingQueryExprSet = queryExprSetList.get(i).stream()
.map(e -> logicalCompatibilityContext.getQueryToViewEdgeExpressionMapping().get(e))
.map(e -> logicalCompatibilityContext.getQueryToViewAllExpressionMapping().get(e))
.collect(Collectors.toSet());
if (!mappingQueryExprSet.equals(viewExprSetList.get(i))) {
return false;
Expand Down Expand Up @@ -391,7 +391,7 @@ private List<Edge> getViewEdges() {
}

private Map<Expression, Expression> getQueryToViewExprMap() {
return logicalCompatibilityContext.getQueryToViewEdgeExpressionMapping();
return logicalCompatibilityContext.getQueryToViewAllExpressionMapping();
}

private Map<Integer, Integer> getQueryToViewNodeIdMap() {
Expand All @@ -414,7 +414,7 @@ private Map<Edge, Edge> constructQueryToViewMapWithExpr() {
}

private Expression getViewExprFromQueryExpr(Expression query) {
return logicalCompatibilityContext.getQueryToViewEdgeExpressionMapping().get(query);
return logicalCompatibilityContext.getQueryToViewAllExpressionMapping().get(query);
}

private void refreshViewEdges() {
Expand Down
97 changes: 80 additions & 17 deletions .../main/java/org/apache/doris/nereids/rules/exploration/mv/LogicalCompatibilityContext.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@

import org.apache.doris.nereids.jobs.joinorder.hypergraph.node.StructInfoNode;
import org.apache.doris.nereids.memo.GroupExpression;
import org.apache.doris.nereids.rules.exploration.mv.StructInfo.ExpressionPosition;
import org.apache.doris.nereids.rules.exploration.mv.mapping.Mapping.MappedRelation;
import org.apache.doris.nereids.rules.exploration.mv.mapping.RelationMapping;
import org.apache.doris.nereids.rules.exploration.mv.mapping.SlotMapping;
Expand All @@ -32,33 +33,55 @@
import org.apache.doris.nereids.util.ExpressionUtils;
import org.apache.doris.nereids.util.Utils;

import com.google.common.base.Suppliers;
import com.google.common.collect.BiMap;
import com.google.common.collect.HashBiMap;

import java.util.HashMap;
import java.util.Map;
import java.util.function.Supplier;

/**
* For outer join we should check the outer join compatibility between query and view
*/
public class LogicalCompatibilityContext {
private final BiMap<StructInfoNode, StructInfoNode> queryToViewNodeMapping;
private final BiMap<Expression, Expression> queryToViewEdgeExpressionMapping;
private final BiMap<Integer, Integer> queryToViewNodeIDMapping;
private final ObjectId planNodeId;
private final Supplier<BiMap<Expression, Expression>> queryToViewJoinEdgeExpressionMappingSupplier;
private final Supplier<BiMap<Expression, Expression>> queryToViewNodeExpressionMappingSupplier;
private final Supplier<BiMap<Expression, Expression>> queryToViewFilterEdgeExpressionMappingSupplier;
@Deprecated
private BiMap<Expression, Expression> queryToViewAllExpressionMapping;

/**
* LogicalCompatibilityContext
*/
public LogicalCompatibilityContext(BiMap<StructInfoNode, StructInfoNode> queryToViewNodeMapping,
BiMap<Expression, Expression> queryToViewEdgeExpressionMapping,
StructInfo queryStructInfo) {
private LogicalCompatibilityContext(BiMap<StructInfoNode, StructInfoNode> queryToViewNodeMapping,
Map<SlotReference, SlotReference> viewToQuerySlotMapping, StructInfo queryStructInfo,
StructInfo viewStructInfo) {

this.queryToViewJoinEdgeExpressionMappingSupplier =
Suppliers.memoize(() -> generateExpressionMapping(viewToQuerySlotMapping,
queryStructInfo.getShuttledExpressionsToExpressionsMap().get(ExpressionPosition.JOIN_EDGE),
viewStructInfo.getShuttledExpressionsToExpressionsMap().get(ExpressionPosition.JOIN_EDGE)));

this.queryToViewNodeExpressionMappingSupplier =
Suppliers.memoize(() -> generateExpressionMapping(viewToQuerySlotMapping,
queryStructInfo.getShuttledExpressionsToExpressionsMap().get(ExpressionPosition.NODE),
viewStructInfo.getShuttledExpressionsToExpressionsMap().get(ExpressionPosition.NODE)));

this.queryToViewFilterEdgeExpressionMappingSupplier =
Suppliers.memoize(() -> generateExpressionMapping(viewToQuerySlotMapping,
queryStructInfo.getShuttledExpressionsToExpressionsMap().get(ExpressionPosition.FILTER_EDGE),
viewStructInfo.getShuttledExpressionsToExpressionsMap().get(ExpressionPosition.FILTER_EDGE)));

this.queryToViewNodeMapping = queryToViewNodeMapping;
this.queryToViewEdgeExpressionMapping = queryToViewEdgeExpressionMapping;
this.queryToViewNodeIDMapping = HashBiMap.create();
queryToViewNodeMapping.forEach((k, v) -> queryToViewNodeIDMapping.put(k.getIndex(), v.getIndex()));

this.planNodeId = queryStructInfo.getTopPlan().getGroupExpression()
.map(GroupExpression::getId).orElseGet(() -> new ObjectId(-1));
queryToViewNodeMapping.forEach((k, v) -> queryToViewNodeIDMapping.put(k.getIndex(), v.getIndex()));
}

public BiMap<StructInfoNode, StructInfoNode> getQueryToViewNodeMapping() {
Expand All @@ -69,8 +92,31 @@ public BiMap<Integer, Integer> getQueryToViewNodeIDMapping() {
return queryToViewNodeIDMapping;
}

public BiMap<Expression, Expression> getQueryToViewEdgeExpressionMapping() {
return queryToViewEdgeExpressionMapping;
/**
* Get all expression mapping in query to view
*/
@Deprecated
public BiMap<Expression, Expression> getQueryToViewAllExpressionMapping() {
if (queryToViewAllExpressionMapping != null) {
return queryToViewAllExpressionMapping;
}
queryToViewAllExpressionMapping = HashBiMap.create();
queryToViewAllExpressionMapping.putAll(getQueryToViewJoinEdgeExpressionMapping());
queryToViewAllExpressionMapping.putAll(getQueryToViewNodeExpressionMapping());
queryToViewAllExpressionMapping.putAll(getQueryToViewFilterEdgeExpressionMapping());
return queryToViewAllExpressionMapping;
}

public BiMap<Expression, Expression> getQueryToViewJoinEdgeExpressionMapping() {
return queryToViewJoinEdgeExpressionMappingSupplier.get();
}

public BiMap<Expression, Expression> getQueryToViewNodeExpressionMapping() {
return queryToViewNodeExpressionMappingSupplier.get();
}

public BiMap<Expression, Expression> getQueryToViewFilterEdgeExpressionMapping() {
return queryToViewFilterEdgeExpressionMappingSupplier.get();
}

public ObjectId getPlanNodeId() {
Expand Down Expand Up @@ -104,24 +150,33 @@ public static LogicalCompatibilityContext from(RelationMapping relationMapping,
// init expression mapping
Map<SlotReference, SlotReference> viewToQuerySlotMapping = queryToViewSlotMapping.inverse()
.toSlotReferenceMap();
Map<Expression, Expression> queryShuttledExprToExprMap =
queryStructInfo.getShuttledHashConjunctsToConjunctsMap();
Map<Expression, Expression> viewShuttledExprToExprMap =
viewStructInfo.getShuttledHashConjunctsToConjunctsMap();
return new LogicalCompatibilityContext(queryToViewNodeMapping,
viewToQuerySlotMapping,
queryStructInfo,
viewStructInfo);
}

private static BiMap<Expression, Expression> generateExpressionMapping(
Map<SlotReference, SlotReference> viewToQuerySlotMapping,
Map<Expression, Expression> queryShuttledExprToExprMap,
Map<Expression, Expression> viewShuttledExprToExprMap) {
final Map<Expression, Expression> viewEdgeToConjunctsMapQueryBased = new HashMap<>();
BiMap<Expression, Expression> queryToViewEdgeMapping = HashBiMap.create();
if (queryShuttledExprToExprMap == null || viewShuttledExprToExprMap == null
|| queryShuttledExprToExprMap.isEmpty() || viewShuttledExprToExprMap.isEmpty()) {
return queryToViewEdgeMapping;
}
viewShuttledExprToExprMap.forEach((shuttledExpr, expr) -> {
viewEdgeToConjunctsMapQueryBased.put(
orderSlotAsc(ExpressionUtils.replace(shuttledExpr, viewToQuerySlotMapping)),
expr);
orderSlotAsc(ExpressionUtils.replace(shuttledExpr, viewToQuerySlotMapping)), expr);
});
BiMap<Expression, Expression> queryToViewEdgeMapping = HashBiMap.create();
queryShuttledExprToExprMap.forEach((exprSet, edge) -> {
Expression viewExpr = viewEdgeToConjunctsMapQueryBased.get(orderSlotAsc(exprSet));
if (viewExpr != null) {
queryToViewEdgeMapping.put(edge, viewExpr);
}
});
return new LogicalCompatibilityContext(queryToViewNodeMapping, queryToViewEdgeMapping, queryStructInfo);
return queryToViewEdgeMapping;
}

private static Expression orderSlotAsc(Expression expression) {
Expand Down Expand Up @@ -151,6 +206,14 @@ public Expression visitEqualTo(EqualTo equalTo, Void context) {
public String toString() {
return Utils.toSqlString("LogicalCompatibilityContext",
"queryToViewNodeMapping", queryToViewNodeMapping.toString(),
"queryToViewEdgeExpressionMapping", queryToViewEdgeExpressionMapping.toString());
"queryToViewJoinEdgeExpressionMapping",
queryToViewJoinEdgeExpressionMappingSupplier.get() == null
? "" : queryToViewJoinEdgeExpressionMappingSupplier.get().toString(),
"queryToViewNodeExpressionMapping",
queryToViewNodeExpressionMappingSupplier.get() == null
? "" : queryToViewNodeExpressionMappingSupplier.get().toString(),
"queryToViewFilterEdgeExpressionMapping",
queryToViewFilterEdgeExpressionMappingSupplier.get() == null
? "" : queryToViewFilterEdgeExpressionMappingSupplier.get().toString());
}
}
64 changes: 49 additions & 15 deletions fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -107,7 +107,7 @@ public class StructInfo {
private final EquivalenceClass equivalenceClass;
// Key is the expression shuttled and the value is the origin expression
// this is for building LogicalCompatibilityContext later.
private final Map<Expression, Expression> shuttledHashConjunctsToConjunctsMap;
private final Map<ExpressionPosition, Map<Expression, Expression>> shuttledExpressionsToExpressionsMap;
// Record the exprId and the corresponding expr map, this is used by expression shuttled
private final Map<ExprId, Expression> namedExprIdAndExprMapping;

Expand All @@ -118,7 +118,7 @@ public StructInfo(Plan originalPlan, ObjectId originalPlanId, HyperGraph hyperGr
Plan bottomPlan, List<CatalogRelation> relations,
Map<RelationId, StructInfoNode> relationIdStructInfoNodeMap,
@Nullable Predicates predicates,
Map<Expression, Expression> shuttledHashConjunctsToConjunctsMap,
Map<ExpressionPosition, Map<Expression, Expression>> shuttledExpressionsToExpressionsMap,
Map<ExprId, Expression> namedExprIdAndExprMapping) {
this.originalPlan = originalPlan;
this.originalPlanId = originalPlanId;
Expand All @@ -140,7 +140,7 @@ public StructInfo(Plan originalPlan, ObjectId originalPlanId, HyperGraph hyperGr
predicatesDerive(this.predicates, topPlan, tableBitSet);
this.splitPredicate = derivedPredicates.key();
this.equivalenceClass = derivedPredicates.value();
this.shuttledHashConjunctsToConjunctsMap = shuttledHashConjunctsToConjunctsMap;
this.shuttledExpressionsToExpressionsMap = shuttledExpressionsToExpressionsMap;
this.namedExprIdAndExprMapping = namedExprIdAndExprMapping;
}

Expand All @@ -150,12 +150,12 @@ public StructInfo(Plan originalPlan, ObjectId originalPlanId, HyperGraph hyperGr
public StructInfo withPredicates(Predicates predicates) {
return new StructInfo(this.originalPlan, this.originalPlanId, this.hyperGraph, this.valid, this.topPlan,
this.bottomPlan, this.relations, this.relationIdStructInfoNodeMap, predicates,
this.shuttledHashConjunctsToConjunctsMap, this.namedExprIdAndExprMapping);
this.shuttledExpressionsToExpressionsMap, this.namedExprIdAndExprMapping);
}

private static boolean collectStructInfoFromGraph(HyperGraph hyperGraph,
Plan topPlan,
Map<Expression, Expression> shuttledHashConjunctsToConjunctsMap,
Map<ExpressionPosition, Map<Expression, Expression>> shuttledExpressionsToExpressionsMap,
Map<ExprId, Expression> namedExprIdAndExprMapping,
List<CatalogRelation> relations,
Map<RelationId, StructInfoNode> relationIdStructInfoNodeMap) {
Expand Down Expand Up @@ -189,7 +189,8 @@ private static boolean collectStructInfoFromGraph(HyperGraph hyperGraph,
topPlan.accept(ExpressionLineageReplacer.INSTANCE, replaceContext);
// Replace expressions by expression map
List<Expression> replacedExpressions = replaceContext.getReplacedExpressions();
shuttledHashConjunctsToConjunctsMap.put(replacedExpressions.get(0), conjunctExpr);
putShuttledExpressionsToExpressionsMap(shuttledExpressionsToExpressionsMap,
ExpressionPosition.JOIN_EDGE, replacedExpressions.get(0), conjunctExpr);
// Record this, will be used in top level expression shuttle later, see the method
// ExpressionLineageReplacer#visitGroupPlan
namedExprIdAndExprMapping.putAll(replaceContext.getExprIdExpressionMap());
Expand All @@ -213,7 +214,8 @@ private static boolean collectStructInfoFromGraph(HyperGraph hyperGraph,
structInfoNode.getPlan().accept(ExpressionLineageReplacer.INSTANCE, replaceContext);
// Replace expressions by expression map
List<Expression> replacedExpressions = replaceContext.getReplacedExpressions();
shuttledHashConjunctsToConjunctsMap.put(replacedExpressions.get(0), expression);
putShuttledExpressionsToExpressionsMap(shuttledExpressionsToExpressionsMap,
ExpressionPosition.NODE, replacedExpressions.get(0), expression);
// Record this, will be used in top level expression shuttle later, see the method
// ExpressionLineageReplacer#visitGroupPlan
namedExprIdAndExprMapping.putAll(replaceContext.getExprIdExpressionMap());
Expand All @@ -226,8 +228,10 @@ private static boolean collectStructInfoFromGraph(HyperGraph hyperGraph,
filterExpressions.forEach(predicate -> {
// this is used for LogicalCompatibilityContext
ExpressionUtils.extractConjunction(predicate).forEach(expr ->
shuttledHashConjunctsToConjunctsMap.put(ExpressionUtils.shuttleExpressionWithLineage(
predicate, topPlan, hyperTableBitSet), predicate));
putShuttledExpressionsToExpressionsMap(shuttledExpressionsToExpressionsMap,
ExpressionPosition.FILTER_EDGE,
ExpressionUtils.shuttleExpressionWithLineage(predicate, topPlan, hyperTableBitSet),
predicate));
});
});
return true;
Expand Down Expand Up @@ -299,7 +303,8 @@ public static StructInfo of(Plan originalPlan, @Nullable Plan topPlan, @Nullable
// collect struct info fromGraph
List<CatalogRelation> relationList = new ArrayList<>();
Map<RelationId, StructInfoNode> relationIdStructInfoNodeMap = new LinkedHashMap<>();
Map<Expression, Expression> shuttledHashConjunctsToConjunctsMap = new LinkedHashMap<>();
Map<ExpressionPosition, Map<Expression, Expression>> shuttledHashConjunctsToConjunctsMap =
new LinkedHashMap<>();
Map<ExprId, Expression> namedExprIdAndExprMapping = new LinkedHashMap<>();
boolean valid = collectStructInfoFromGraph(hyperGraph, topPlan, shuttledHashConjunctsToConjunctsMap,
namedExprIdAndExprMapping,
Expand Down Expand Up @@ -359,8 +364,21 @@ public Map<RelationId, StructInfoNode> getRelationIdStructInfoNodeMap() {
return relationIdStructInfoNodeMap;
}

public Map<Expression, Expression> getShuttledHashConjunctsToConjunctsMap() {
return shuttledHashConjunctsToConjunctsMap;
public Map<ExpressionPosition, Map<Expression, Expression>> getShuttledExpressionsToExpressionsMap() {
return shuttledExpressionsToExpressionsMap;
}

private static void putShuttledExpressionsToExpressionsMap(
Map<ExpressionPosition, Map<Expression, Expression>> shuttledExpressionsToExpressionsMap,
ExpressionPosition expressionPosition,
Expression key, Expression value) {
Map<Expression, Expression> expressionExpressionMap = shuttledExpressionsToExpressionsMap.get(
expressionPosition);
if (expressionExpressionMap == null) {
expressionExpressionMap = new LinkedHashMap<>();
shuttledExpressionsToExpressionsMap.put(expressionPosition, expressionExpressionMap);
}
expressionExpressionMap.put(key, value);
}

public List<? extends Expression> getExpressions() {
Expand Down Expand Up @@ -444,7 +462,9 @@ public Void visit(Plan plan, PlanSplitContext context) {
}
}

/** Judge if source contains all target */
/**
* Judge if source contains all target
*/
public static boolean containsAll(BitSet source, BitSet target) {
if (source.size() < target.size()) {
return false;
Expand Down Expand Up @@ -657,7 +677,9 @@ public Plan visitLogicalOlapScan(LogicalOlapScan olapScan, Pair<MTMV, Set<Long>>
}
}

/**Collect partitions which scan used according to given table */
/**
* Collect partitions which scan used according to given table
*/
public static class QueryScanPartitionsCollector extends DefaultPlanVisitor<Plan, Map<Long, Set<PartitionItem>>> {
@Override
public Plan visitLogicalCatalogRelation(LogicalCatalogRelation catalogRelation,
Expand All @@ -681,7 +703,9 @@ public Plan visitLogicalCatalogRelation(LogicalCatalogRelation catalogRelation,
}
}

/**Add filter on table scan according to table filter map */
/**
* Add filter on table scan according to table filter map
*/
public static Plan addFilterOnTableScan(Plan queryPlan, Map<TableIf, Set<Expression>> filterOnOriginPlan,
CascadesContext parentCascadesContext) {
// Firstly, construct filter form invalid partition, this filter should be added on origin plan
Expand All @@ -696,4 +720,14 @@ public static Plan addFilterOnTableScan(Plan queryPlan, Map<TableIf, Set<Express
return context.getRewritePlan();
}, queryPlanWithUnionFilter, queryPlan);
}

/**
* Expressions may appear in three place in hype graph, this identifies the position where
* expression appear in hyper graph
*/
public static enum ExpressionPosition {
JOIN_EDGE,
NODE,
FILTER_EDGE
}
}