Skip to content

[feat](Nereids): lazy get expression map when comparing hypergraph #34753

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
morrySnow merged 2 commits into from
May 17, 2024
Merged

[feat](Nereids): lazy get expression map when comparing hypergraph #34753

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
cd3f4cc
lazey get expression map
keanji-x May 13, 2024
c35e1ea
remove some useless code
keanji-x May 15, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
114 changes: 74 additions & 40 deletions ...ore/src/main/java/org/apache/doris/nereids/rules/exploration/mv/HyperGraphComparator.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Objects;
import java.util.Set;
import java.util.stream.Collectors;
import javax.annotation.Nullable;
Expand Down Expand Up @@ -120,30 +121,37 @@ private ComparisonResult isLogicCompatible() {
}

// 3 try to construct a map which can be mapped from edge to edge
Map<Edge, Edge> queryToView = constructQueryToViewMapWithExpr();
if (!makeViewJoinCompatible(queryToView)) {
Map<Edge, Edge> queryToViewJoinEdge = constructQueryToViewJoinMapWithExpr();
if (!makeViewJoinCompatible(queryToViewJoinEdge)) {
return ComparisonResult.newInvalidResWithErrorMessage("Join types are not compatible\n");
}
refreshViewEdges();

// 4 compare them by expression and nodes. Note compare edges after inferring for nodes
boolean matchNodes = queryToView.entrySet().stream()
boolean matchNodes = queryToViewJoinEdge.entrySet().stream()
.allMatch(e -> compareEdgeWithNode(e.getKey(), e.getValue()));
if (!matchNodes) {
return ComparisonResult.newInvalidResWithErrorMessage("Join nodes are not compatible\n");
}
Map<Edge, Edge> queryToViewFilterEdge = constructQueryToViewFilterMapWithExpr();
matchNodes = queryToViewFilterEdge.entrySet().stream()
.allMatch(e -> compareEdgeWithNode(e.getKey(), e.getValue()));
if (!matchNodes) {
return ComparisonResult.newInvalidResWithErrorMessage("Join nodes are not compatible\n");
}
queryToView.forEach(this::compareEdgeWithExpr);

queryToViewJoinEdge.forEach(this::compareJoinEdgeWithExpr);
queryToViewFilterEdge.forEach(this::compareFilterEdgeWithExpr);
// 5 process residual edges
Sets.difference(getQueryJoinEdgeSet(), queryToView.keySet())
Sets.difference(getQueryJoinEdgeSet(), queryToViewJoinEdge.keySet())
.forEach(e -> pullUpQueryExprWithEdge.put(e, e.getExpressions()));
Sets.difference(getQueryFilterEdgeSet(), queryToView.keySet())
Sets.difference(getQueryFilterEdgeSet(), queryToViewFilterEdge.keySet())
.forEach(e -> pullUpQueryExprWithEdge.put(e, e.getExpressions()));
Sets.difference(getViewJoinEdgeSet(), Sets.newHashSet(queryToView.values()))
Sets.difference(getViewJoinEdgeSet(), Sets.newHashSet(queryToViewJoinEdge.values()))
.stream()
.filter(e -> !LongBitmap.isOverlap(e.getReferenceNodes(), eliminateViewNodesMap))
.forEach(e -> pullUpViewExprWithEdge.put(e, e.getExpressions()));
Sets.difference(getViewFilterEdgeSet(), Sets.newHashSet(queryToView.values()))
Sets.difference(getViewFilterEdgeSet(), Sets.newHashSet(queryToViewFilterEdge.values()))
.stream()
.filter(e -> !LongBitmap.isOverlap(e.getReferenceNodes(), eliminateViewNodesMap))
.forEach(e -> pullUpViewExprWithEdge.put(e, e.getExpressions()));
Expand Down Expand Up @@ -238,7 +246,7 @@ private boolean compareNodeWithExpr(StructInfoNode query, StructInfoNode view) {
int size = queryExprSetList.size();
for (int i = 0; i < size; i++) {
Set<Expression> mappingQueryExprSet = queryExprSetList.get(i).stream()
.map(e -> logicalCompatibilityContext.getQueryToViewAllExpressionMapping().get(e))
.map(logicalCompatibilityContext::getViewNodeExprFromQuery)
.collect(Collectors.toSet());
if (!mappingQueryExprSet.equals(viewExprSetList.get(i))) {
return false;
Expand Down Expand Up @@ -350,12 +358,6 @@ private Set<FilterEdge> getQueryFilterEdgeSet() {
return ImmutableSet.copyOf(queryHyperGraph.getFilterEdges());
}

private List<Edge> getQueryEdges() {
return ImmutableList.<Edge>builder()
.addAll(getQueryJoinEdges())
.addAll(getQueryFilterEdges()).build();
}

private boolean makeViewJoinCompatible(Map<Edge, Edge> queryToView) {
for (Entry<Edge, Edge> entry : queryToView.entrySet()) {
if (entry.getKey() instanceof JoinEdge && entry.getValue() instanceof JoinEdge) {
Expand Down Expand Up @@ -384,37 +386,50 @@ private List<FilterEdge> getViewFilterEdges() {
return viewHyperGraph.getFilterEdges();
}

private List<Edge> getViewEdges() {
return ImmutableList.<Edge>builder()
.addAll(getViewJoinEdges())
.addAll(getViewFilterEdges()).build();
}

private Map<Expression, Expression> getQueryToViewExprMap() {
return logicalCompatibilityContext.getQueryToViewAllExpressionMapping();
}

private Map<Integer, Integer> getQueryToViewNodeIdMap() {
return logicalCompatibilityContext.getQueryToViewNodeIDMapping();
}

private Map<Edge, Edge> constructQueryToViewMapWithExpr() {
Map<Expression, Edge> viewExprToEdge = getViewEdges().stream()
private Map<Edge, Edge> constructQueryToViewJoinMapWithExpr() {
Map<Expression, Edge> viewExprToEdge = getViewJoinEdges().stream()
.flatMap(e -> e.getExpressions().stream().map(expr -> Pair.of(expr, e)))
.collect(ImmutableMap.toImmutableMap(p -> p.first, p -> p.second));
Map<Expression, Edge> queryExprToEdge = getQueryEdges().stream()
Map<Expression, Edge> queryExprToEdge = getQueryJoinEdges().stream()
.flatMap(e -> e.getExpressions().stream().map(expr -> Pair.of(expr, e)))
.collect(ImmutableMap.toImmutableMap(p -> p.first, p -> p.second));
return queryExprToEdge.entrySet().stream()
.filter(entry -> viewExprToEdge.containsKey(getViewExprFromQueryExpr(entry.getKey())))
.map(entry -> Pair.of(entry.getValue(),
viewExprToEdge.get(getViewExprFromQueryExpr(entry.getKey()))))
.distinct()
.collect(ImmutableMap.toImmutableMap(p -> p.first, p -> p.second));

HashMap<Edge, Edge> edgeMap = new HashMap<>();
for (Entry<Expression, Edge> entry : queryExprToEdge.entrySet()) {
if (edgeMap.containsKey(entry.getValue())) {
continue;
}
Expression viewExpr = logicalCompatibilityContext.getViewJoinExprFromQuery(entry.getKey());
if (viewExprToEdge.containsKey(viewExpr)) {
edgeMap.put(entry.getValue(), Objects.requireNonNull(viewExprToEdge.get(viewExpr)));
}
}
return edgeMap;
}

private Expression getViewExprFromQueryExpr(Expression query) {
return logicalCompatibilityContext.getQueryToViewAllExpressionMapping().get(query);
private Map<Edge, Edge> constructQueryToViewFilterMapWithExpr() {
Map<Expression, Edge> viewExprToEdge = getViewFilterEdges().stream()
.flatMap(e -> e.getExpressions().stream().map(expr -> Pair.of(expr, e)))
.collect(ImmutableMap.toImmutableMap(p -> p.first, p -> p.second));
Map<Expression, Edge> queryExprToEdge = getQueryFilterEdges().stream()
.flatMap(e -> e.getExpressions().stream().map(expr -> Pair.of(expr, e)))
.collect(ImmutableMap.toImmutableMap(p -> p.first, p -> p.second));

HashMap<Edge, Edge> edgeMap = new HashMap<>();
for (Entry<Expression, Edge> entry : queryExprToEdge.entrySet()) {
if (edgeMap.containsKey(entry.getValue())) {
continue;
}
Expression viewExpr = logicalCompatibilityContext.getViewFilterExprFromQuery(entry.getKey());
if (viewExprToEdge.containsKey(viewExpr)) {
edgeMap.put(entry.getValue(), Objects.requireNonNull(viewExprToEdge.get(viewExpr)));
}
}
return edgeMap;
}

private void refreshViewEdges() {
Expand Down Expand Up @@ -510,16 +525,35 @@ private long rewriteQueryNodeMap(long bitmap) {
return newBitmap;
}

private void compareEdgeWithExpr(Edge query, Edge view) {
private void compareJoinEdgeWithExpr(Edge query, Edge view) {
Set<? extends Expression> queryExprSet = query.getExpressionSet();
Set<? extends Expression> viewExprSet = view.getExpressionSet();

Set<Expression> exprMappedOfView = new HashSet<>();
List<Expression> residualQueryExpr = new ArrayList<>();
for (Expression queryExpr : queryExprSet) {
Expression viewExpr = logicalCompatibilityContext.getViewJoinExprFromQuery(queryExpr);
if (viewExprSet.contains(viewExpr)) {
exprMappedOfView.add(viewExpr);
} else {
residualQueryExpr.add(queryExpr);
}
}
List<Expression> residualViewExpr = ImmutableList.copyOf(Sets.difference(viewExprSet, exprMappedOfView));
pullUpQueryExprWithEdge.put(query, residualQueryExpr);
pullUpViewExprWithEdge.put(query, residualViewExpr);
}

private void compareFilterEdgeWithExpr(Edge query, Edge view) {
Set<? extends Expression> queryExprSet = query.getExpressionSet();
Set<? extends Expression> viewExprSet = view.getExpressionSet();

Set<Expression> exprMappedOfView = new HashSet<>();
List<Expression> residualQueryExpr = new ArrayList<>();
for (Expression queryExpr : queryExprSet) {
if (getQueryToViewExprMap().containsKey(queryExpr) && viewExprSet.contains(
getQueryToViewExprMap().get(queryExpr))) {
exprMappedOfView.add(getQueryToViewExprMap().get(queryExpr));
Expression viewExpr = logicalCompatibilityContext.getViewFilterExprFromQuery(queryExpr);
if (viewExprSet.contains(viewExpr)) {
exprMappedOfView.add(viewExpr);
} else {
residualQueryExpr.add(queryExpr);
}
Expand Down
31 changes: 6 additions & 25 deletions .../main/java/org/apache/doris/nereids/rules/exploration/mv/LogicalCompatibilityContext.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -92,35 +92,16 @@ public BiMap<Integer, Integer> getQueryToViewNodeIDMapping() {
return queryToViewNodeIDMapping;
}

/**
* Get all expression mapping in query to view
*/
@Deprecated
public BiMap<Expression, Expression> getQueryToViewAllExpressionMapping() {
if (queryToViewAllExpressionMapping != null) {
return queryToViewAllExpressionMapping;
}
queryToViewAllExpressionMapping = HashBiMap.create();
queryToViewAllExpressionMapping.putAll(getQueryToViewJoinEdgeExpressionMapping());
queryToViewAllExpressionMapping.putAll(getQueryToViewNodeExpressionMapping());
queryToViewAllExpressionMapping.putAll(getQueryToViewFilterEdgeExpressionMapping());
return queryToViewAllExpressionMapping;
}

public BiMap<Expression, Expression> getQueryToViewJoinEdgeExpressionMapping() {
return queryToViewJoinEdgeExpressionMappingSupplier.get();
}

public BiMap<Expression, Expression> getQueryToViewNodeExpressionMapping() {
return queryToViewNodeExpressionMappingSupplier.get();
public Expression getViewJoinExprFromQuery(Expression queryJoinExpr) {
return queryToViewJoinEdgeExpressionMappingSupplier.get().get(queryJoinExpr);
}

public BiMap<Expression, Expression> getQueryToViewFilterEdgeExpressionMapping() {
return queryToViewFilterEdgeExpressionMappingSupplier.get();
public Expression getViewFilterExprFromQuery(Expression queryJoinExpr) {
return queryToViewFilterEdgeExpressionMappingSupplier.get().get(queryJoinExpr);
}

public ObjectId getPlanNodeId() {
return planNodeId;
public Expression getViewNodeExprFromQuery(Expression queryJoinExpr) {
return queryToViewNodeExpressionMappingSupplier.get().get(queryJoinExpr);
}

/**
Expand Down