diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/HyperGraphComparator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/HyperGraphComparator.java index 0c9dc9c33deece..d180254c7fe1a6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/HyperGraphComparator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/HyperGraphComparator.java @@ -46,6 +46,7 @@ import java.util.List; import java.util.Map; import java.util.Map.Entry; +import java.util.Objects; import java.util.Set; import java.util.stream.Collectors; import javax.annotation.Nullable; @@ -120,30 +121,37 @@ private ComparisonResult isLogicCompatible() { } // 3 try to construct a map which can be mapped from edge to edge - Map queryToView = constructQueryToViewMapWithExpr(); - if (!makeViewJoinCompatible(queryToView)) { + Map queryToViewJoinEdge = constructQueryToViewJoinMapWithExpr(); + if (!makeViewJoinCompatible(queryToViewJoinEdge)) { return ComparisonResult.newInvalidResWithErrorMessage("Join types are not compatible\n"); } refreshViewEdges(); // 4 compare them by expression and nodes. Note compare edges after inferring for nodes - boolean matchNodes = queryToView.entrySet().stream() + boolean matchNodes = queryToViewJoinEdge.entrySet().stream() + .allMatch(e -> compareEdgeWithNode(e.getKey(), e.getValue())); + if (!matchNodes) { + return ComparisonResult.newInvalidResWithErrorMessage("Join nodes are not compatible\n"); + } + Map queryToViewFilterEdge = constructQueryToViewFilterMapWithExpr(); + matchNodes = queryToViewFilterEdge.entrySet().stream() .allMatch(e -> compareEdgeWithNode(e.getKey(), e.getValue())); if (!matchNodes) { return ComparisonResult.newInvalidResWithErrorMessage("Join nodes are not compatible\n"); } - queryToView.forEach(this::compareEdgeWithExpr); + queryToViewJoinEdge.forEach(this::compareJoinEdgeWithExpr); + queryToViewFilterEdge.forEach(this::compareFilterEdgeWithExpr); // 5 process residual edges - Sets.difference(getQueryJoinEdgeSet(), queryToView.keySet()) + Sets.difference(getQueryJoinEdgeSet(), queryToViewJoinEdge.keySet()) .forEach(e -> pullUpQueryExprWithEdge.put(e, e.getExpressions())); - Sets.difference(getQueryFilterEdgeSet(), queryToView.keySet()) + Sets.difference(getQueryFilterEdgeSet(), queryToViewFilterEdge.keySet()) .forEach(e -> pullUpQueryExprWithEdge.put(e, e.getExpressions())); - Sets.difference(getViewJoinEdgeSet(), Sets.newHashSet(queryToView.values())) + Sets.difference(getViewJoinEdgeSet(), Sets.newHashSet(queryToViewJoinEdge.values())) .stream() .filter(e -> !LongBitmap.isOverlap(e.getReferenceNodes(), eliminateViewNodesMap)) .forEach(e -> pullUpViewExprWithEdge.put(e, e.getExpressions())); - Sets.difference(getViewFilterEdgeSet(), Sets.newHashSet(queryToView.values())) + Sets.difference(getViewFilterEdgeSet(), Sets.newHashSet(queryToViewFilterEdge.values())) .stream() .filter(e -> !LongBitmap.isOverlap(e.getReferenceNodes(), eliminateViewNodesMap)) .forEach(e -> pullUpViewExprWithEdge.put(e, e.getExpressions())); @@ -238,7 +246,7 @@ private boolean compareNodeWithExpr(StructInfoNode query, StructInfoNode view) { int size = queryExprSetList.size(); for (int i = 0; i < size; i++) { Set mappingQueryExprSet = queryExprSetList.get(i).stream() - .map(e -> logicalCompatibilityContext.getQueryToViewAllExpressionMapping().get(e)) + .map(logicalCompatibilityContext::getViewNodeExprFromQuery) .collect(Collectors.toSet()); if (!mappingQueryExprSet.equals(viewExprSetList.get(i))) { return false; @@ -350,12 +358,6 @@ private Set getQueryFilterEdgeSet() { return ImmutableSet.copyOf(queryHyperGraph.getFilterEdges()); } - private List getQueryEdges() { - return ImmutableList.builder() - .addAll(getQueryJoinEdges()) - .addAll(getQueryFilterEdges()).build(); - } - private boolean makeViewJoinCompatible(Map queryToView) { for (Entry entry : queryToView.entrySet()) { if (entry.getKey() instanceof JoinEdge && entry.getValue() instanceof JoinEdge) { @@ -384,37 +386,50 @@ private List getViewFilterEdges() { return viewHyperGraph.getFilterEdges(); } - private List getViewEdges() { - return ImmutableList.builder() - .addAll(getViewJoinEdges()) - .addAll(getViewFilterEdges()).build(); - } - - private Map getQueryToViewExprMap() { - return logicalCompatibilityContext.getQueryToViewAllExpressionMapping(); - } - private Map getQueryToViewNodeIdMap() { return logicalCompatibilityContext.getQueryToViewNodeIDMapping(); } - private Map constructQueryToViewMapWithExpr() { - Map viewExprToEdge = getViewEdges().stream() + private Map constructQueryToViewJoinMapWithExpr() { + Map viewExprToEdge = getViewJoinEdges().stream() .flatMap(e -> e.getExpressions().stream().map(expr -> Pair.of(expr, e))) .collect(ImmutableMap.toImmutableMap(p -> p.first, p -> p.second)); - Map queryExprToEdge = getQueryEdges().stream() + Map queryExprToEdge = getQueryJoinEdges().stream() .flatMap(e -> e.getExpressions().stream().map(expr -> Pair.of(expr, e))) .collect(ImmutableMap.toImmutableMap(p -> p.first, p -> p.second)); - return queryExprToEdge.entrySet().stream() - .filter(entry -> viewExprToEdge.containsKey(getViewExprFromQueryExpr(entry.getKey()))) - .map(entry -> Pair.of(entry.getValue(), - viewExprToEdge.get(getViewExprFromQueryExpr(entry.getKey())))) - .distinct() - .collect(ImmutableMap.toImmutableMap(p -> p.first, p -> p.second)); + + HashMap edgeMap = new HashMap<>(); + for (Entry entry : queryExprToEdge.entrySet()) { + if (edgeMap.containsKey(entry.getValue())) { + continue; + } + Expression viewExpr = logicalCompatibilityContext.getViewJoinExprFromQuery(entry.getKey()); + if (viewExprToEdge.containsKey(viewExpr)) { + edgeMap.put(entry.getValue(), Objects.requireNonNull(viewExprToEdge.get(viewExpr))); + } + } + return edgeMap; } - private Expression getViewExprFromQueryExpr(Expression query) { - return logicalCompatibilityContext.getQueryToViewAllExpressionMapping().get(query); + private Map constructQueryToViewFilterMapWithExpr() { + Map viewExprToEdge = getViewFilterEdges().stream() + .flatMap(e -> e.getExpressions().stream().map(expr -> Pair.of(expr, e))) + .collect(ImmutableMap.toImmutableMap(p -> p.first, p -> p.second)); + Map queryExprToEdge = getQueryFilterEdges().stream() + .flatMap(e -> e.getExpressions().stream().map(expr -> Pair.of(expr, e))) + .collect(ImmutableMap.toImmutableMap(p -> p.first, p -> p.second)); + + HashMap edgeMap = new HashMap<>(); + for (Entry entry : queryExprToEdge.entrySet()) { + if (edgeMap.containsKey(entry.getValue())) { + continue; + } + Expression viewExpr = logicalCompatibilityContext.getViewFilterExprFromQuery(entry.getKey()); + if (viewExprToEdge.containsKey(viewExpr)) { + edgeMap.put(entry.getValue(), Objects.requireNonNull(viewExprToEdge.get(viewExpr))); + } + } + return edgeMap; } private void refreshViewEdges() { @@ -510,16 +525,35 @@ private long rewriteQueryNodeMap(long bitmap) { return newBitmap; } - private void compareEdgeWithExpr(Edge query, Edge view) { + private void compareJoinEdgeWithExpr(Edge query, Edge view) { + Set queryExprSet = query.getExpressionSet(); + Set viewExprSet = view.getExpressionSet(); + + Set exprMappedOfView = new HashSet<>(); + List residualQueryExpr = new ArrayList<>(); + for (Expression queryExpr : queryExprSet) { + Expression viewExpr = logicalCompatibilityContext.getViewJoinExprFromQuery(queryExpr); + if (viewExprSet.contains(viewExpr)) { + exprMappedOfView.add(viewExpr); + } else { + residualQueryExpr.add(queryExpr); + } + } + List residualViewExpr = ImmutableList.copyOf(Sets.difference(viewExprSet, exprMappedOfView)); + pullUpQueryExprWithEdge.put(query, residualQueryExpr); + pullUpViewExprWithEdge.put(query, residualViewExpr); + } + + private void compareFilterEdgeWithExpr(Edge query, Edge view) { Set queryExprSet = query.getExpressionSet(); Set viewExprSet = view.getExpressionSet(); Set exprMappedOfView = new HashSet<>(); List residualQueryExpr = new ArrayList<>(); for (Expression queryExpr : queryExprSet) { - if (getQueryToViewExprMap().containsKey(queryExpr) && viewExprSet.contains( - getQueryToViewExprMap().get(queryExpr))) { - exprMappedOfView.add(getQueryToViewExprMap().get(queryExpr)); + Expression viewExpr = logicalCompatibilityContext.getViewFilterExprFromQuery(queryExpr); + if (viewExprSet.contains(viewExpr)) { + exprMappedOfView.add(viewExpr); } else { residualQueryExpr.add(queryExpr); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/LogicalCompatibilityContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/LogicalCompatibilityContext.java index b4ed509f300bf7..eef89b2512c15f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/LogicalCompatibilityContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/LogicalCompatibilityContext.java @@ -92,35 +92,16 @@ public BiMap getQueryToViewNodeIDMapping() { return queryToViewNodeIDMapping; } - /** - * Get all expression mapping in query to view - */ - @Deprecated - public BiMap getQueryToViewAllExpressionMapping() { - if (queryToViewAllExpressionMapping != null) { - return queryToViewAllExpressionMapping; - } - queryToViewAllExpressionMapping = HashBiMap.create(); - queryToViewAllExpressionMapping.putAll(getQueryToViewJoinEdgeExpressionMapping()); - queryToViewAllExpressionMapping.putAll(getQueryToViewNodeExpressionMapping()); - queryToViewAllExpressionMapping.putAll(getQueryToViewFilterEdgeExpressionMapping()); - return queryToViewAllExpressionMapping; - } - - public BiMap getQueryToViewJoinEdgeExpressionMapping() { - return queryToViewJoinEdgeExpressionMappingSupplier.get(); - } - - public BiMap getQueryToViewNodeExpressionMapping() { - return queryToViewNodeExpressionMappingSupplier.get(); + public Expression getViewJoinExprFromQuery(Expression queryJoinExpr) { + return queryToViewJoinEdgeExpressionMappingSupplier.get().get(queryJoinExpr); } - public BiMap getQueryToViewFilterEdgeExpressionMapping() { - return queryToViewFilterEdgeExpressionMappingSupplier.get(); + public Expression getViewFilterExprFromQuery(Expression queryJoinExpr) { + return queryToViewFilterEdgeExpressionMappingSupplier.get().get(queryJoinExpr); } - public ObjectId getPlanNodeId() { - return planNodeId; + public Expression getViewNodeExprFromQuery(Expression queryJoinExpr) { + return queryToViewNodeExpressionMappingSupplier.get().get(queryJoinExpr); } /**