From f7db5398760b540f34c8c1a1226550871401ee95 Mon Sep 17 00:00:00 2001 From: seawinde Date: Wed, 24 Apr 2024 08:32:03 +0800 Subject: [PATCH 1/4] [improvement](mtmv) Optimize the nested materialized view rewrite performance --- .../org/apache/doris/nereids/memo/Memo.java | 4 ++ .../doris/nereids/memo/StructInfoMap.java | 47 ++++++++++++------- 2 files changed, 34 insertions(+), 17 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Memo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Memo.java index 854ff0f51cb1c2..db12c02e79a185 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Memo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Memo.java @@ -411,6 +411,9 @@ private CopyInResult doCopyIn(Plan plan, @Nullable Group targetGroup, @Nullable Preconditions.checkState(groupExpressions.containsKey(groupExpr.get())); return CopyInResult.of(false, groupExpr.get()); } + if (targetGroup != null) { + targetGroup.getstructInfoMap().setRefreshed(false); + } List childrenGroups = Lists.newArrayList(); for (int i = 0; i < plan.children().size(); i++) { // skip useless project. @@ -559,6 +562,7 @@ public void mergeGroup(Group source, Group destination, HashMap pla if (source == root) { root = destination; } + destination.getstructInfoMap().setRefreshed(false); groups.remove(source.getGroupId()); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/StructInfoMap.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/StructInfoMap.java index a14c2070a8fd9b..21ef82f96fda11 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/StructInfoMap.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/StructInfoMap.java @@ -29,10 +29,10 @@ import java.util.Collection; import java.util.HashMap; import java.util.HashSet; +import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Set; -import java.util.stream.Collectors; import javax.annotation.Nullable; /** @@ -41,6 +41,7 @@ public class StructInfoMap { private final Map>> groupExpressionMap = new HashMap<>(); private final Map infoMap = new HashMap<>(); + private boolean refreshed; /** * get struct info according to table map @@ -79,6 +80,14 @@ public Pair> getGroupExpressionWithChildren(BitSet return groupExpressionMap.get(tableMap); } + public boolean isRefreshed() { + return refreshed; + } + + public void setRefreshed(boolean refreshed) { + this.refreshed = refreshed; + } + private StructInfo constructStructInfo(GroupExpression groupExpression, List children, BitSet tableMap, Plan originPlan) { // this plan is not origin plan, should record origin plan in struct info @@ -111,7 +120,7 @@ public boolean refresh(Group group) { int originSize = groupExpressionMap.size(); for (GroupExpression groupExpression : group.getLogicalExpressions()) { List> childrenTableMap = new ArrayList<>(); - boolean needRefresh = false; + boolean needRefresh = groupExpressionMap.isEmpty(); if (groupExpression.children().isEmpty()) { BitSet leaf = constructLeaf(groupExpression); groupExpressionMap.put(leaf, Pair.of(groupExpression, new ArrayList<>())); @@ -119,18 +128,22 @@ public boolean refresh(Group group) { } for (Group child : groupExpression.children()) { - if (!refreshedGroup.contains(child)) { + if (!refreshedGroup.contains(child) && !child.getstructInfoMap().isRefreshed()) { StructInfoMap childStructInfoMap = child.getstructInfoMap(); needRefresh |= childStructInfoMap.refresh(child); + childStructInfoMap.setRefreshed(true); } refreshedGroup.add(child); childrenTableMap.add(child.getstructInfoMap().getTableMaps()); } // if cumulative child table map is different from current // or current group expression map is empty, should update the groupExpressionMap currently - Set>> bitSetWithChildren = cartesianProduct(childrenTableMap); - for (Pair> bitSetWithChild : bitSetWithChildren) { - groupExpressionMap.putIfAbsent(bitSetWithChild.first, Pair.of(groupExpression, bitSetWithChild.second)); + Collection>> bitSetWithChildren = cartesianProduct(childrenTableMap); + if (needRefresh) { + for (Pair> bitSetWithChild : bitSetWithChildren) { + groupExpressionMap.putIfAbsent(bitSetWithChild.first, + Pair.of(groupExpression, bitSetWithChild.second)); + } } } return originSize != groupExpressionMap.size(); @@ -147,17 +160,17 @@ private BitSet constructLeaf(GroupExpression groupExpression) { return tableMap; } - private Set>> cartesianProduct(List> childrenTableMap) { - return Sets.cartesianProduct(childrenTableMap) - .stream() - .map(bitSetList -> { - BitSet bitSet = new BitSet(); - for (BitSet b : bitSetList) { - bitSet.or(b); - } - return Pair.of(bitSet, bitSetList); - }) - .collect(Collectors.toSet()); + private Collection>> cartesianProduct(List> childrenTableMap) { + Set> cartesianLists = Sets.cartesianProduct(childrenTableMap); + List>> resultPairSet = new LinkedList<>(); + for (List bitSetList : cartesianLists) { + BitSet bitSet = new BitSet(); + for (BitSet b : bitSetList) { + bitSet.or(b); + } + resultPairSet.add(Pair.of(bitSet, bitSetList)); + } + return resultPairSet; } @Override From 72907b8c834d51ced9cddbcf28ee9aebe3c3cc7a Mon Sep 17 00:00:00 2001 From: seawinde Date: Wed, 24 Apr 2024 10:10:43 +0800 Subject: [PATCH 2/4] continue optimizing --- .../java/org/apache/doris/nereids/memo/StructInfoMap.java | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/StructInfoMap.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/StructInfoMap.java index 21ef82f96fda11..ad2d47fb6f21ba 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/StructInfoMap.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/StructInfoMap.java @@ -136,6 +136,14 @@ public boolean refresh(Group group) { refreshedGroup.add(child); childrenTableMap.add(child.getstructInfoMap().getTableMaps()); } + // if one same groupExpression have refreshed, continue + BitSet oneOfGroupExpressionTableSet = new BitSet(); + for (Set groupExpressionBitSet : childrenTableMap) { + oneOfGroupExpressionTableSet.or(groupExpressionBitSet.iterator().next()); + } + if (groupExpressionMap.containsKey(oneOfGroupExpressionTableSet)) { + continue; + } // if cumulative child table map is different from current // or current group expression map is empty, should update the groupExpressionMap currently Collection>> bitSetWithChildren = cartesianProduct(childrenTableMap); From cb3c0afe5a3e9e989f15dfa9ced43df737e9ed57 Mon Sep 17 00:00:00 2001 From: seawinde Date: Wed, 24 Apr 2024 10:32:08 +0800 Subject: [PATCH 3/4] continue again --- .../java/org/apache/doris/nereids/memo/StructInfoMap.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/StructInfoMap.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/StructInfoMap.java index ad2d47fb6f21ba..4119c6f2f89967 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/StructInfoMap.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/StructInfoMap.java @@ -29,6 +29,7 @@ import java.util.Collection; import java.util.HashMap; import java.util.HashSet; +import java.util.Iterator; import java.util.LinkedList; import java.util.List; import java.util.Map; @@ -139,7 +140,10 @@ public boolean refresh(Group group) { // if one same groupExpression have refreshed, continue BitSet oneOfGroupExpressionTableSet = new BitSet(); for (Set groupExpressionBitSet : childrenTableMap) { - oneOfGroupExpressionTableSet.or(groupExpressionBitSet.iterator().next()); + Iterator iterator = groupExpressionBitSet.iterator(); + if (iterator.hasNext()) { + oneOfGroupExpressionTableSet.or(iterator.next()); + } } if (groupExpressionMap.containsKey(oneOfGroupExpressionTableSet)) { continue; From 8c0f81fd2bb86bbec19e221fd37ff2d0e13216d6 Mon Sep 17 00:00:00 2001 From: seawinde Date: Wed, 24 Apr 2024 15:53:29 +0800 Subject: [PATCH 4/4] optimize contains all --- .../nereids/rules/exploration/mv/StructInfo.java | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java index 4979f11c28e5db..757004071ee4d3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java @@ -443,10 +443,18 @@ public Void visit(Plan plan, PlanSplitContext context) { } } + /** Judge if source contains all target */ public static boolean containsAll(BitSet source, BitSet target) { - BitSet intersection = (BitSet) source.clone(); - intersection.and(target); - return intersection.equals(target); + if (source.size() < target.size()) { + return false; + } + for (int i = target.nextSetBit(0); i >= 0; i = target.nextSetBit(i + 1)) { + boolean contains = source.get(i); + if (!contains) { + return false; + } + } + return true; } /**