Skip to content

[improvement](mtmv)Optimize nested mv performance final pick 213 #34192

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
yiguolei merged 1 commit into from
Apr 27, 2024
Merged

[improvement](mtmv)Optimize nested mv performance final pick 213 #34192

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Memo.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -416,7 +416,9 @@ private CopyInResult doCopyIn(Plan plan, @Nullable Group targetGroup, @Nullable
throw new IllegalStateException("Insert a plan into targetGroup but differ in logicalproperties");
}
// TODO Support sync materialized view in the future
if (plan instanceof CatalogRelation && ((CatalogRelation) plan).getTable() instanceof MTMV) {
if (plan instanceof LogicalPlan && plan instanceof CatalogRelation
&& ((CatalogRelation) plan).getTable() instanceof MTMV
&& !plan.getGroupExpression().isPresent()) {
refreshVersion.incrementAndGet();
}
Optional<GroupExpression> groupExpr = plan.getGroupExpression();
Expand Down
14 changes: 8 additions & 6 deletions fe/fe-core/src/main/java/org/apache/doris/nereids/memo/StructInfoMap.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@ public class StructInfoMap {
return structInfo;
}
if (groupExpressionMap.isEmpty() || !groupExpressionMap.containsKey(tableMap)) {
refresh(group, memo.getRefreshVersion(), foldTableMap);
refresh(group, memo.getRefreshVersion());
group.getstructInfoMap().setRefreshVersion(memo.getRefreshVersion());
}
if (groupExpressionMap.containsKey(tableMap)) {
Expand Down Expand Up @@ -118,7 +118,10 @@ private Plan constructPlan(GroupExpression groupExpression, List<BitSet> childre
* @param group the root group
*
*/
public void refresh(Group group, long refreshVersion, BitSet targetBitSet) {
public void refresh(Group group, long memoVersion) {
if (memoVersion == group.getstructInfoMap().refreshVersion) {
return;
}
Set<Integer> refreshedGroup = new HashSet<>();
for (GroupExpression groupExpression : group.getLogicalExpressions()) {
List<Set<BitSet>> childrenTableMap = new LinkedList<>();
Expand All @@ -129,10 +132,9 @@ public void refresh(Group group, long refreshVersion, BitSet targetBitSet) {
}
for (Group child : groupExpression.children()) {
StructInfoMap childStructInfoMap = child.getstructInfoMap();
if (!refreshedGroup.contains(child.getGroupId().asInt())
&& refreshVersion != childStructInfoMap.getRefreshVersion()) {
childStructInfoMap.refresh(child, refreshVersion, targetBitSet);
childStructInfoMap.setRefreshVersion(refreshVersion);
if (!refreshedGroup.contains(child.getGroupId().asInt())) {
childStructInfoMap.refresh(child, memoVersion);
childStructInfoMap.setRefreshVersion(memoVersion);
}
refreshedGroup.add(child.getGroupId().asInt());
childrenTableMap.add(child.getstructInfoMap().getTableMaps());
Expand Down
3 changes: 1 addition & 2 deletions ...re/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -150,8 +150,7 @@ public static List<StructInfo> extractStructInfo(Plan plan, CascadesContext casc
StructInfoMap structInfoMap = ownerGroup.getstructInfoMap();
if (cascadesContext.getMemo().getRefreshVersion() != structInfoMap.getRefreshVersion()
|| structInfoMap.getTableMaps().isEmpty()) {
structInfoMap.refresh(ownerGroup, cascadesContext.getMemo().getRefreshVersion(),
materializedViewTableSet);
structInfoMap.refresh(ownerGroup, cascadesContext.getMemo().getRefreshVersion());
structInfoMap.setRefreshVersion(cascadesContext.getMemo().getRefreshVersion());
}
Set<BitSet> queryTableSets = structInfoMap.getTableMaps();
Expand Down
14 changes: 7 additions & 7 deletions fe/fe-core/src/test/java/org/apache/doris/nereids/memo/StructInfoMapTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ void testTableMap() throws Exception {
Group root = c1.getMemo().getRoot();
Set<BitSet> tableMaps = root.getstructInfoMap().getTableMaps();
Assertions.assertTrue(tableMaps.isEmpty());
root.getstructInfoMap().refresh(root, 1, new BitSet());
root.getstructInfoMap().refresh(root, 1);
Assertions.assertEquals(1, tableMaps.size());
new MockUp<MTMVRelationManager>() {
@Mock
Expand All @@ -76,7 +76,7 @@ public boolean isMVPartitionValid(MTMV mtmv, ConnectContext ctx) {
.optimize()
.printlnBestPlanTree();
root = c1.getMemo().getRoot();
root.getstructInfoMap().refresh(root, 1, new BitSet());
root.getstructInfoMap().refresh(root, 1);
tableMaps = root.getstructInfoMap().getTableMaps();
Assertions.assertEquals(2, tableMaps.size());
dropMvByNereids("drop materialized view mv1");
Expand All @@ -97,8 +97,8 @@ void testLazyRefresh() throws Exception {
Group root = c1.getMemo().getRoot();
Set<BitSet> tableMaps = root.getstructInfoMap().getTableMaps();
Assertions.assertTrue(tableMaps.isEmpty());
root.getstructInfoMap().refresh(root, 1, new BitSet());
root.getstructInfoMap().refresh(root, 1, new BitSet());
root.getstructInfoMap().refresh(root, 1);
root.getstructInfoMap().refresh(root, 1);
Assertions.assertEquals(1, tableMaps.size());
new MockUp<MTMVRelationManager>() {
@Mock
Expand All @@ -124,8 +124,8 @@ public boolean isMVPartitionValid(MTMV mtmv, ConnectContext ctx) {
.optimize()
.printlnBestPlanTree();
root = c1.getMemo().getRoot();
root.getstructInfoMap().refresh(root, 1, new BitSet());
root.getstructInfoMap().refresh(root, 1, new BitSet());
root.getstructInfoMap().refresh(root, 1);
root.getstructInfoMap().refresh(root, 1);
tableMaps = root.getstructInfoMap().getTableMaps();
Assertions.assertEquals(2, tableMaps.size());
dropMvByNereids("drop materialized view mv1");
Expand Down Expand Up @@ -162,7 +162,7 @@ public boolean isMVPartitionValid(MTMV mtmv, ConnectContext ctx) {
.rewrite()
.optimize();
Group root = c1.getMemo().getRoot();
root.getstructInfoMap().refresh(root, 1, new BitSet());
root.getstructInfoMap().refresh(root, 1);
StructInfoMap structInfoMap = root.getstructInfoMap();
Assertions.assertEquals(2, structInfoMap.getTableMaps().size());
BitSet mvMap = structInfoMap.getTableMaps().stream()
Expand Down