From 48b2237a85233b5b094744165cde8d665eee73f8 Mon Sep 17 00:00:00 2001 From: seawinde Date: Mon, 10 Mar 2025 15:01:53 +0800 Subject: [PATCH] [opt](mtmv) Opt materialized view rewrite performance when the num of struct infos are huge (#48782) ### What problem does this PR solve? Opt materialized view rewrite performance when the num of struct infos are huge Optimize the recursive algorithm to reduce the number of recursive calls. If a group has already been refreshed, skip subsequent refreshes. --- .../apache/doris/nereids/memo/StructInfoMap.java | 9 ++++----- .../exploration/mv/MaterializedViewUtils.java | 2 +- .../doris/nereids/memo/StructInfoMapTest.java | 15 ++++++++------- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/StructInfoMap.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/StructInfoMap.java index 4aa4f146b874da..c39be5b569089f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/StructInfoMap.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/StructInfoMap.java @@ -60,7 +60,7 @@ public class StructInfoMap { return structInfo; } if (groupExpressionMap.isEmpty() || !groupExpressionMap.containsKey(tableMap)) { - refresh(group, cascadesContext); + refresh(group, cascadesContext, new HashSet<>()); group.getstructInfoMap().setRefreshVersion(cascadesContext.getMemo().getRefreshVersion()); } if (groupExpressionMap.containsKey(tableMap)) { @@ -116,13 +116,13 @@ private Plan constructPlan(GroupExpression groupExpression, List childre * @param group the root group * */ - public void refresh(Group group, CascadesContext cascadesContext) { + public void refresh(Group group, CascadesContext cascadesContext, Set refreshedGroup) { StructInfoMap structInfoMap = group.getstructInfoMap(); + refreshedGroup.add(group.getGroupId().asInt()); long memoVersion = cascadesContext.getMemo().getRefreshVersion(); if (!structInfoMap.getTableMaps().isEmpty() && memoVersion == structInfoMap.refreshVersion) { return; } - Set refreshedGroup = new HashSet<>(); for (GroupExpression groupExpression : group.getLogicalExpressions()) { List> childrenTableMap = new LinkedList<>(); if (groupExpression.children().isEmpty()) { @@ -136,10 +136,9 @@ public void refresh(Group group, CascadesContext cascadesContext) { for (Group child : groupExpression.children()) { StructInfoMap childStructInfoMap = child.getstructInfoMap(); if (!refreshedGroup.contains(child.getGroupId().asInt())) { - childStructInfoMap.refresh(child, cascadesContext); + childStructInfoMap.refresh(child, cascadesContext, refreshedGroup); childStructInfoMap.setRefreshVersion(memoVersion); } - refreshedGroup.add(child.getGroupId().asInt()); childrenTableMap.add(child.getstructInfoMap().getTableMaps()); } // if one same groupExpression have refreshed, continue diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java index 2a37ee6219da07..65208f3aeb9bb4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java @@ -194,7 +194,7 @@ public static List extractStructInfo(Plan plan, Plan originalPlan, C Group ownerGroup = plan.getGroupExpression().get().getOwnerGroup(); StructInfoMap structInfoMap = ownerGroup.getstructInfoMap(); // Refresh struct info in current level plan from top to bottom - structInfoMap.refresh(ownerGroup, cascadesContext); + structInfoMap.refresh(ownerGroup, cascadesContext, new HashSet<>()); structInfoMap.setRefreshVersion(cascadesContext.getMemo().getRefreshVersion()); Set queryTableSets = structInfoMap.getTableMaps(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/memo/StructInfoMapTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/memo/StructInfoMapTest.java index db77da76c4b6cb..6b10176e22aa9c 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/memo/StructInfoMapTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/memo/StructInfoMapTest.java @@ -33,6 +33,7 @@ import org.junit.jupiter.api.Test; import java.util.BitSet; +import java.util.HashSet; import java.util.Set; import java.util.stream.Collectors; @@ -60,7 +61,7 @@ public BitSet getDisableNereidsRules() { Group root = c1.getMemo().getRoot(); Set tableMaps = root.getstructInfoMap().getTableMaps(); Assertions.assertTrue(tableMaps.isEmpty()); - root.getstructInfoMap().refresh(root, c1); + root.getstructInfoMap().refresh(root, c1, new HashSet<>()); Assertions.assertEquals(1, tableMaps.size()); new MockUp() { @Mock @@ -88,7 +89,7 @@ public boolean isMVPartitionValid(MTMV mtmv, ConnectContext ctx, boolean forceCo .optimize() .printlnBestPlanTree(); root = c1.getMemo().getRoot(); - root.getstructInfoMap().refresh(root, c1); + root.getstructInfoMap().refresh(root, c1, new HashSet<>()); tableMaps = root.getstructInfoMap().getTableMaps(); Assertions.assertEquals(2, tableMaps.size()); dropMvByNereids("drop materialized view mv1"); @@ -117,8 +118,8 @@ public BitSet getDisableNereidsRules() { Group root = c1.getMemo().getRoot(); Set tableMaps = root.getstructInfoMap().getTableMaps(); Assertions.assertTrue(tableMaps.isEmpty()); - root.getstructInfoMap().refresh(root, c1); - root.getstructInfoMap().refresh(root, c1); + root.getstructInfoMap().refresh(root, c1, new HashSet<>()); + root.getstructInfoMap().refresh(root, c1, new HashSet<>()); Assertions.assertEquals(1, tableMaps.size()); new MockUp() { @Mock @@ -145,8 +146,8 @@ public boolean isMVPartitionValid(MTMV mtmv, ConnectContext ctx, boolean isMVPar .optimize() .printlnBestPlanTree(); root = c1.getMemo().getRoot(); - root.getstructInfoMap().refresh(root, c1); - root.getstructInfoMap().refresh(root, c1); + root.getstructInfoMap().refresh(root, c1, new HashSet<>()); + root.getstructInfoMap().refresh(root, c1, new HashSet<>()); tableMaps = root.getstructInfoMap().getTableMaps(); Assertions.assertEquals(2, tableMaps.size()); dropMvByNereids("drop materialized view mv1"); @@ -192,7 +193,7 @@ public boolean isMVPartitionValid(MTMV mtmv, ConnectContext ctx, boolean isMVPar .rewrite() .optimize(); Group root = c1.getMemo().getRoot(); - root.getstructInfoMap().refresh(root, c1); + root.getstructInfoMap().refresh(root, c1, new HashSet<>()); StructInfoMap structInfoMap = root.getstructInfoMap(); Assertions.assertEquals(2, structInfoMap.getTableMaps().size()); BitSet mvMap = structInfoMap.getTableMaps().stream()