From b90c331c1b1da8ab04d528f5618853aff5467362 Mon Sep 17 00:00:00 2001 From: seawinde <149132972+seawinde@users.noreply.github.com> Date: Fri, 26 Apr 2024 19:31:09 +0800 Subject: [PATCH] [improvement](mtmv) Optimize the nested materialized view performance (#34163) Record increase refersh version more accurately. The refreshVersion in the memo will increase when mv rewrite successfully. In query rewrite, if refresh version is different from the current struct info map in group, will refresh the group struct info or not. --- .../java/org/apache/doris/nereids/memo/Memo.java | 4 +++- .../apache/doris/nereids/memo/StructInfoMap.java | 14 ++++++++------ .../exploration/mv/MaterializedViewUtils.java | 3 +-- .../doris/nereids/memo/StructInfoMapTest.java | 14 +++++++------- 4 files changed, 19 insertions(+), 16 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Memo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Memo.java index 8793cb5be51f75..1e6b285187d6a6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Memo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Memo.java @@ -416,7 +416,9 @@ private CopyInResult doCopyIn(Plan plan, @Nullable Group targetGroup, @Nullable throw new IllegalStateException("Insert a plan into targetGroup but differ in logicalproperties"); } // TODO Support sync materialized view in the future - if (plan instanceof CatalogRelation && ((CatalogRelation) plan).getTable() instanceof MTMV) { + if (plan instanceof LogicalPlan && plan instanceof CatalogRelation + && ((CatalogRelation) plan).getTable() instanceof MTMV + && !plan.getGroupExpression().isPresent()) { refreshVersion.incrementAndGet(); } Optional groupExpr = plan.getGroupExpression(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/StructInfoMap.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/StructInfoMap.java index efa2bef1792417..ae07c2043abaf7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/StructInfoMap.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/StructInfoMap.java @@ -59,7 +59,7 @@ public class StructInfoMap { return structInfo; } if (groupExpressionMap.isEmpty() || !groupExpressionMap.containsKey(tableMap)) { - refresh(group, memo.getRefreshVersion(), foldTableMap); + refresh(group, memo.getRefreshVersion()); group.getstructInfoMap().setRefreshVersion(memo.getRefreshVersion()); } if (groupExpressionMap.containsKey(tableMap)) { @@ -118,7 +118,10 @@ private Plan constructPlan(GroupExpression groupExpression, List childre * @param group the root group * */ - public void refresh(Group group, long refreshVersion, BitSet targetBitSet) { + public void refresh(Group group, long memoVersion) { + if (memoVersion == group.getstructInfoMap().refreshVersion) { + return; + } Set refreshedGroup = new HashSet<>(); for (GroupExpression groupExpression : group.getLogicalExpressions()) { List> childrenTableMap = new LinkedList<>(); @@ -129,10 +132,9 @@ public void refresh(Group group, long refreshVersion, BitSet targetBitSet) { } for (Group child : groupExpression.children()) { StructInfoMap childStructInfoMap = child.getstructInfoMap(); - if (!refreshedGroup.contains(child.getGroupId().asInt()) - && refreshVersion != childStructInfoMap.getRefreshVersion()) { - childStructInfoMap.refresh(child, refreshVersion, targetBitSet); - childStructInfoMap.setRefreshVersion(refreshVersion); + if (!refreshedGroup.contains(child.getGroupId().asInt())) { + childStructInfoMap.refresh(child, memoVersion); + childStructInfoMap.setRefreshVersion(memoVersion); } refreshedGroup.add(child.getGroupId().asInt()); childrenTableMap.add(child.getstructInfoMap().getTableMaps()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java index 5f7dc419eafd22..73029b4ec141d3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java @@ -150,8 +150,7 @@ public static List extractStructInfo(Plan plan, CascadesContext casc StructInfoMap structInfoMap = ownerGroup.getstructInfoMap(); if (cascadesContext.getMemo().getRefreshVersion() != structInfoMap.getRefreshVersion() || structInfoMap.getTableMaps().isEmpty()) { - structInfoMap.refresh(ownerGroup, cascadesContext.getMemo().getRefreshVersion(), - materializedViewTableSet); + structInfoMap.refresh(ownerGroup, cascadesContext.getMemo().getRefreshVersion()); structInfoMap.setRefreshVersion(cascadesContext.getMemo().getRefreshVersion()); } Set queryTableSets = structInfoMap.getTableMaps(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/memo/StructInfoMapTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/memo/StructInfoMapTest.java index 9192f86cf3bb41..217cbf2f1a28e2 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/memo/StructInfoMapTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/memo/StructInfoMapTest.java @@ -50,7 +50,7 @@ void testTableMap() throws Exception { Group root = c1.getMemo().getRoot(); Set tableMaps = root.getstructInfoMap().getTableMaps(); Assertions.assertTrue(tableMaps.isEmpty()); - root.getstructInfoMap().refresh(root, 1, new BitSet()); + root.getstructInfoMap().refresh(root, 1); Assertions.assertEquals(1, tableMaps.size()); new MockUp() { @Mock @@ -76,7 +76,7 @@ public boolean isMVPartitionValid(MTMV mtmv, ConnectContext ctx) { .optimize() .printlnBestPlanTree(); root = c1.getMemo().getRoot(); - root.getstructInfoMap().refresh(root, 1, new BitSet()); + root.getstructInfoMap().refresh(root, 1); tableMaps = root.getstructInfoMap().getTableMaps(); Assertions.assertEquals(2, tableMaps.size()); dropMvByNereids("drop materialized view mv1"); @@ -97,8 +97,8 @@ void testLazyRefresh() throws Exception { Group root = c1.getMemo().getRoot(); Set tableMaps = root.getstructInfoMap().getTableMaps(); Assertions.assertTrue(tableMaps.isEmpty()); - root.getstructInfoMap().refresh(root, 1, new BitSet()); - root.getstructInfoMap().refresh(root, 1, new BitSet()); + root.getstructInfoMap().refresh(root, 1); + root.getstructInfoMap().refresh(root, 1); Assertions.assertEquals(1, tableMaps.size()); new MockUp() { @Mock @@ -124,8 +124,8 @@ public boolean isMVPartitionValid(MTMV mtmv, ConnectContext ctx) { .optimize() .printlnBestPlanTree(); root = c1.getMemo().getRoot(); - root.getstructInfoMap().refresh(root, 1, new BitSet()); - root.getstructInfoMap().refresh(root, 1, new BitSet()); + root.getstructInfoMap().refresh(root, 1); + root.getstructInfoMap().refresh(root, 1); tableMaps = root.getstructInfoMap().getTableMaps(); Assertions.assertEquals(2, tableMaps.size()); dropMvByNereids("drop materialized view mv1"); @@ -162,7 +162,7 @@ public boolean isMVPartitionValid(MTMV mtmv, ConnectContext ctx) { .rewrite() .optimize(); Group root = c1.getMemo().getRoot(); - root.getstructInfoMap().refresh(root, 1, new BitSet()); + root.getstructInfoMap().refresh(root, 1); StructInfoMap structInfoMap = root.getstructInfoMap(); Assertions.assertEquals(2, structInfoMap.getTableMaps().size()); BitSet mvMap = structInfoMap.getTableMaps().stream()