diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Memo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Memo.java
index 8793cb5be51f75..1e6b285187d6a6 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Memo.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Memo.java
@@ -416,7 +416,9 @@ private CopyInResult doCopyIn(Plan plan, @Nullable Group targetGroup, @Nullable
             throw new IllegalStateException("Insert a plan into targetGroup but differ in logicalproperties");
         }
         // TODO Support sync materialized view in the future
-        if (plan instanceof CatalogRelation && ((CatalogRelation) plan).getTable() instanceof MTMV) {
+        if (plan instanceof LogicalPlan && plan instanceof CatalogRelation
+                && ((CatalogRelation) plan).getTable() instanceof MTMV
+                && !plan.getGroupExpression().isPresent()) {
             refreshVersion.incrementAndGet();
         }
         Optional<GroupExpression> groupExpr = plan.getGroupExpression();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/StructInfoMap.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/StructInfoMap.java
index efa2bef1792417..ae07c2043abaf7 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/StructInfoMap.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/StructInfoMap.java
@@ -59,7 +59,7 @@ public class StructInfoMap {
             return structInfo;
         }
         if (groupExpressionMap.isEmpty() || !groupExpressionMap.containsKey(tableMap)) {
-            refresh(group, memo.getRefreshVersion(), foldTableMap);
+            refresh(group, memo.getRefreshVersion());
             group.getstructInfoMap().setRefreshVersion(memo.getRefreshVersion());
         }
         if (groupExpressionMap.containsKey(tableMap)) {
@@ -118,7 +118,10 @@ private Plan constructPlan(GroupExpression groupExpression, List<BitSet> childre
      * @param group the root group
      *
      */
-    public void refresh(Group group, long refreshVersion, BitSet targetBitSet) {
+    public void refresh(Group group, long memoVersion) {
+        if (memoVersion == group.getstructInfoMap().refreshVersion) {
+            return;
+        }
         Set<Integer> refreshedGroup = new HashSet<>();
         for (GroupExpression groupExpression : group.getLogicalExpressions()) {
             List<Set<BitSet>> childrenTableMap = new LinkedList<>();
@@ -129,10 +132,9 @@ public void refresh(Group group, long refreshVersion, BitSet targetBitSet) {
             }
             for (Group child : groupExpression.children()) {
                 StructInfoMap childStructInfoMap = child.getstructInfoMap();
-                if (!refreshedGroup.contains(child.getGroupId().asInt())
-                        && refreshVersion != childStructInfoMap.getRefreshVersion()) {
-                    childStructInfoMap.refresh(child, refreshVersion, targetBitSet);
-                    childStructInfoMap.setRefreshVersion(refreshVersion);
+                if (!refreshedGroup.contains(child.getGroupId().asInt())) {
+                    childStructInfoMap.refresh(child, memoVersion);
+                    childStructInfoMap.setRefreshVersion(memoVersion);
                 }
                 refreshedGroup.add(child.getGroupId().asInt());
                 childrenTableMap.add(child.getstructInfoMap().getTableMaps());
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java
index 5f7dc419eafd22..73029b4ec141d3 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java
@@ -150,8 +150,7 @@ public static List<StructInfo> extractStructInfo(Plan plan, CascadesContext casc
             StructInfoMap structInfoMap = ownerGroup.getstructInfoMap();
             if (cascadesContext.getMemo().getRefreshVersion() != structInfoMap.getRefreshVersion()
                     || structInfoMap.getTableMaps().isEmpty()) {
-                structInfoMap.refresh(ownerGroup, cascadesContext.getMemo().getRefreshVersion(),
-                        materializedViewTableSet);
+                structInfoMap.refresh(ownerGroup, cascadesContext.getMemo().getRefreshVersion());
                 structInfoMap.setRefreshVersion(cascadesContext.getMemo().getRefreshVersion());
             }
             Set<BitSet> queryTableSets = structInfoMap.getTableMaps();
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/memo/StructInfoMapTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/memo/StructInfoMapTest.java
index 9192f86cf3bb41..217cbf2f1a28e2 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/memo/StructInfoMapTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/memo/StructInfoMapTest.java
@@ -50,7 +50,7 @@ void testTableMap() throws Exception {
         Group root = c1.getMemo().getRoot();
         Set<BitSet> tableMaps = root.getstructInfoMap().getTableMaps();
         Assertions.assertTrue(tableMaps.isEmpty());
-        root.getstructInfoMap().refresh(root, 1, new BitSet());
+        root.getstructInfoMap().refresh(root, 1);
         Assertions.assertEquals(1, tableMaps.size());
         new MockUp<MTMVRelationManager>() {
             @Mock
@@ -76,7 +76,7 @@ public boolean isMVPartitionValid(MTMV mtmv, ConnectContext ctx) {
                 .optimize()
                 .printlnBestPlanTree();
         root = c1.getMemo().getRoot();
-        root.getstructInfoMap().refresh(root, 1, new BitSet());
+        root.getstructInfoMap().refresh(root, 1);
         tableMaps = root.getstructInfoMap().getTableMaps();
         Assertions.assertEquals(2, tableMaps.size());
         dropMvByNereids("drop materialized view mv1");
@@ -97,8 +97,8 @@ void testLazyRefresh() throws Exception {
         Group root = c1.getMemo().getRoot();
         Set<BitSet> tableMaps = root.getstructInfoMap().getTableMaps();
         Assertions.assertTrue(tableMaps.isEmpty());
-        root.getstructInfoMap().refresh(root, 1, new BitSet());
-        root.getstructInfoMap().refresh(root, 1, new BitSet());
+        root.getstructInfoMap().refresh(root, 1);
+        root.getstructInfoMap().refresh(root, 1);
         Assertions.assertEquals(1, tableMaps.size());
         new MockUp<MTMVRelationManager>() {
             @Mock
@@ -124,8 +124,8 @@ public boolean isMVPartitionValid(MTMV mtmv, ConnectContext ctx) {
                 .optimize()
                 .printlnBestPlanTree();
         root = c1.getMemo().getRoot();
-        root.getstructInfoMap().refresh(root, 1, new BitSet());
-        root.getstructInfoMap().refresh(root, 1, new BitSet());
+        root.getstructInfoMap().refresh(root, 1);
+        root.getstructInfoMap().refresh(root, 1);
         tableMaps = root.getstructInfoMap().getTableMaps();
         Assertions.assertEquals(2, tableMaps.size());
         dropMvByNereids("drop materialized view mv1");
@@ -162,7 +162,7 @@ public boolean isMVPartitionValid(MTMV mtmv, ConnectContext ctx) {
                 .rewrite()
                 .optimize();
         Group root = c1.getMemo().getRoot();
-        root.getstructInfoMap().refresh(root, 1, new BitSet());
+        root.getstructInfoMap().refresh(root, 1);
         StructInfoMap structInfoMap = root.getstructInfoMap();
         Assertions.assertEquals(2, structInfoMap.getTableMaps().size());
         BitSet mvMap = structInfoMap.getTableMaps().stream()