From eb2494e18a5a32816b4654d55869827bd26ecfa9 Mon Sep 17 00:00:00 2001 From: seawinde Date: Thu, 4 Dec 2025 14:20:09 +0800 Subject: [PATCH 1/4] [fix](mtmv) Fix hudi materialized view union all rewritten plan execute fail because of invalid slot (#58643) Related PR: #57558 #58413 Problem Summary: This fix addresses the following three issues: 1. When invoking the method org.apache.doris.nereids.trees.plans.logical.LogicalHudiScan#withRelationId, the output needs to be recalculated to meet expectations. 2. After compensating with a union all due to partial partition invalidation of a materialized view, during the next round of transparent rewriting, the rewriting for the child of the union allshould use the query partitioncorresponding to the specific relation id to prevent infinite loops. 3. Currently, in the `test_hudi_rewrite_mtmv` test, if the plan rewritten by the materialized view transparent rewriting is not selected by the CBO, it is difficult to troubleshoot because explain memo planis not used. Therefore, the corresponding test method is modified. --- .../mv/AbstractMaterializedViewRule.java | 2 +- .../exploration/mv/PartitionCompensator.java | 40 +++- .../mv/PartitionCompensatorTest.java | 199 ++++++++++++++++++ .../hudi_mtmv/test_hudi_rewrite_mtmv.groovy | 8 +- 4 files changed, 238 insertions(+), 11 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java index 3a6f08f8cc8a1f..39f164f48b110f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java @@ -295,7 +295,7 @@ protected List doRewrite(StructInfo queryStructInfo, CascadesContext casca continue; } Pair>, Map>> invalidPartitions; - if (PartitionCompensator.needUnionRewrite(materializationContext) + if (PartitionCompensator.needUnionRewrite(materializationContext, cascadesContext.getStatementContext()) && sessionVariable.isEnableMaterializedViewUnionRewrite()) { MTMV mtmv = ((AsyncMaterializationContext) materializationContext).getMtmv(); BaseTableInfo relatedTableInfo = mtmv.getMvPartitionInfo().getRelatedTableInfo(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PartitionCompensator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PartitionCompensator.java index 3c086de0919cff..8ff3975bc9cf0a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PartitionCompensator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PartitionCompensator.java @@ -23,8 +23,10 @@ import org.apache.doris.catalog.PartitionType; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.Pair; +import org.apache.doris.datasource.ExternalTable; import org.apache.doris.mtmv.BaseTableInfo; import org.apache.doris.mtmv.MTMVPartitionInfo; +import org.apache.doris.mtmv.MTMVRelatedTableIf; import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.StatementContext; import org.apache.doris.nereids.trees.plans.Plan; @@ -151,14 +153,37 @@ public static boolean needUnionRewrite( /** * Check if need union compensate or not + * If query base table all partitions with ALL_PARTITIONS or ALL_PARTITIONS_LIST, should not do union compensate + * because it means query all partitions from base table and prune partition failed */ - public static boolean needUnionRewrite(MaterializationContext materializationContext) { + public static boolean needUnionRewrite(MaterializationContext materializationContext, + StatementContext statementContext) throws AnalysisException { if (!(materializationContext instanceof AsyncMaterializationContext)) { return false; } MTMV mtmv = ((AsyncMaterializationContext) materializationContext).getMtmv(); - PartitionType type = mtmv.getPartitionInfo().getType(); BaseTableInfo relatedTableInfo = mtmv.getMvPartitionInfo().getRelatedTableInfo(); + PartitionType type = mtmv.getPartitionInfo().getType(); + MTMVPartitionInfo mvPartitionInfo = mtmv.getMvPartitionInfo(); + MTMVRelatedTableIf pctTable = mvPartitionInfo.getRelatedTable(); + Multimap, Pair>> tableUsedPartitionNameMap = + statementContext.getTableUsedPartitionNameMap(); + if (pctTable instanceof ExternalTable && !((ExternalTable) pctTable).supportInternalPartitionPruned()) { + // if pct table is external table and not support internal partition pruned, + // we consider query all partitions from pct table, this would cause loop union compensate, + // so we skip union compensate in this case + return false; + } + Collection>> tableUsedPartitions + = tableUsedPartitionNameMap.get(pctTable.getFullQualifiers()); + if (ALL_PARTITIONS_LIST.equals(tableUsedPartitions) + || tableUsedPartitions.stream().anyMatch(ALL_PARTITIONS::equals)) { + // If query base table all partitions with ALL_PARTITIONS or ALL_PARTITIONS_LIST, + // should not do union compensate, because it means query all partitions from base table + // and prune partition failed + return false; + } + return !PartitionType.UNPARTITIONED.equals(type) && relatedTableInfo != null; } @@ -166,11 +191,11 @@ public static boolean needUnionRewrite(MaterializationContext materializationCon * Get query used partitions * this is calculated from tableUsedPartitionNameMap and tables in statementContext * - * @param customRelationIdSet if union compensate occurs, the new query used partitions is changed, + * @param currentUsedRelationIdSet if union compensate occurs, the new query used partitions is changed, * so need to get used partitions by relation id set */ public static Map, Set> getQueryUsedPartitions(StatementContext statementContext, - BitSet customRelationIdSet) { + BitSet currentUsedRelationIdSet) { // get table used partitions // if table is not in statementContext().getTables() which means the table is partition prune as empty relation Multimap, Pair>> tableUsedPartitionNameMap = statementContext @@ -195,6 +220,13 @@ public static Map, Set> getQueryUsedPartitions(StatementCon queryUsedRelatedTablePartitionsMap.put(queryUsedTable, null); continue tableLoop; } + // If currentUsedRelationIdSet is not empty, need check relation id to get concrete used partitions + BitSet usedPartitionRelation = new BitSet(); + usedPartitionRelation.set(tableUsedPartitionPair.key().asInt()); + if (!currentUsedRelationIdSet.isEmpty() + && !currentUsedRelationIdSet.intersects(usedPartitionRelation)) { + continue; + } usedPartitionSet.addAll(tableUsedPartitionPair.value()); } queryUsedRelatedTablePartitionsMap.put(queryUsedTable, usedPartitionSet); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/PartitionCompensatorTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/PartitionCompensatorTest.java index 25c0a679d8d388..b1505dd69a4f42 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/PartitionCompensatorTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/PartitionCompensatorTest.java @@ -17,21 +17,39 @@ package org.apache.doris.nereids.rules.exploration.mv; +import org.apache.doris.catalog.DatabaseIf; +import org.apache.doris.catalog.MTMV; +import org.apache.doris.catalog.PartitionInfo; +import org.apache.doris.catalog.PartitionType; +import org.apache.doris.catalog.TableIf; +import org.apache.doris.common.AnalysisException; import org.apache.doris.common.Pair; +import org.apache.doris.datasource.CatalogIf; +import org.apache.doris.datasource.hive.HMSExternalTable; +import org.apache.doris.mtmv.BaseColInfo; +import org.apache.doris.mtmv.BaseTableInfo; +import org.apache.doris.mtmv.MTMVPartitionInfo; +import org.apache.doris.mtmv.MTMVRelatedTableIf; +import org.apache.doris.nereids.StatementContext; import org.apache.doris.nereids.trees.plans.RelationId; import org.apache.doris.nereids.util.PlanChecker; import org.apache.doris.utframe.TestWithFeService; +import com.google.common.collect.ArrayListMultimap; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Multimap; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import org.mockito.Mockito; import java.util.BitSet; +import java.util.Collections; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.stream.Collectors; public class PartitionCompensatorTest extends TestWithFeService { @@ -190,4 +208,185 @@ public void testGetAllTableUsedPartitionList() { Assertions.assertEquals(orderTableUsedPartition, ImmutableSet.of("p1", "p2", "p3", "p4")); }); } + + @Test + public void testNeedUnionRewriteUnpartitionedOrNoPctInfos() throws Exception { + MaterializationContext ctx1 = mockCtx( + PartitionType.UNPARTITIONED, + ImmutableList.of(new BaseColInfo("c", newBaseTableInfo())), + ImmutableSet.of(), + false); + StatementContext sc1 = Mockito.mock(StatementContext.class); + Mockito.when(sc1.getTableUsedPartitionNameMap()).thenReturn(ArrayListMultimap.create()); + Assertions.assertFalse(PartitionCompensator.needUnionRewrite(ctx1, sc1)); + + MaterializationContext ctx2 = mockCtx( + PartitionType.RANGE, + Collections.emptyList(), + ImmutableSet.of(Mockito.mock(MTMVRelatedTableIf.class)), + false); + StatementContext sc2 = Mockito.mock(StatementContext.class); + Mockito.when(sc2.getTableUsedPartitionNameMap()).thenReturn(ArrayListMultimap.create()); + Assertions.assertFalse(PartitionCompensator.needUnionRewrite(ctx2, sc2)); + } + + @Test + public void testNeedUnionRewriteEmptyPctTables() throws Exception { + MaterializationContext ctx = mockCtx( + PartitionType.RANGE, + ImmutableList.of(), + Collections.emptySet(), + false); + StatementContext sc = Mockito.mock(StatementContext.class); + Mockito.when(sc.getTableUsedPartitionNameMap()).thenReturn(ArrayListMultimap.create()); + Assertions.assertFalse(PartitionCompensator.needUnionRewrite(ctx, sc)); + } + + @Test + public void testNeedUnionRewriteExternalNoPrune() throws Exception { + MaterializationContext ctx = mockCtx( + PartitionType.LIST, + ImmutableList.of(new BaseColInfo("c", newBaseTableInfo())), + ImmutableSet.of(Mockito.mock(MTMVRelatedTableIf.class)), + true); + StatementContext sc = Mockito.mock(StatementContext.class); + Mockito.when(sc.getTableUsedPartitionNameMap()).thenReturn(ArrayListMultimap.create()); + Assertions.assertFalse(PartitionCompensator.needUnionRewrite(ctx, sc)); + } + + @Test + public void testNeedUnionRewritePositive() throws Exception { + MaterializationContext ctx = mockCtx( + PartitionType.LIST, + ImmutableList.of(new BaseColInfo("c", newBaseTableInfo())), + ImmutableSet.of(Mockito.mock(MTMVRelatedTableIf.class)), + false); + StatementContext sc = Mockito.mock(StatementContext.class); + Mockito.when(sc.getTableUsedPartitionNameMap()).thenReturn(ArrayListMultimap.create()); + Assertions.assertTrue(PartitionCompensator.needUnionRewrite(ctx, sc)); + } + + @Test + public void testNotNeedUnionRewriteWhenAllPartitions() throws Exception { + BaseTableInfo tableInfo = newBaseTableInfo(); + MaterializationContext ctx = mockCtx( + PartitionType.LIST, + ImmutableList.of(new BaseColInfo("c", tableInfo)), + ImmutableSet.of(Mockito.mock(MTMVRelatedTableIf.class)), + false); + StatementContext sc = Mockito.mock(StatementContext.class); + + ArrayListMultimap, Pair>> t = ArrayListMultimap.create(); + t.put(ImmutableList.of(), PartitionCompensator.ALL_PARTITIONS); + Mockito.when(sc.getTableUsedPartitionNameMap()).thenReturn(t); + Assertions.assertFalse(PartitionCompensator.needUnionRewrite(ctx, sc)); + } + + @Test + public void testGetQueryUsedPartitionsAllAndPartial() { + // Prepare qualifiers + List lineitemQualifier = ImmutableList.of( + "internal", "partition_compensate_test", "lineitem_list_partition"); + List ordersQualifier = ImmutableList.of( + "internal", "partition_compensate_test", "orders_list_partition"); + + Multimap, Pair>> tableUsedPartitionNameMap + = connectContext.getStatementContext().getTableUsedPartitionNameMap(); + tableUsedPartitionNameMap.clear(); + + tableUsedPartitionNameMap.put(lineitemQualifier, PartitionCompensator.ALL_PARTITIONS); + + RelationId ridA = new RelationId(1); + RelationId ridB = new RelationId(2); + tableUsedPartitionNameMap.put(ordersQualifier, Pair.of(ridA, ImmutableSet.of("p1", "p2"))); + tableUsedPartitionNameMap.put(ordersQualifier, Pair.of(ridB, ImmutableSet.of("p3"))); + + Map, Set> result = PartitionCompensator.getQueryUsedPartitions( + connectContext.getStatementContext(), new BitSet()); + Assertions.assertNull(result.get(lineitemQualifier)); // all partitions + Assertions.assertEquals(ImmutableSet.of("p1", "p2", "p3"), result.get(ordersQualifier)); + + BitSet filterRidA = new BitSet(); + filterRidA.set(ridA.asInt()); + Map, Set> resultRidA = PartitionCompensator.getQueryUsedPartitions( + connectContext.getStatementContext(), filterRidA); + Assertions.assertNull(resultRidA.get(lineitemQualifier)); + Assertions.assertEquals(ImmutableSet.of("p1", "p2"), resultRidA.get(ordersQualifier)); + + BitSet filterRidB = new BitSet(); + filterRidB.set(ridB.asInt()); + Map, Set> resultRidB = PartitionCompensator.getQueryUsedPartitions( + connectContext.getStatementContext(), filterRidB); + Assertions.assertNull(resultRidB.get(lineitemQualifier)); + Assertions.assertEquals(ImmutableSet.of("p3"), resultRidB.get(ordersQualifier)); + + tableUsedPartitionNameMap.put(ordersQualifier, PartitionCompensator.ALL_PARTITIONS); + Map, Set> resultAllOrders = PartitionCompensator.getQueryUsedPartitions( + connectContext.getStatementContext(), new BitSet()); + Assertions.assertNull(resultAllOrders.get(ordersQualifier)); + } + + @Test + public void testGetQueryUsedPartitionsEmptyCollectionMeansNoPartitions() { + List qualifier = ImmutableList.of( + "internal", "partition_compensate_test", "lineitem_list_partition"); + Multimap, Pair>> tableUsedPartitionNameMap + = connectContext.getStatementContext().getTableUsedPartitionNameMap(); + tableUsedPartitionNameMap.clear(); + // Put an empty set via a distinct relation id to simulate no partitions used + RelationId rid = new RelationId(3); + tableUsedPartitionNameMap.put(qualifier, Pair.of(rid, ImmutableSet.of())); + + Map, Set> result = PartitionCompensator.getQueryUsedPartitions( + connectContext.getStatementContext(), new BitSet()); + Assertions.assertEquals(ImmutableSet.of(), result.get(qualifier)); + } + + private static MaterializationContext mockCtx( + PartitionType type, + List pctInfos, + Set pctTables, + boolean externalNoPrune) throws AnalysisException { + + MTMV mtmv = Mockito.mock(MTMV.class); + PartitionInfo pi = Mockito.mock(PartitionInfo.class); + Mockito.when(mtmv.getPartitionInfo()).thenReturn(pi); + Mockito.when(pi.getType()).thenReturn(type); + + MTMVPartitionInfo mpi = Mockito.mock(MTMVPartitionInfo.class); + Mockito.when(mtmv.getMvPartitionInfo()).thenReturn(mpi); + Mockito.when(mpi.getPctInfos()).thenReturn(pctInfos); + Mockito.when(mpi.getPctTables()).thenReturn(pctTables); + + if (externalNoPrune) { + HMSExternalTable ext = Mockito.mock(HMSExternalTable.class); + Mockito.when(ext.supportInternalPartitionPruned()).thenReturn(false); + Set tbls = new HashSet<>(pctTables); + tbls.add(ext); + Mockito.when(mpi.getPctTables()).thenReturn( + tbls.stream().map(MTMVRelatedTableIf.class::cast).collect(Collectors.toSet())); + } + + AsyncMaterializationContext ctx = Mockito.mock(AsyncMaterializationContext.class); + Mockito.when(ctx.getMtmv()).thenReturn(mtmv); + return ctx; + } + + private static BaseTableInfo newBaseTableInfo() { + CatalogIf catalog = Mockito.mock(CatalogIf.class); + Mockito.when(catalog.getId()).thenReturn(1L); + Mockito.when(catalog.getName()).thenReturn("internal"); + + DatabaseIf db = Mockito.mock(DatabaseIf.class); + Mockito.when(db.getId()).thenReturn(2L); + Mockito.when(db.getFullName()).thenReturn("partition_compensate_test"); + Mockito.when(db.getCatalog()).thenReturn(catalog); + + TableIf table = Mockito.mock(TableIf.class); + Mockito.when(table.getId()).thenReturn(3L); + Mockito.when(table.getName()).thenReturn("t"); + Mockito.when(table.getDatabase()).thenReturn(db); + + return new BaseTableInfo(table); + } } diff --git a/regression-test/suites/external_table_p2/hudi/hudi_mtmv/test_hudi_rewrite_mtmv.groovy b/regression-test/suites/external_table_p2/hudi/hudi_mtmv/test_hudi_rewrite_mtmv.groovy index 70a11d3633acf4..680f7eaa93d48c 100644 --- a/regression-test/suites/external_table_p2/hudi/hudi_mtmv/test_hudi_rewrite_mtmv.groovy +++ b/regression-test/suites/external_table_p2/hudi/hudi_mtmv/test_hudi_rewrite_mtmv.groovy @@ -62,9 +62,7 @@ suite("test_hudi_rewrite_mtmv", "p2,external,hudi,external_remote,external_remot waitingMTMVTaskFinishedByMvName(mvName) order_qt_refresh_one_partition "SELECT * FROM ${mvName} " - def explainOnePartition = sql """ explain ${mvSql} """ - logger.info("explainOnePartition: " + explainOnePartition.toString()) - assertTrue(explainOnePartition.toString().contains("VUNION")) + mv_rewrite_success(mvSql, mvName) order_qt_refresh_one_partition_rewrite "${mvSql}" mv_rewrite_success("${mvSql}", "${mvName}") @@ -79,9 +77,7 @@ suite("test_hudi_rewrite_mtmv", "p2,external,hudi,external_remote,external_remot waitingMTMVTaskFinishedByMvName(mvName) order_qt_refresh_auto "SELECT * FROM ${mvName} " - def explainAllPartition = sql """ explain ${mvSql}; """ - logger.info("explainAllPartition: " + explainAllPartition.toString()) - assertTrue(explainAllPartition.toString().contains("VOlapScanNode")) + mv_rewrite_success(mvSql, mvName) order_qt_refresh_all_partition_rewrite "${mvSql}" mv_rewrite_success("${mvSql}", "${mvName}") From cbe6b8cef0a00f9d9074dde32e368f243b94a7a6 Mon Sep 17 00:00:00 2001 From: seawinde Date: Thu, 4 Dec 2025 19:10:57 +0800 Subject: [PATCH 2/4] fix code compile --- .../rules/exploration/mv/AbstractMaterializedViewRule.java | 2 +- .../nereids/rules/exploration/mv/PartitionCompensator.java | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java index 39f164f48b110f..c6c44b9fb4a920 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java @@ -198,7 +198,7 @@ protected List getValidQueryStructInfos(Plan queryPlan, CascadesCont * only one materialization every time. Different query pattern should override the sub logic. */ protected List doRewrite(StructInfo queryStructInfo, CascadesContext cascadesContext, - MaterializationContext materializationContext) { + MaterializationContext materializationContext) throws AnalysisException { List rewriteResults = new ArrayList<>(); StructInfo viewStructInfo = materializationContext.getStructInfo(); MatchMode matchMode = decideMatchMode(queryStructInfo.getRelations(), viewStructInfo.getRelations()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PartitionCompensator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PartitionCompensator.java index 8ff3975bc9cf0a..b65335fd26978d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PartitionCompensator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PartitionCompensator.java @@ -183,7 +183,6 @@ public static boolean needUnionRewrite(MaterializationContext materializationCon // and prune partition failed return false; } - return !PartitionType.UNPARTITIONED.equals(type) && relatedTableInfo != null; } From 8322c1a5ba484f9c152d0a4229f9fcdd100a4ad7 Mon Sep 17 00:00:00 2001 From: seawinde Date: Thu, 4 Dec 2025 21:00:40 +0800 Subject: [PATCH 3/4] fix test --- .../mv/PartitionCompensatorTest.java | 62 ++++--------------- 1 file changed, 11 insertions(+), 51 deletions(-) diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/PartitionCompensatorTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/PartitionCompensatorTest.java index b1505dd69a4f42..5efdff2b12ed70 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/PartitionCompensatorTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/PartitionCompensatorTest.java @@ -26,7 +26,6 @@ import org.apache.doris.common.Pair; import org.apache.doris.datasource.CatalogIf; import org.apache.doris.datasource.hive.HMSExternalTable; -import org.apache.doris.mtmv.BaseColInfo; import org.apache.doris.mtmv.BaseTableInfo; import org.apache.doris.mtmv.MTMVPartitionInfo; import org.apache.doris.mtmv.MTMVRelatedTableIf; @@ -44,12 +43,9 @@ import org.mockito.Mockito; import java.util.BitSet; -import java.util.Collections; -import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; -import java.util.stream.Collectors; public class PartitionCompensatorTest extends TestWithFeService { @@ -209,45 +205,12 @@ public void testGetAllTableUsedPartitionList() { }); } - @Test - public void testNeedUnionRewriteUnpartitionedOrNoPctInfos() throws Exception { - MaterializationContext ctx1 = mockCtx( - PartitionType.UNPARTITIONED, - ImmutableList.of(new BaseColInfo("c", newBaseTableInfo())), - ImmutableSet.of(), - false); - StatementContext sc1 = Mockito.mock(StatementContext.class); - Mockito.when(sc1.getTableUsedPartitionNameMap()).thenReturn(ArrayListMultimap.create()); - Assertions.assertFalse(PartitionCompensator.needUnionRewrite(ctx1, sc1)); - - MaterializationContext ctx2 = mockCtx( - PartitionType.RANGE, - Collections.emptyList(), - ImmutableSet.of(Mockito.mock(MTMVRelatedTableIf.class)), - false); - StatementContext sc2 = Mockito.mock(StatementContext.class); - Mockito.when(sc2.getTableUsedPartitionNameMap()).thenReturn(ArrayListMultimap.create()); - Assertions.assertFalse(PartitionCompensator.needUnionRewrite(ctx2, sc2)); - } - - @Test - public void testNeedUnionRewriteEmptyPctTables() throws Exception { - MaterializationContext ctx = mockCtx( - PartitionType.RANGE, - ImmutableList.of(), - Collections.emptySet(), - false); - StatementContext sc = Mockito.mock(StatementContext.class); - Mockito.when(sc.getTableUsedPartitionNameMap()).thenReturn(ArrayListMultimap.create()); - Assertions.assertFalse(PartitionCompensator.needUnionRewrite(ctx, sc)); - } - @Test public void testNeedUnionRewriteExternalNoPrune() throws Exception { MaterializationContext ctx = mockCtx( PartitionType.LIST, - ImmutableList.of(new BaseColInfo("c", newBaseTableInfo())), - ImmutableSet.of(Mockito.mock(MTMVRelatedTableIf.class)), + newBaseTableInfo(), + Mockito.mock(MTMVRelatedTableIf.class), true); StatementContext sc = Mockito.mock(StatementContext.class); Mockito.when(sc.getTableUsedPartitionNameMap()).thenReturn(ArrayListMultimap.create()); @@ -258,8 +221,8 @@ public void testNeedUnionRewriteExternalNoPrune() throws Exception { public void testNeedUnionRewritePositive() throws Exception { MaterializationContext ctx = mockCtx( PartitionType.LIST, - ImmutableList.of(new BaseColInfo("c", newBaseTableInfo())), - ImmutableSet.of(Mockito.mock(MTMVRelatedTableIf.class)), + newBaseTableInfo(), + Mockito.mock(MTMVRelatedTableIf.class), false); StatementContext sc = Mockito.mock(StatementContext.class); Mockito.when(sc.getTableUsedPartitionNameMap()).thenReturn(ArrayListMultimap.create()); @@ -271,8 +234,8 @@ public void testNotNeedUnionRewriteWhenAllPartitions() throws Exception { BaseTableInfo tableInfo = newBaseTableInfo(); MaterializationContext ctx = mockCtx( PartitionType.LIST, - ImmutableList.of(new BaseColInfo("c", tableInfo)), - ImmutableSet.of(Mockito.mock(MTMVRelatedTableIf.class)), + tableInfo, + Mockito.mock(MTMVRelatedTableIf.class), false); StatementContext sc = Mockito.mock(StatementContext.class); @@ -344,8 +307,8 @@ public void testGetQueryUsedPartitionsEmptyCollectionMeansNoPartitions() { private static MaterializationContext mockCtx( PartitionType type, - List pctInfos, - Set pctTables, + BaseTableInfo pctInfo, + MTMVRelatedTableIf pctTable, boolean externalNoPrune) throws AnalysisException { MTMV mtmv = Mockito.mock(MTMV.class); @@ -355,16 +318,13 @@ private static MaterializationContext mockCtx( MTMVPartitionInfo mpi = Mockito.mock(MTMVPartitionInfo.class); Mockito.when(mtmv.getMvPartitionInfo()).thenReturn(mpi); - Mockito.when(mpi.getPctInfos()).thenReturn(pctInfos); - Mockito.when(mpi.getPctTables()).thenReturn(pctTables); + Mockito.when(mpi.getRelatedTableInfo()).thenReturn(pctInfo); + Mockito.when(mpi.getRelatedTable()).thenReturn(pctTable); if (externalNoPrune) { HMSExternalTable ext = Mockito.mock(HMSExternalTable.class); Mockito.when(ext.supportInternalPartitionPruned()).thenReturn(false); - Set tbls = new HashSet<>(pctTables); - tbls.add(ext); - Mockito.when(mpi.getPctTables()).thenReturn( - tbls.stream().map(MTMVRelatedTableIf.class::cast).collect(Collectors.toSet())); + Mockito.when(mpi.getRelatedTable()).thenReturn(ext); } AsyncMaterializationContext ctx = Mockito.mock(AsyncMaterializationContext.class); From c9ef6e58b91a3b65afa0386cbed60901411d6d92 Mon Sep 17 00:00:00 2001 From: seawinde Date: Fri, 5 Dec 2025 11:08:52 +0800 Subject: [PATCH 4/4] fix test fail --- .../exploration/mv/PartitionCompensator.java | 23 ++++++++++--------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PartitionCompensator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PartitionCompensator.java index b65335fd26978d..f17d5364ff0a45 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PartitionCompensator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PartitionCompensator.java @@ -163,9 +163,13 @@ public static boolean needUnionRewrite(MaterializationContext materializationCon } MTMV mtmv = ((AsyncMaterializationContext) materializationContext).getMtmv(); BaseTableInfo relatedTableInfo = mtmv.getMvPartitionInfo().getRelatedTableInfo(); - PartitionType type = mtmv.getPartitionInfo().getType(); - MTMVPartitionInfo mvPartitionInfo = mtmv.getMvPartitionInfo(); - MTMVRelatedTableIf pctTable = mvPartitionInfo.getRelatedTable(); + if (relatedTableInfo == null) { + return false; + } + if (PartitionType.UNPARTITIONED.equals(mtmv.getPartitionInfo().getType())) { + return false; + } + MTMVRelatedTableIf pctTable = mtmv.getMvPartitionInfo().getRelatedTable(); Multimap, Pair>> tableUsedPartitionNameMap = statementContext.getTableUsedPartitionNameMap(); if (pctTable instanceof ExternalTable && !((ExternalTable) pctTable).supportInternalPartitionPruned()) { @@ -176,14 +180,11 @@ public static boolean needUnionRewrite(MaterializationContext materializationCon } Collection>> tableUsedPartitions = tableUsedPartitionNameMap.get(pctTable.getFullQualifiers()); - if (ALL_PARTITIONS_LIST.equals(tableUsedPartitions) - || tableUsedPartitions.stream().anyMatch(ALL_PARTITIONS::equals)) { - // If query base table all partitions with ALL_PARTITIONS or ALL_PARTITIONS_LIST, - // should not do union compensate, because it means query all partitions from base table - // and prune partition failed - return false; - } - return !PartitionType.UNPARTITIONED.equals(type) && relatedTableInfo != null; + // If query base table all partitions with ALL_PARTITIONS or ALL_PARTITIONS_LIST, + // should not do union compensate, because it means query all partitions from base table + // and prune partition failed + return !ALL_PARTITIONS_LIST.equals(tableUsedPartitions) + && tableUsedPartitions.stream().noneMatch(ALL_PARTITIONS::equals); } /**