From 2a4e3e5ce23562ff30cea86e7db1e3e59789251f Mon Sep 17 00:00:00 2001 From: seawinde Date: Wed, 16 Apr 2025 15:14:44 +0800 Subject: [PATCH 1/3] [fix](mtmv) Fix exception when calc mtmv function dependency but the table in mtmv def sql is deleted --- .../src/main/java/org/apache/doris/catalog/MTMV.java | 2 +- .../exploration/mv/AsyncMaterializationContext.java | 3 +-- .../nereids/trees/plans/logical/LogicalOlapScan.java | 11 +++++------ .../suites/mtmv_p0/test_env_db_dropped_mtmv.groovy | 5 +++++ 4 files changed, 12 insertions(+), 9 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java index d96a4997a80e12..6bb88436600b89 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java @@ -310,7 +310,7 @@ public Set getExcludedTriggerTables() { /** * Called when in query, Should use one connection context in query */ - public MTMVCache getOrGenerateCache(ConnectContext connectionContext) throws AnalysisException { + public MTMVCache getOrGenerateCache(ConnectContext connectionContext) { readMvLock(); try { if (cache != null) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AsyncMaterializationContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AsyncMaterializationContext.java index 593ad986ca797c..8da199efc5bbd8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AsyncMaterializationContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AsyncMaterializationContext.java @@ -19,7 +19,6 @@ import org.apache.doris.catalog.MTMV; import org.apache.doris.catalog.Table; -import org.apache.doris.common.AnalysisException; import org.apache.doris.common.Id; import org.apache.doris.common.Pair; import org.apache.doris.mtmv.MTMVCache; @@ -102,7 +101,7 @@ public Optional> getPlanStatistics(CascadesContext cascades MTMVCache mtmvCache; try { mtmvCache = mtmv.getOrGenerateCache(cascadesContext.getConnectContext()); - } catch (AnalysisException e) { + } catch (Exception e) { LOG.warn(String.format("get mv plan statistics fail, materialization qualifier is %s", generateMaterializationIdentifier()), e); return Optional.empty(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java index ce57f026ea5f1c..5c623ed91b2a9c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java @@ -22,7 +22,6 @@ import org.apache.doris.catalog.MTMV; import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.Table; -import org.apache.doris.common.AnalysisException; import org.apache.doris.mtmv.MTMVCache; import org.apache.doris.nereids.memo.GroupExpression; import org.apache.doris.nereids.properties.DataTrait; @@ -565,7 +564,7 @@ AGGREGATE KEY (siteid,citycode,username) MTMVCache cache; try { cache = mtmv.getOrGenerateCache(ConnectContext.get()); - } catch (AnalysisException e) { + } catch (Exception e) { LOG.warn(String.format("LogicalOlapScan computeUnique fail, mv name is %s", mtmv.getName()), e); return; } @@ -604,7 +603,7 @@ public void computeUniform(DataTrait.Builder builder) { MTMVCache cache; try { cache = mtmv.getOrGenerateCache(ConnectContext.get()); - } catch (AnalysisException e) { + } catch (Exception e) { LOG.warn(String.format("LogicalOlapScan computeUniform fail, mv name is %s", mtmv.getName()), e); return; } @@ -625,7 +624,7 @@ public void computeEqualSet(DataTrait.Builder builder) { MTMVCache cache; try { cache = mtmv.getOrGenerateCache(ConnectContext.get()); - } catch (AnalysisException e) { + } catch (Exception e) { LOG.warn(String.format("LogicalOlapScan computeEqualSet fail, mv name is %s", mtmv.getName()), e); return; } @@ -646,7 +645,7 @@ public void computeFd(DataTrait.Builder builder) { MTMVCache cache; try { cache = mtmv.getOrGenerateCache(ConnectContext.get()); - } catch (AnalysisException e) { + } catch (Exception e) { LOG.warn(String.format("LogicalOlapScan computeFd fail, mv name is %s", mtmv.getName()), e); return; } @@ -677,7 +676,7 @@ Map constructReplaceMap(MTMV mtmv) { MTMVCache cache; try { cache = mtmv.getOrGenerateCache(ConnectContext.get()); - } catch (AnalysisException e) { + } catch (Exception e) { LOG.warn(String.format("LogicalOlapScan constructReplaceMap fail, mv name is %s", mtmv.getName()), e); return replaceMap; } diff --git a/regression-test/suites/mtmv_p0/test_env_db_dropped_mtmv.groovy b/regression-test/suites/mtmv_p0/test_env_db_dropped_mtmv.groovy index 516f5a648c7676..6e63ae0a53b886 100644 --- a/regression-test/suites/mtmv_p0/test_env_db_dropped_mtmv.groovy +++ b/regression-test/suites/mtmv_p0/test_env_db_dropped_mtmv.groovy @@ -90,4 +90,9 @@ suite("test_env_db_dropped_mtmv") { def msg = sql """select ErrorMsg from tasks('type'='mv') where JobName = '${jobName}' order by CreateTime DESC limit 1""" logger.info(msg.toString()) assertTrue(msg.toString().contains("does not exist")) + + // select ${dbName2}.${mvName} should not throw exception + sql """ + select count(*) from ${dbName2}.${mvName} + """ } From 4cb878ade627e38c0e15a5bc725ba43030ef5197 Mon Sep 17 00:00:00 2001 From: seawinde Date: Wed, 16 Apr 2025 15:34:26 +0800 Subject: [PATCH 2/3] add test result check --- regression-test/data/mtmv_p0/test_env_db_dropped_mtmv.out | 6 ++++++ .../suites/mtmv_p0/test_env_db_dropped_mtmv.groovy | 5 ++--- 2 files changed, 8 insertions(+), 3 deletions(-) create mode 100644 regression-test/data/mtmv_p0/test_env_db_dropped_mtmv.out diff --git a/regression-test/data/mtmv_p0/test_env_db_dropped_mtmv.out b/regression-test/data/mtmv_p0/test_env_db_dropped_mtmv.out new file mode 100644 index 00000000000000..a3f2c1adc58fde --- /dev/null +++ b/regression-test/data/mtmv_p0/test_env_db_dropped_mtmv.out @@ -0,0 +1,6 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !query1 -- +0 + +-- !query2 -- + diff --git a/regression-test/suites/mtmv_p0/test_env_db_dropped_mtmv.groovy b/regression-test/suites/mtmv_p0/test_env_db_dropped_mtmv.groovy index 6e63ae0a53b886..5e25cbf4394422 100644 --- a/regression-test/suites/mtmv_p0/test_env_db_dropped_mtmv.groovy +++ b/regression-test/suites/mtmv_p0/test_env_db_dropped_mtmv.groovy @@ -92,7 +92,6 @@ suite("test_env_db_dropped_mtmv") { assertTrue(msg.toString().contains("does not exist")) // select ${dbName2}.${mvName} should not throw exception - sql """ - select count(*) from ${dbName2}.${mvName} - """ + order_qt_query1 "select count(*) from ${dbName2}.${mvName};" + order_qt_query2 "select * from ${dbName2}.${mvName};" } From 0024b431ee92a6f4d6d298097ee3061f6c38b78d Mon Sep 17 00:00:00 2001 From: seawinde Date: Wed, 16 Apr 2025 16:18:11 +0800 Subject: [PATCH 3/3] add exception --- fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java | 3 ++- fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVCache.java | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java index 6bb88436600b89..716e00b10f2438 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java @@ -310,7 +310,8 @@ public Set getExcludedTriggerTables() { /** * Called when in query, Should use one connection context in query */ - public MTMVCache getOrGenerateCache(ConnectContext connectionContext) { + public MTMVCache getOrGenerateCache(ConnectContext connectionContext) throws + org.apache.doris.nereids.exceptions.AnalysisException { readMvLock(); try { if (cache != null) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVCache.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVCache.java index bc99fe8bfd1e49..94fd5415341fc0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVCache.java @@ -20,6 +20,7 @@ import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.NereidsPlanner; import org.apache.doris.nereids.StatementContext; +import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.jobs.executor.Rewriter; import org.apache.doris.nereids.parser.NereidsParser; import org.apache.doris.nereids.properties.PhysicalProperties; @@ -97,7 +98,7 @@ public StructInfo getStructInfo() { public static MTMVCache from(String defSql, ConnectContext createCacheContext, boolean needCost, boolean needLock, - ConnectContext currentContext) { + ConnectContext currentContext) throws AnalysisException { StatementContext mvSqlStatementContext = new StatementContext(createCacheContext, new OriginStatement(defSql, 0)); if (!needLock) {