From 61832ce0ea0fef355153927bd147f7740a173b81 Mon Sep 17 00:00:00 2001 From: zhangdong Date: Thu, 6 Nov 2025 11:20:20 +0800 Subject: [PATCH 01/15] can not format external view --- .../src/main/java/org/apache/doris/mtmv/MTMVPlanUtil.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPlanUtil.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPlanUtil.java index a29915a846166c..2a5f8fba6688b2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPlanUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPlanUtil.java @@ -41,6 +41,7 @@ import org.apache.doris.common.util.PropertyAnalyzer; import org.apache.doris.common.util.Util; import org.apache.doris.datasource.CatalogIf; +import org.apache.doris.datasource.ExternalTable; import org.apache.doris.job.exception.JobException; import org.apache.doris.mtmv.MTMVPartitionInfo.MTMVPartitionType; import org.apache.doris.nereids.NereidsPlanner; @@ -440,6 +441,12 @@ public static MTMVAnalyzeQueryInfo analyzeQuery(ConnectContext ctx, Map Date: Thu, 6 Nov 2025 11:53:07 +0800 Subject: [PATCH 02/15] add case --- .../suites/mtmv_p0/test_hive_mtmv.groovy | 12 ++++++++++++ .../suites/mtmv_p0/test_multi_pct_mtmv.groovy | 19 +++++++++++++++++++ 2 files changed, 31 insertions(+) diff --git a/regression-test/suites/mtmv_p0/test_hive_mtmv.groovy b/regression-test/suites/mtmv_p0/test_hive_mtmv.groovy index 7b3abc2300f383..020990f5a440db 100644 --- a/regression-test/suites/mtmv_p0/test_hive_mtmv.groovy +++ b/regression-test/suites/mtmv_p0/test_hive_mtmv.groovy @@ -48,6 +48,18 @@ suite("test_hive_mtmv", "p0,external,hive,external_docker,external_docker_hive") exception "internal" } + test { + sql """ + CREATE MATERIALIZED VIEW ${mvName} + BUILD DEFERRED REFRESH AUTO ON MANUAL + DISTRIBUTED BY RANDOM BUCKETS 2 + PROPERTIES ('replication_num' = '1') + AS + SELECT * FROM ${catalog_name}.`default`.test_view1; + """ + exception "contain" + } + sql """ CREATE MATERIALIZED VIEW ${mvName} BUILD DEFERRED REFRESH AUTO ON MANUAL diff --git a/regression-test/suites/mtmv_p0/test_multi_pct_mtmv.groovy b/regression-test/suites/mtmv_p0/test_multi_pct_mtmv.groovy index 17a39441a08a26..1baaab981c7640 100644 --- a/regression-test/suites/mtmv_p0/test_multi_pct_mtmv.groovy +++ b/regression-test/suites/mtmv_p0/test_multi_pct_mtmv.groovy @@ -172,4 +172,23 @@ suite("test_multi_pct_mtmv","mtmv") { """ waitingMTMVTaskFinishedByMvName(mvName) order_qt_partitions_4 "select PartitionName,UnsyncTables from partitions('catalog'='internal','database'='${dbName}','table'='${mvName}') order by PartitionId desc;" + + sql """ + alter table ${tableName2} add partition p201704 VALUES [("2017-04-01"), ("2017-05-01")),; + """ + order_qt_partitions_5 "select PartitionName,UnsyncTables from partitions('catalog'='internal','database'='${dbName}','table'='${mvName}') order by PartitionId desc;" + + sql """ + REFRESH MATERIALIZED VIEW ${mvName} partitions(p_20170401_20170501); + """ + order_qt_partitions_6 "select PartitionName,UnsyncTables from partitions('catalog'='internal','database'='${dbName}','table'='${mvName}') order by PartitionId desc;" + + sql """ + insert into ${tableName2} values("2017-04-01",10); + """ + sql """ + REFRESH MATERIALIZED VIEW ${mvName} AUTO + """ + waitingMTMVTaskFinishedByMvName(mvName) + order_qt_refresh_mode_overwrite "select RefreshMode from tasks('type'='mv') where MvName='${mvName}' order by CreateTime desc limit 1" } From 60d957f071c72cf610f3e50dd9673b1f7e81da64 Mon Sep 17 00:00:00 2001 From: zhangdong Date: Thu, 6 Nov 2025 12:02:34 +0800 Subject: [PATCH 03/15] add case --- regression-test/suites/mtmv_p0/test_multi_pct_mtmv.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/regression-test/suites/mtmv_p0/test_multi_pct_mtmv.groovy b/regression-test/suites/mtmv_p0/test_multi_pct_mtmv.groovy index 1baaab981c7640..9736ebf7680c6f 100644 --- a/regression-test/suites/mtmv_p0/test_multi_pct_mtmv.groovy +++ b/regression-test/suites/mtmv_p0/test_multi_pct_mtmv.groovy @@ -174,7 +174,7 @@ suite("test_multi_pct_mtmv","mtmv") { order_qt_partitions_4 "select PartitionName,UnsyncTables from partitions('catalog'='internal','database'='${dbName}','table'='${mvName}') order by PartitionId desc;" sql """ - alter table ${tableName2} add partition p201704 VALUES [("2017-04-01"), ("2017-05-01")),; + alter table ${tableName2} add partition p201704 VALUES [("2017-04-01"),("2017-05-01")); """ order_qt_partitions_5 "select PartitionName,UnsyncTables from partitions('catalog'='internal','database'='${dbName}','table'='${mvName}') order by PartitionId desc;" From da59cebc61f3c0eb4ae05beb85dc9b2fb9540acf Mon Sep 17 00:00:00 2001 From: zhangdong <493738387@qq.com> Date: Thu, 6 Nov 2025 12:03:54 +0800 Subject: [PATCH 04/15] 1 --- regression-test/data/mtmv_p0/test_multi_pct_mtmv.out | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/regression-test/data/mtmv_p0/test_multi_pct_mtmv.out b/regression-test/data/mtmv_p0/test_multi_pct_mtmv.out index 040c49df8cee48..7344edf4f1fc29 100644 --- a/regression-test/data/mtmv_p0/test_multi_pct_mtmv.out +++ b/regression-test/data/mtmv_p0/test_multi_pct_mtmv.out @@ -41,3 +41,15 @@ p_20170301_20170401 [test_multi_pct_mtmv_table2] p_20170101_20170201 [] p_20170201_20170301 [] +-- !partitions_5 -- +p_20170101_20170201 [] +p_20170201_20170301 [] + +-- !partitions_6 -- +p_20170101_20170201 [] +p_20170201_20170301 [] +p_20170401_20170501 [test_multi_pct_mtmv_table3, test_multi_pct_mtmv_table2] + +-- !refresh_mode_overwrite -- +PARTIAL + From 4d62b754f2a0e47bc7ee0086fcd29fbd2fdb298f Mon Sep 17 00:00:00 2001 From: zhangdong Date: Thu, 6 Nov 2025 12:05:40 +0800 Subject: [PATCH 05/15] add case --- regression-test/suites/mtmv_p0/test_multi_pct_mtmv.groovy | 1 + 1 file changed, 1 insertion(+) diff --git a/regression-test/suites/mtmv_p0/test_multi_pct_mtmv.groovy b/regression-test/suites/mtmv_p0/test_multi_pct_mtmv.groovy index 9736ebf7680c6f..940d31535d0678 100644 --- a/regression-test/suites/mtmv_p0/test_multi_pct_mtmv.groovy +++ b/regression-test/suites/mtmv_p0/test_multi_pct_mtmv.groovy @@ -181,6 +181,7 @@ suite("test_multi_pct_mtmv","mtmv") { sql """ REFRESH MATERIALIZED VIEW ${mvName} partitions(p_20170401_20170501); """ + waitingMTMVTaskFinishedByMvName(mvName) order_qt_partitions_6 "select PartitionName,UnsyncTables from partitions('catalog'='internal','database'='${dbName}','table'='${mvName}') order by PartitionId desc;" sql """ From 616bff1a0b215843240eaeb8f1f77504834d8b57 Mon Sep 17 00:00:00 2001 From: zhangdong <493738387@qq.com> Date: Thu, 6 Nov 2025 12:07:33 +0800 Subject: [PATCH 06/15] 1 --- regression-test/data/mtmv_p0/test_multi_pct_mtmv.out | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/regression-test/data/mtmv_p0/test_multi_pct_mtmv.out b/regression-test/data/mtmv_p0/test_multi_pct_mtmv.out index 7344edf4f1fc29..67ee5cf54e40ed 100644 --- a/regression-test/data/mtmv_p0/test_multi_pct_mtmv.out +++ b/regression-test/data/mtmv_p0/test_multi_pct_mtmv.out @@ -48,7 +48,7 @@ p_20170201_20170301 [] -- !partitions_6 -- p_20170101_20170201 [] p_20170201_20170301 [] -p_20170401_20170501 [test_multi_pct_mtmv_table3, test_multi_pct_mtmv_table2] +p_20170401_20170501 [] -- !refresh_mode_overwrite -- PARTIAL From 4cf8ea01b246fad6f8a4764909618764ce6ff890 Mon Sep 17 00:00:00 2001 From: zhangdong Date: Thu, 6 Nov 2025 12:10:32 +0800 Subject: [PATCH 07/15] add case --- regression-test/suites/mtmv_p0/test_multi_pct_mtmv.groovy | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/regression-test/suites/mtmv_p0/test_multi_pct_mtmv.groovy b/regression-test/suites/mtmv_p0/test_multi_pct_mtmv.groovy index 940d31535d0678..4eb7a83ab70aab 100644 --- a/regression-test/suites/mtmv_p0/test_multi_pct_mtmv.groovy +++ b/regression-test/suites/mtmv_p0/test_multi_pct_mtmv.groovy @@ -192,4 +192,11 @@ suite("test_multi_pct_mtmv","mtmv") { """ waitingMTMVTaskFinishedByMvName(mvName) order_qt_refresh_mode_overwrite "select RefreshMode from tasks('type'='mv') where MvName='${mvName}' order by CreateTime desc limit 1" + + test { + sql """ + REFRESH MATERIALIZED VIEW ${mvName} partitions(p_20180501_20180601); + """ + exception "exist" + } } From d3b2b10342baceace23a79e9578ea7bd3134d70e Mon Sep 17 00:00:00 2001 From: zhangdong Date: Thu, 6 Nov 2025 14:36:10 +0800 Subject: [PATCH 08/15] add case --- .../test_multi_pct_null_partition_mtmv.groovy | 121 ++++++++++++++++++ 1 file changed, 121 insertions(+) create mode 100644 regression-test/suites/mtmv_p0/test_multi_pct_null_partition_mtmv.groovy diff --git a/regression-test/suites/mtmv_p0/test_multi_pct_null_partition_mtmv.groovy b/regression-test/suites/mtmv_p0/test_multi_pct_null_partition_mtmv.groovy new file mode 100644 index 00000000000000..d41c1205cceb5f --- /dev/null +++ b/regression-test/suites/mtmv_p0/test_multi_pct_null_partition_mtmv.groovy @@ -0,0 +1,121 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.junit.Assert; + +suite("test_multi_pct_null_partition_mtmv","mtmv") { + String suiteName = "test_multi_pct_null_partition_mtmv" + String dbName = context.config.getDbNameByFile(context.file) + String tableName1 = "${suiteName}_table1" + String tableName2 = "${suiteName}_table2" + String mvName = "${suiteName}_mv" + sql """drop table if exists `${tableName1}`""" + sql """drop table if exists `${tableName2}`""" + sql """drop materialized view if exists ${mvName};""" + + sql """ + CREATE TABLE ${tableName1} + ( + k1 INT not null, + k2 INT not null + ) + PARTITION BY RANGE(`k1`) + ( + PARTITION p20 values less than (20), + PARTITION p100 values less than (100), + PARTITION pMAX values less than (maxvalue) + ) + DISTRIBUTED BY HASH(k2) BUCKETS 2 + PROPERTIES ( + "replication_num" = "1" + ); + """ + + sql """ + CREATE TABLE ${tableName2} + ( + k1 INT not null, + k2 INT not null + ) + PARTITION BY RANGE(`k1`) + ( + PARTITION p10 values less than (10), + PARTITION p100 values less than (100), + PARTITION pMAX values less than (maxvalue) + ) + DISTRIBUTED BY HASH(k2) BUCKETS 2 + PROPERTIES ( + "replication_num" = "1" + ); + """ + + String mvSql = "SELECT * from ${tableName1} t1 union all select * from ${tableName2} t2;"; + test { + sql """ + CREATE MATERIALIZED VIEW ${mvName} + BUILD DEFERRED REFRESH AUTO ON MANUAL + partition by(k1) + DISTRIBUTED BY RANDOM BUCKETS 2 + PROPERTIES ( + 'replication_num' = '1' + ) + AS + ${mvSql} + """ + exception "intersected" + } + + sql """drop table if exists `${tableName1}`""" + + sql """ + CREATE TABLE ${tableName1} + ( + k1 INT not null, + k2 INT not null + ) + PARTITION BY RANGE(`k1`) + ( + PARTITION p10 values less than (10), + PARTITION p100 values less than (100), + PARTITION pMAX values less than (maxvalue) + ) + DISTRIBUTED BY HASH(k2) BUCKETS 2 + PROPERTIES ( + "replication_num" = "1" + ); + """ + sql """ + CREATE MATERIALIZED VIEW ${mvName} + BUILD DEFERRED REFRESH AUTO ON MANUAL + partition by(k1) + DISTRIBUTED BY RANDOM BUCKETS 2 + PROPERTIES ( + 'replication_num' = '1' + ) + AS + ${mvSql} + """ + + sql """ + insert into ${tableName1} values(null,1); + """ + sql """ + REFRESH MATERIALIZED VIEW ${mvName} partitions(p_2147483648_10); + """ + waitingMTMVTaskFinishedByMvName(mvName) + order_qt_select "select * from ${mvName}" +} From 36f3975f4126b5280cd0d71bc20e1d44c7feab62 Mon Sep 17 00:00:00 2001 From: zhangdong Date: Thu, 6 Nov 2025 14:40:11 +0800 Subject: [PATCH 09/15] add case --- .../test_multi_pct_null_partition_mtmv.groovy | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/regression-test/suites/mtmv_p0/test_multi_pct_null_partition_mtmv.groovy b/regression-test/suites/mtmv_p0/test_multi_pct_null_partition_mtmv.groovy index d41c1205cceb5f..3277a646b95b35 100644 --- a/regression-test/suites/mtmv_p0/test_multi_pct_null_partition_mtmv.groovy +++ b/regression-test/suites/mtmv_p0/test_multi_pct_null_partition_mtmv.groovy @@ -30,8 +30,8 @@ suite("test_multi_pct_null_partition_mtmv","mtmv") { sql """ CREATE TABLE ${tableName1} ( - k1 INT not null, - k2 INT not null + k1 INT, + k2 INT ) PARTITION BY RANGE(`k1`) ( @@ -48,8 +48,8 @@ suite("test_multi_pct_null_partition_mtmv","mtmv") { sql """ CREATE TABLE ${tableName2} ( - k1 INT not null, - k2 INT not null + k1 INT, + k2 INT ) PARTITION BY RANGE(`k1`) ( @@ -84,8 +84,8 @@ suite("test_multi_pct_null_partition_mtmv","mtmv") { sql """ CREATE TABLE ${tableName1} ( - k1 INT not null, - k2 INT not null + k1 INT, + k2 INT ) PARTITION BY RANGE(`k1`) ( From b9e52bd7a167d3426017af43e8e4ef972b4f46b8 Mon Sep 17 00:00:00 2001 From: zhangdong <493738387@qq.com> Date: Thu, 6 Nov 2025 14:43:07 +0800 Subject: [PATCH 10/15] 1 --- .../data/mtmv_p0/test_multi_pct_null_partition_mtmv.out | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 regression-test/data/mtmv_p0/test_multi_pct_null_partition_mtmv.out diff --git a/regression-test/data/mtmv_p0/test_multi_pct_null_partition_mtmv.out b/regression-test/data/mtmv_p0/test_multi_pct_null_partition_mtmv.out new file mode 100644 index 00000000000000..c709a07ed8240a --- /dev/null +++ b/regression-test/data/mtmv_p0/test_multi_pct_null_partition_mtmv.out @@ -0,0 +1,4 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !select -- +\N 1 + From db3cc49a3678ecb6a719305a3927e6cd96283436 Mon Sep 17 00:00:00 2001 From: zhangdong Date: Thu, 6 Nov 2025 15:04:44 +0800 Subject: [PATCH 11/15] add case --- .../suites/mtmv_p0/test_hive_pct_mtmv.groovy | 161 ++++++++++++++++++ 1 file changed, 161 insertions(+) create mode 100644 regression-test/suites/mtmv_p0/test_hive_pct_mtmv.groovy diff --git a/regression-test/suites/mtmv_p0/test_hive_pct_mtmv.groovy b/regression-test/suites/mtmv_p0/test_hive_pct_mtmv.groovy new file mode 100644 index 00000000000000..e13592767dcc04 --- /dev/null +++ b/regression-test/suites/mtmv_p0/test_hive_pct_mtmv.groovy @@ -0,0 +1,161 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_hive_pct_mtmv", "p0,external,hive,external_docker,external_docker_hive,nonConcurrent") { + String enabled = context.config.otherConfigs.get("enableHiveTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + logger.info("diable Hive test.") + return; + } + String suiteName = "test_hive_pct_mtmv" + String dbName = "${suiteName}_db" + String tableName1 = "${suiteName}_table1" + String tableName2 = "${suiteName}_table2" + String mvName = "${suiteName}_mv" + String mvDbName = context.config.getDbNameByFile(context.file) + for (String hivePrefix : ["hive3"]) { + setHivePrefix(hivePrefix) + + def autogather_off_str = """ set hive.stats.column.autogather = false; """ + def autogather_on_str = """ set hive.stats.column.autogather = true; """ + def drop_table_str1 = """ drop table if exists ${dbName}.${tableName1} """ + def drop_table_str2 = """ drop table if exists ${dbName}.${tableName2} """ + def drop_database_str = """ drop database if exists ${dbName}""" + def create_database_str = """ create database ${dbName}""" + def create_table_str1 = """ CREATE TABLE ${dbName}.${tableName1} ( + `k1` int) + PARTITIONED BY ( + `year` int, + `region` string) + STORED AS ORC; + """ + def add_partition_str1 = """ + alter table ${dbName}.${tableName1} add if not exists + partition(year=2020,region="bj") + partition(year=2020,region="sh") + partition(year=2021,region="bj") + partition(year=2021,region="sh") + """ + def insert_str1 = """insert into ${dbName}.${tableName1} PARTITION(year=2020,region="bj") values(1)""" + def insert_str2 = """insert into ${dbName}.${tableName1} PARTITION(year=2020,region="sh") values(2)""" + def insert_str3 = """insert into ${dbName}.${tableName1} PARTITION(year=2021,region="bj") values(3)""" + def insert_str4 = """insert into ${dbName}.${tableName1} PARTITION(year=2021,region="sh") values(4)""" + + def create_table_str2 = """ CREATE TABLE ${dbName}.${tableName2} ( + `k1` int, + `year` int + ) + PARTITIONED BY ( + `region` string) + STORED AS ORC; + """ + def add_partition_str2 = """ + alter table ${dbName}.${tableName2} add if not exists + partition(region="bj") + partition(region="sh") + partition(region="tj") + """ + def insert_str5 = """insert into ${dbName}.${tableName2} PARTITION(region="bj") values(5,2020)""" + def insert_str6 = """insert into ${dbName}.${tableName2} PARTITION(region="sh") values(6,2020)""" + def insert_str7 = """insert into ${dbName}.${tableName2} PARTITION(region="tj") values(7,2020)""" + + logger.info("hive sql: " + autogather_off_str) + hive_docker """ ${autogather_off_str} """ + logger.info("hive sql: " + drop_table_str1) + hive_docker """ ${drop_table_str1} """ + logger.info("hive sql: " + drop_table_str2) + hive_docker """ ${drop_table_str3} """ + logger.info("hive sql: " + drop_database_str) + hive_docker """ ${drop_database_str} """ + logger.info("hive sql: " + create_database_str) + hive_docker """ ${create_database_str}""" + logger.info("hive sql: " + create_table_str1) + hive_docker """ ${create_table_str1} """ + logger.info("hive sql: " + add_partition_str1) + hive_docker """ ${add_partition_str1} """ + logger.info("hive sql: " + insert_str1) + hive_docker """ ${insert_str1} """ + logger.info("hive sql: " + insert_str2) + hive_docker """ ${insert_str2} """ + logger.info("hive sql: " + insert_str3) + hive_docker """ ${insert_str3} """ + logger.info("hive sql: " + insert_str4) + hive_docker """ ${insert_str4} """ + logger.info("hive sql: " + create_table_str2) + hive_docker """ ${create_table_str2} """ + logger.info("hive sql: " + add_partition_str2) + hive_docker """ ${add_partition_str2} """ + logger.info("hive sql: " + insert_str5) + hive_docker """ ${insert_str5} """ + logger.info("hive sql: " + insert_str6) + hive_docker """ ${insert_str6} """ + logger.info("hive sql: " + insert_str7) + hive_docker """ ${insert_str7} """ + + // prepare catalog + String hms_port = context.config.otherConfigs.get(hivePrefix + "HmsPort") + String catalog_name = "${hivePrefix}_${suiteName}_catalog" + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + + sql """drop catalog if exists ${catalog_name}""" + sql """create catalog if not exists ${catalog_name} properties ( + "type"="hms", + 'hive.metastore.uris' = 'thrift://${externalEnvIp}:${hms_port}' + );""" + + order_qt_select_base_table "SELECT k1,year,region FROM ${catalog_name}.${dbName}.${tableName1} union all SELECT k1,year,region FROM ${catalog_name}.${dbName}.${tableName2} " + + + // prepare mtmv + sql """drop materialized view if exists ${mvName};""" + + sql """ + CREATE MATERIALIZED VIEW ${mvName} + BUILD DEFERRED REFRESH AUTO ON MANUAL + partition by(`region`) + DISTRIBUTED BY RANDOM BUCKETS 2 + PROPERTIES ('replication_num' = '1') + AS + SELECT k1,year,region FROM ${catalog_name}.${dbName}.${tableName1} union all SELECT k1,year,region FROM ${catalog_name}.${dbName}.${tableName2}; + """ + order_qt_partitions_1 "select PartitionName,UnsyncTables from partitions('catalog'='internal','database'='${dbName}','table'='${mvName}') order by PartitionId desc;" + sql """ + REFRESH MATERIALIZED VIEW ${mvName} AUTO + """ + waitingMTMVTaskFinishedByMvName(mvName) + order_qt_select_mv1 "SELECT * from ${mvName}" + + // hive data change + def insert_str8 = """ + insert into ${dbName}.${tableName1} PARTITION(year=2020,region="bj") values(7); + """ + logger.info("hive sql: " + insert_str8) + hive_docker """ ${insert_str8} """ + sql """ + REFRESH catalog ${catalog_name} + """ + sql """ + REFRESH MATERIALIZED VIEW ${mvName} partitions(p_bj); + """ + waitingMTMVTaskFinishedByMvName(mvName) + order_qt_select_mv2 "SELECT * from ${mvName}" + + sql """drop materialized view if exists ${mvName};""" + sql """drop catalog if exists ${catalog_name}""" + } +} + From fcff6c87db4e1300806654f2a5e9aca42065e5d6 Mon Sep 17 00:00:00 2001 From: zhangdong Date: Thu, 6 Nov 2025 15:07:55 +0800 Subject: [PATCH 12/15] add case --- regression-test/suites/mtmv_p0/test_hive_pct_mtmv.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/regression-test/suites/mtmv_p0/test_hive_pct_mtmv.groovy b/regression-test/suites/mtmv_p0/test_hive_pct_mtmv.groovy index e13592767dcc04..7be15cd1218efc 100644 --- a/regression-test/suites/mtmv_p0/test_hive_pct_mtmv.groovy +++ b/regression-test/suites/mtmv_p0/test_hive_pct_mtmv.groovy @@ -78,7 +78,7 @@ suite("test_hive_pct_mtmv", "p0,external,hive,external_docker,external_docker_hi logger.info("hive sql: " + drop_table_str1) hive_docker """ ${drop_table_str1} """ logger.info("hive sql: " + drop_table_str2) - hive_docker """ ${drop_table_str3} """ + hive_docker """ ${drop_table_str2} """ logger.info("hive sql: " + drop_database_str) hive_docker """ ${drop_database_str} """ logger.info("hive sql: " + create_database_str) From 92b051a183b0114049e41f06b4f797944c214141 Mon Sep 17 00:00:00 2001 From: zhangdong Date: Thu, 6 Nov 2025 15:11:41 +0800 Subject: [PATCH 13/15] add case --- regression-test/suites/mtmv_p0/test_hive_pct_mtmv.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/regression-test/suites/mtmv_p0/test_hive_pct_mtmv.groovy b/regression-test/suites/mtmv_p0/test_hive_pct_mtmv.groovy index 7be15cd1218efc..7ca9c8f0db400f 100644 --- a/regression-test/suites/mtmv_p0/test_hive_pct_mtmv.groovy +++ b/regression-test/suites/mtmv_p0/test_hive_pct_mtmv.groovy @@ -132,7 +132,7 @@ suite("test_hive_pct_mtmv", "p0,external,hive,external_docker,external_docker_hi AS SELECT k1,year,region FROM ${catalog_name}.${dbName}.${tableName1} union all SELECT k1,year,region FROM ${catalog_name}.${dbName}.${tableName2}; """ - order_qt_partitions_1 "select PartitionName,UnsyncTables from partitions('catalog'='internal','database'='${dbName}','table'='${mvName}') order by PartitionId desc;" + order_qt_partitions_1 "select PartitionName,UnsyncTables from partitions('catalog'='internal','database'='${mvDbName}','table'='${mvName}') order by PartitionId desc;" sql """ REFRESH MATERIALIZED VIEW ${mvName} AUTO """ From b316cbc5d3468ae8ee24277bb3d43ebfb3fa95e7 Mon Sep 17 00:00:00 2001 From: zhangdong <493738387@qq.com> Date: Thu, 6 Nov 2025 15:23:47 +0800 Subject: [PATCH 14/15] 1 --- .../data/mtmv_p0/test_hive_pct_mtmv.out | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 regression-test/data/mtmv_p0/test_hive_pct_mtmv.out diff --git a/regression-test/data/mtmv_p0/test_hive_pct_mtmv.out b/regression-test/data/mtmv_p0/test_hive_pct_mtmv.out new file mode 100644 index 00000000000000..6ecb3f835c0a14 --- /dev/null +++ b/regression-test/data/mtmv_p0/test_hive_pct_mtmv.out @@ -0,0 +1,34 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !select_base_table -- +1 2020 bj +2 2020 sh +3 2021 bj +4 2021 sh +5 2020 bj +6 2020 sh +7 2020 tj + +-- !partitions_1 -- +p_bj [test_hive_pct_mtmv_table2, test_hive_pct_mtmv_table1] +p_sh [test_hive_pct_mtmv_table2, test_hive_pct_mtmv_table1] +p_tj [test_hive_pct_mtmv_table2] + +-- !select_mv1 -- +1 2020 bj +2 2020 sh +3 2021 bj +4 2021 sh +5 2020 bj +6 2020 sh +7 2020 tj + +-- !select_mv2 -- +1 2020 bj +2 2020 sh +3 2021 bj +4 2021 sh +5 2020 bj +6 2020 sh +7 2020 bj +7 2020 tj + From be27057a7cfe1bf3e1f3f9ac566353bc91dd6440 Mon Sep 17 00:00:00 2001 From: zhangdong Date: Fri, 7 Nov 2025 17:47:20 +0800 Subject: [PATCH 15/15] fix display --- .../apache/doris/mtmv/MTMVPartitionInfo.java | 20 +++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionInfo.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionInfo.java index 868c8713ef597f..b6fbe19b446fa6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionInfo.java @@ -28,6 +28,8 @@ import com.google.common.collect.Sets; import com.google.gson.annotations.SerializedName; import org.apache.commons.collections.CollectionUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import java.io.IOException; import java.util.List; @@ -38,6 +40,7 @@ * MTMVPartitionInfo */ public class MTMVPartitionInfo implements GsonPostProcessable { + private static final Logger LOG = LogManager.getLogger(MTMVPartitionInfo.class); public enum MTMVPartitionType { FOLLOW_BASE_TABLE, @@ -196,7 +199,7 @@ public String toInfoString() { + "partitionType=" + partitionType + ", pctInfos=" + pctInfos + ", partitionCol='" + partitionCol + '\'' - + ", expr='" + expr + '\'' + + ", expr='" + getExprString() + '\'' + '}'; } @@ -210,11 +213,24 @@ public String toNameString() { + "partitionType=" + partitionType + ", pctInfos=" + pctInfos + ", partitionCol='" + partitionCol + '\'' - + ", expr='" + expr + '\'' + + ", expr='" + getExprString() + '\'' + '}'; } } + private String getExprString() { + if (expr == null) { + return null; + } + try { + return MTMVPartitionExprFactory.getExprService(expr).toSql(this); + } catch (AnalysisException e) { + // should not happen + LOG.warn("getExprString", e); + return null; + } + } + public void compatible(CatalogMgr catalogMgr) throws Exception { if (relatedTable == null) { return;