From 247b60d9fe2686e9c8776de9a343289292951ddd Mon Sep 17 00:00:00 2001 From: zhangdong <493738387@qq.com> Date: Mon, 22 Jul 2024 17:36:35 +0800 Subject: [PATCH 1/3] 1 --- .../doris/job/extensions/mtmv/MTMVTask.java | 3 ++- .../suites/mtmv_p0/test_hive_refresh_mtmv.groovy | 15 --------------- 2 files changed, 2 insertions(+), 16 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVTask.java b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVTask.java index b50d755951f9df..89d334ca4f1b5f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVTask.java +++ b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVTask.java @@ -171,6 +171,7 @@ public void run() throws JobException { // Every time a task is run, the relation is regenerated because baseTables and baseViews may change, // such as deleting a table and creating a view with the same name this.relation = MTMVPlanUtil.generateMTMVRelation(mtmv, ctx); + refreshHmsTable(); // Now, the MTMV first ensures consistency with the data in the cache. // To be completely consistent with hive, you need to manually refresh the cache // refreshHmsTable(); @@ -282,7 +283,7 @@ public void before() throws JobException { } /** - * // Before obtaining information from hmsTable, refresh to ensure that the data is up-to-date + * Before obtaining information from hmsTable, refresh to ensure that the data is up-to-date * * @throws AnalysisException * @throws DdlException diff --git a/regression-test/suites/mtmv_p0/test_hive_refresh_mtmv.groovy b/regression-test/suites/mtmv_p0/test_hive_refresh_mtmv.groovy index ac19ecc0605664..81c840c4246ad0 100644 --- a/regression-test/suites/mtmv_p0/test_hive_refresh_mtmv.groovy +++ b/regression-test/suites/mtmv_p0/test_hive_refresh_mtmv.groovy @@ -104,9 +104,6 @@ suite("test_hive_refresh_mtmv", "p0,external,hive,external_docker,external_docke // hive data change logger.info("hive sql: " + insert_str) hive_docker """ ${insert_str} """ - sql """ - REFRESH catalog ${catalog_name} - """ sql """ REFRESH MATERIALIZED VIEW ${mvName} auto """ @@ -120,9 +117,6 @@ suite("test_hive_refresh_mtmv", "p0,external,hive,external_docker,external_docke """ logger.info("hive sql: " + add_partition2021_str) hive_docker """ ${add_partition2021_str} """ - sql """ - REFRESH catalog ${catalog_name} - """ sql """ REFRESH MATERIALIZED VIEW ${mvName} auto """ @@ -139,9 +133,6 @@ suite("test_hive_refresh_mtmv", "p0,external,hive,external_docker,external_docke """ logger.info("hive sql: " + drop_partition2021_str) hive_docker """ ${drop_partition2021_str} """ - sql """ - REFRESH catalog ${catalog_name} - """ sql """ REFRESH MATERIALIZED VIEW ${mvName} auto """ @@ -157,9 +148,6 @@ suite("test_hive_refresh_mtmv", "p0,external,hive,external_docker,external_docke """ logger.info("hive sql: " + rename_column_str) hive_docker """ ${rename_column_str} """ - sql """ - REFRESH catalog ${catalog_name} - """ sql """ REFRESH MATERIALIZED VIEW ${mvName} complete """ @@ -172,9 +160,6 @@ suite("test_hive_refresh_mtmv", "p0,external,hive,external_docker,external_docke """ logger.info("hive sql: " + recover_column_str) hive_docker """ ${recover_column_str} """ - sql """ - REFRESH catalog ${catalog_name} - """ sql """ REFRESH MATERIALIZED VIEW ${mvName} complete """ From 89de2fe899600a9d27e7276846bc313f6124ecca Mon Sep 17 00:00:00 2001 From: zhangdong <493738387@qq.com> Date: Tue, 23 Jul 2024 10:43:54 +0800 Subject: [PATCH 2/3] 1 --- .../java/org/apache/doris/job/extensions/mtmv/MTMVTask.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVTask.java b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVTask.java index 89d334ca4f1b5f..ab5aaa33586bd4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVTask.java +++ b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVTask.java @@ -168,10 +168,10 @@ public void run() throws JobException { super.getTaskId(), taskSessionContext); } } + refreshHmsTable(); // Every time a task is run, the relation is regenerated because baseTables and baseViews may change, // such as deleting a table and creating a view with the same name this.relation = MTMVPlanUtil.generateMTMVRelation(mtmv, ctx); - refreshHmsTable(); // Now, the MTMV first ensures consistency with the data in the cache. // To be completely consistent with hive, you need to manually refresh the cache // refreshHmsTable(); From 0efb3b88230355e1a1b92bf152703527d4bd6579 Mon Sep 17 00:00:00 2001 From: zhangdong <493738387@qq.com> Date: Wed, 31 Jul 2024 12:35:14 +0800 Subject: [PATCH 3/3] 1 --- .../java/org/apache/doris/job/extensions/mtmv/MTMVTask.java | 3 +-- regression-test/suites/mtmv_p0/test_hive_refresh_mtmv.groovy | 4 ++++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVTask.java b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVTask.java index ab5aaa33586bd4..e53b89806ae4c1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVTask.java +++ b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVTask.java @@ -168,13 +168,12 @@ public void run() throws JobException { super.getTaskId(), taskSessionContext); } } - refreshHmsTable(); // Every time a task is run, the relation is regenerated because baseTables and baseViews may change, // such as deleting a table and creating a view with the same name this.relation = MTMVPlanUtil.generateMTMVRelation(mtmv, ctx); // Now, the MTMV first ensures consistency with the data in the cache. // To be completely consistent with hive, you need to manually refresh the cache - // refreshHmsTable(); + refreshHmsTable(); if (mtmv.getMvPartitionInfo().getPartitionType() != MTMVPartitionType.SELF_MANAGE) { MTMVPartitionUtil.alignMvPartition(mtmv); } diff --git a/regression-test/suites/mtmv_p0/test_hive_refresh_mtmv.groovy b/regression-test/suites/mtmv_p0/test_hive_refresh_mtmv.groovy index 81c840c4246ad0..75ec61a518b060 100644 --- a/regression-test/suites/mtmv_p0/test_hive_refresh_mtmv.groovy +++ b/regression-test/suites/mtmv_p0/test_hive_refresh_mtmv.groovy @@ -160,6 +160,10 @@ suite("test_hive_refresh_mtmv", "p0,external,hive,external_docker,external_docke """ logger.info("hive sql: " + recover_column_str) hive_docker """ ${recover_column_str} """ + // schema change need refresh catalog + sql """ + REFRESH catalog ${catalog_name} + """ sql """ REFRESH MATERIALIZED VIEW ${mvName} complete """