From ac64d064e7560fb3af93d57097c4de42516d6e68 Mon Sep 17 00:00:00 2001 From: Pxl Date: Fri, 9 Aug 2024 17:06:11 +0800 Subject: [PATCH 1/2] [Bug](materialized-view) fix analyze where clause failed on mv (#39061) ## Proposed changes fix analyze where clause failed on mv do not analyze slot after replaceSlot to avoid duplicate columns in desc --- .../org/apache/doris/analysis/SlotRef.java | 5 ++ .../data/mv_p0/routine_load_mapping/test2 | 0 .../routine_load_mapping.groovy | 66 +++++++++++++++++++ 3 files changed, 71 insertions(+) create mode 100644 regression-test/data/mv_p0/routine_load_mapping/test2 diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/SlotRef.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/SlotRef.java index 4cd58f23650b52..92f5224e0bc046 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/SlotRef.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/SlotRef.java @@ -694,6 +694,11 @@ public Expr getResultValue(boolean forPushDownPredicatesToView) throws AnalysisE @Override public void replaceSlot(TupleDescriptor tuple) { + // do not analyze slot after replaceSlot to avoid duplicate columns in desc desc = tuple.getColumnSlot(col); + type = desc.getType(); + if (!isAnalyzed) { + analysisDone(); + } } } diff --git a/regression-test/data/mv_p0/routine_load_mapping/test2 b/regression-test/data/mv_p0/routine_load_mapping/test2 new file mode 100644 index 00000000000000..e69de29bb2d1d6 diff --git a/regression-test/suites/mv_p0/routine_load_mapping/routine_load_mapping.groovy b/regression-test/suites/mv_p0/routine_load_mapping/routine_load_mapping.groovy index 1188e3031c0ff7..ca3a3560e01cfe 100644 --- a/regression-test/suites/mv_p0/routine_load_mapping/routine_load_mapping.groovy +++ b/regression-test/suites/mv_p0/routine_load_mapping/routine_load_mapping.groovy @@ -49,4 +49,70 @@ suite ("routine_load_mapping") { qt_select "select * from test order by 1,2,3;" qt_select_mv "select * from test index m_view order by 1,2;" + + + sql """ DROP TABLE IF EXISTS rt_new; """ + + sql """ + CREATE TABLE `rt_new` ( + `battery_id` VARCHAR(50) NULL , + `create_time` DATETIME(3) NULL , + `imei` VARCHAR(50) NULL , + `event_id` VARCHAR(50) NULL , + `event_name` VARCHAR(50) NULL, + `heart_type` INT NULL +) ENGINE=OLAP +DUPLICATE KEY(`battery_id`, `create_time`) +PARTITION BY RANGE(`create_time`) +(PARTITION p20240421 VALUES [('2024-04-21 00:00:00'), ('2024-04-22 00:00:00')), +PARTITION p20240422 VALUES [('2024-04-22 00:00:00'), ('2024-04-23 00:00:00')), +PARTITION p20240804 VALUES [('2024-08-04 00:00:00'), ('2024-08-05 00:00:00'))) +DISTRIBUTED BY HASH(`battery_id`) BUCKETS AUTO +PROPERTIES ( +"replication_allocation" = "tag.location.default: 1", +"file_cache_ttl_seconds" = "0", +"is_being_synced" = "false", +"dynamic_partition.enable" = "true", +"dynamic_partition.time_unit" = "DAY", +"dynamic_partition.time_zone" = "Asia/Shanghai", +"dynamic_partition.start" = "-2147483648", +"dynamic_partition.end" = "3", +"dynamic_partition.prefix" = "p", +"dynamic_partition.buckets" = "10", +"dynamic_partition.create_history_partition" = "true", +"dynamic_partition.history_partition_num" = "100", +"dynamic_partition.hot_partition_num" = "0", +"dynamic_partition.reserved_history_periods" = "NULL", +"storage_medium" = "hdd", +"storage_format" = "V2", +"inverted_index_storage_format" = "V2", +"light_schema_change" = "true", +"disable_auto_compaction" = "false", +"enable_single_replica_compaction" = "false", +"group_commit_interval_ms" = "10000", +"group_commit_data_bytes" = "134217728" +); + """ + + createMV("""CREATE MATERIALIZED VIEW location_rt_mv AS + SELECT + battery_id, + create_time + FROM + rt_new + WHERE + heart_type = 1 + ;""") + + sql """ ALTER TABLE rt_new MODIFY COLUMN event_id VARCHAR(51) NULL;""" + Thread.sleep(1000) + + streamLoad { + table "rt_new" + set 'column_separator', ',' + set 'columns', '`battery_id`,`create_time`,`imei`,`event_id`,`event_name`,`heart_type`' + + file './test2' + time 10000 // limit inflight 10s + } } From 1a49075a2ce62c2f10c77d139ac15832d772e179 Mon Sep 17 00:00:00 2001 From: BiteTheDDDDt Date: Tue, 13 Aug 2024 14:29:09 +0800 Subject: [PATCH 2/2] fix --- .../mv_p0/routine_load_mapping/routine_load_mapping.groovy | 2 -- 1 file changed, 2 deletions(-) diff --git a/regression-test/suites/mv_p0/routine_load_mapping/routine_load_mapping.groovy b/regression-test/suites/mv_p0/routine_load_mapping/routine_load_mapping.groovy index ca3a3560e01cfe..491282f7c851ab 100644 --- a/regression-test/suites/mv_p0/routine_load_mapping/routine_load_mapping.groovy +++ b/regression-test/suites/mv_p0/routine_load_mapping/routine_load_mapping.groovy @@ -70,8 +70,6 @@ PARTITION p20240804 VALUES [('2024-08-04 00:00:00'), ('2024-08-05 00:00:00'))) DISTRIBUTED BY HASH(`battery_id`) BUCKETS AUTO PROPERTIES ( "replication_allocation" = "tag.location.default: 1", -"file_cache_ttl_seconds" = "0", -"is_being_synced" = "false", "dynamic_partition.enable" = "true", "dynamic_partition.time_unit" = "DAY", "dynamic_partition.time_zone" = "Asia/Shanghai",