From 26cc15293c2932ead17e00ba64425afc05cb73d3 Mon Sep 17 00:00:00 2001 From: starocean999 <12095047@qq.com> Date: Fri, 29 Mar 2024 03:21:21 +0800 Subject: [PATCH 1/2] [fix](nereids)column name should be case insensitive when selecting mv --- .../AbstractSelectMaterializedIndexRule.java | 10 +- .../suites/nereids_p0/test_mv_select.groovy | 169 ++++++++++++++++++ 2 files changed, 175 insertions(+), 4 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/AbstractSelectMaterializedIndexRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/AbstractSelectMaterializedIndexRule.java index 8ecabcd8918c3c..c77cda4c8f8125 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/AbstractSelectMaterializedIndexRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/AbstractSelectMaterializedIndexRule.java @@ -219,8 +219,10 @@ protected static List matchPrefixMost( Set predicates, Map exprIdToName) { Map> split = filterCanUsePrefixIndexAndSplitByEquality(predicates, exprIdToName); - Set equalColNames = split.getOrDefault(true, ImmutableSet.of()); - Set nonEqualColNames = split.getOrDefault(false, ImmutableSet.of()); + Set equalColNames = split.getOrDefault(true, ImmutableSet.of()).stream() + .map(String::toLowerCase).collect(Collectors.toSet()); + Set nonEqualColNames = split.getOrDefault(false, ImmutableSet.of()).stream() + .map(String::toLowerCase).collect(Collectors.toSet()); if (!(equalColNames.isEmpty() && nonEqualColNames.isEmpty())) { List matchingResult = matchKeyPrefixMost(scan.getTable(), candidate, @@ -358,9 +360,9 @@ private static int indexKeyPrefixMatchCount( Set nonEqualColNames) { int matchCount = 0; for (Column column : table.getSchemaByIndexId(index.getId())) { - if (equalColNames.contains(normalizeName(column.getNameWithoutMvPrefix()))) { + if (equalColNames.contains(normalizeName(column.getNameWithoutMvPrefix().toLowerCase()))) { matchCount++; - } else if (nonEqualColNames.contains(normalizeName(column.getNameWithoutMvPrefix()))) { + } else if (nonEqualColNames.contains(normalizeName(column.getNameWithoutMvPrefix().toLowerCase()))) { // un-equivalence predicate's columns can match only first column in index. matchCount++; break; diff --git a/regression-test/suites/nereids_p0/test_mv_select.groovy b/regression-test/suites/nereids_p0/test_mv_select.groovy index 096cfd50fab16b..0b474856cbfdec 100644 --- a/regression-test/suites/nereids_p0/test_mv_select.groovy +++ b/regression-test/suites/nereids_p0/test_mv_select.groovy @@ -42,4 +42,173 @@ suite("test_mv_select") { group by Uid;""") contains "mv_test_table_t" } + + sql """drop table if exists SkuUniqDailyCounter""" + sql """CREATE TABLE `SkuUniqDailyCounter` ( + `ProductId` bigint(20) NOT NULL, + `DateCode` int(11) NOT NULL, + `BrandCid` bigint(20) NOT NULL, + `ShopId` varchar(30) NOT NULL, + `BigCid` int(11) REPLACE NULL, + `FirstCid` int(11) REPLACE NULL, + `SecondCid` int(11) REPLACE NULL, + `ThirdCid` int(11) REPLACE NULL, + `FourthCid` int(11) REPLACE NULL, + `Price` int(11) REPLACE NULL, + `CosRatio` int(11) REPLACE NULL, + `PmtOrders` int(11) REPLACE NULL, + `PmtViews` int(11) REPLACE NULL, + `PmtUsers` int(11) REPLACE NULL, + `VideoPrice` int(11) REPLACE NULL, + `LivePrice` int(11) REPLACE NULL, + `LiveSales` int(11) REPLACE NULL, + `LiveOrders` int(11) REPLACE NULL, + `LiveGmv` bigint(20) REPLACE NULL, + `VideoSales` int(11) REPLACE NULL, + `VideoGmv` bigint(20) REPLACE NULL, + `SubtractSales` int(11) REPLACE NULL, + `SubtractGmv` bigint(20) REPLACE NULL, + `FinalSales` int(11) REPLACE NULL, + `FinalGmv` bigint(20) REPLACE NULL, + `RoomIds` bitmap BITMAP_UNION NOT NULL, + `LiveUids` bitmap BITMAP_UNION NOT NULL, + `VideoAwemeIds` bitmap BITMAP_UNION NOT NULL, + `VideoUids` bitmap BITMAP_UNION NOT NULL, + `Uids` bitmap BITMAP_UNION NOT NULL, + `PlayCount` bigint(20) REPLACE NULL, + `LikeCount` bigint(20) REPLACE NULL, + `SelfViews` int(11) REPLACE NULL, + `FinalViews` int(11) REPLACE NULL, + `SelfSales` int(11) REPLACE NULL, + `LiveUsers` bigint(20) REPLACE NULL, + `LivePriceArray` bitmap BITMAP_UNION NOT NULL, + `VideoPriceArray` bitmap BITMAP_UNION NOT NULL, + `NearlyLivePrice` int(11) REPLACE NULL, + `NearlyVideoPrice` int(11) REPLACE NULL, + `NaturalSales` int(11) REPLACE NULL, + `NaturalGmv` bigint(20) REPLACE NULL, + INDEX idx_shopid (`ShopId`) USING BITMAP COMMENT '' + ) ENGINE=OLAP + AGGREGATE KEY(`ProductId`, `DateCode`, `BrandCid`, `ShopId`) + COMMENT 'OLAP' + DISTRIBUTED BY HASH(`ProductId`) BUCKETS 8 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1", + "is_being_synced" = "false", + "storage_format" = "V2", + "light_schema_change" = "true", + "disable_auto_compaction" = "false", + "enable_single_replica_compaction" = "false" + );""" + def delta_time = 1000 + def wait_for_latest_op_on_table_finish = { table_name, OpTimeout -> + for(int t = delta_time; t <= OpTimeout; t += delta_time){ + alter_res = sql """SHOW ALTER TABLE COLUMN WHERE TableName = "${table_name}" ORDER BY CreateTime DESC LIMIT 1;""" + alter_res = alter_res.toString() + if(alter_res.contains("FINISHED")) { + sleep(10000) // wait change table state to normal + logger.info(table_name + " latest alter job finished, detail: " + alter_res) + break + } + useTime = t + sleep(delta_time) + } + assertTrue(useTime <= OpTimeout, "wait_for_latest_op_on_table_finish timeout") + } + sql """ALTER TABLE SkuUniqDailyCounter + ADD ROLLUP rollup_index_shopid(ShopId, + DateCode, + ProductId, + BrandCid, + BigCid, + FirstCid, + SecondCid, + ThirdCid, + FourthCid, + Price, + CosRatio, + PmtOrders, + PmtViews, + PmtUsers, + VideoPrice, + LivePrice, + LiveSales, + LiveOrders, + LiveGmv, + VideoSales, + VideoGmv, + SubtractSales, + SubtractGmv, + FinalSales, + FinalGmv, + RoomIds, + LiveUids, + VideoAwemeIds, + VideoUids, + Uids, + PlayCount, + LikeCount, + SelfViews, + FinalViews, + SelfSales, + LiveUsers, + LivePriceArray, + VideoPriceArray, + NearlyLivePrice, + NearlyVideoPrice, + NaturalSales, + NaturalGmv + ); """ + wait_for_latest_op_on_table_finish("SkuUniqDailyCounter",60000); + sql """ALTER TABLE SkuUniqDailyCounter + ADD ROLLUP rollup_index_brandcid(BrandCid, + DateCode, + ProductId, + ShopId, + BigCid, + FirstCid, + SecondCid, + ThirdCid, + FourthCid, + Price, + CosRatio, + PmtOrders, + PmtViews, + PmtUsers, + VideoPrice, + LivePrice, + LiveSales, + LiveOrders, + LiveGmv, + VideoSales, + VideoGmv, + SubtractSales, + SubtractGmv, + FinalSales, + FinalGmv, + RoomIds, + LiveUids, + VideoAwemeIds, + VideoUids, + Uids, + PlayCount, + LikeCount, + SelfViews, + FinalViews, + SelfSales, + LiveUsers, + LivePriceArray, + VideoPriceArray, + NearlyLivePrice, + NearlyVideoPrice, + NaturalSales, + NaturalGmv + );""" + wait_for_latest_op_on_table_finish("SkuUniqDailyCounter",60000); + + explain { + sql ("""select ProductId,sum(FinalSales) Sales,Sum(FinalGmv) Gmv ,SUM(NaturalSales) NaturalSales,sum(NaturalGmv) NaturalGmv ,BITMAP_UNION_COUNT(Uids) as Users from SkuUniqDailyCounter where BrandCid=742502946 and DateCode >=20240315 and DateCode <= 20240328 Group by ProductId order by Gmv DESC LIMIT 10;""") + contains "rollup_index_brandcid" + } + } \ No newline at end of file From d4992e1315811c45fe52196164a7932759c6b0a6 Mon Sep 17 00:00:00 2001 From: starocean999 <12095047@qq.com> Date: Fri, 29 Mar 2024 10:52:22 +0800 Subject: [PATCH 2/2] update case --- .../suites/nereids_p0/test_mv_select.groovy | 155 ++---------------- 1 file changed, 18 insertions(+), 137 deletions(-) diff --git a/regression-test/suites/nereids_p0/test_mv_select.groovy b/regression-test/suites/nereids_p0/test_mv_select.groovy index 0b474856cbfdec..4d7c93828e16ea 100644 --- a/regression-test/suites/nereids_p0/test_mv_select.groovy +++ b/regression-test/suites/nereids_p0/test_mv_select.groovy @@ -45,60 +45,16 @@ suite("test_mv_select") { sql """drop table if exists SkuUniqDailyCounter""" sql """CREATE TABLE `SkuUniqDailyCounter` ( - `ProductId` bigint(20) NOT NULL, - `DateCode` int(11) NOT NULL, - `BrandCid` bigint(20) NOT NULL, - `ShopId` varchar(30) NOT NULL, - `BigCid` int(11) REPLACE NULL, - `FirstCid` int(11) REPLACE NULL, - `SecondCid` int(11) REPLACE NULL, - `ThirdCid` int(11) REPLACE NULL, - `FourthCid` int(11) REPLACE NULL, - `Price` int(11) REPLACE NULL, - `CosRatio` int(11) REPLACE NULL, - `PmtOrders` int(11) REPLACE NULL, - `PmtViews` int(11) REPLACE NULL, - `PmtUsers` int(11) REPLACE NULL, - `VideoPrice` int(11) REPLACE NULL, - `LivePrice` int(11) REPLACE NULL, - `LiveSales` int(11) REPLACE NULL, - `LiveOrders` int(11) REPLACE NULL, - `LiveGmv` bigint(20) REPLACE NULL, - `VideoSales` int(11) REPLACE NULL, - `VideoGmv` bigint(20) REPLACE NULL, - `SubtractSales` int(11) REPLACE NULL, - `SubtractGmv` bigint(20) REPLACE NULL, - `FinalSales` int(11) REPLACE NULL, - `FinalGmv` bigint(20) REPLACE NULL, - `RoomIds` bitmap BITMAP_UNION NOT NULL, - `LiveUids` bitmap BITMAP_UNION NOT NULL, - `VideoAwemeIds` bitmap BITMAP_UNION NOT NULL, - `VideoUids` bitmap BITMAP_UNION NOT NULL, - `Uids` bitmap BITMAP_UNION NOT NULL, - `PlayCount` bigint(20) REPLACE NULL, - `LikeCount` bigint(20) REPLACE NULL, - `SelfViews` int(11) REPLACE NULL, - `FinalViews` int(11) REPLACE NULL, - `SelfSales` int(11) REPLACE NULL, - `LiveUsers` bigint(20) REPLACE NULL, - `LivePriceArray` bitmap BITMAP_UNION NOT NULL, - `VideoPriceArray` bitmap BITMAP_UNION NOT NULL, - `NearlyLivePrice` int(11) REPLACE NULL, - `NearlyVideoPrice` int(11) REPLACE NULL, - `NaturalSales` int(11) REPLACE NULL, - `NaturalGmv` bigint(20) REPLACE NULL, - INDEX idx_shopid (`ShopId`) USING BITMAP COMMENT '' + `Pd` bigint(20) NOT NULL, + `Dc` int(11) NOT NULL, + `Bc` bigint(20) NOT NULL, + `Fs` int(11) REPLACE NULL ) ENGINE=OLAP - AGGREGATE KEY(`ProductId`, `DateCode`, `BrandCid`, `ShopId`) + AGGREGATE KEY(`Pd`, `Dc`, `Bc`) COMMENT 'OLAP' - DISTRIBUTED BY HASH(`ProductId`) BUCKETS 8 + DISTRIBUTED BY HASH(`Dc`) BUCKETS 8 PROPERTIES ( - "replication_allocation" = "tag.location.default: 1", - "is_being_synced" = "false", - "storage_format" = "V2", - "light_schema_change" = "true", - "disable_auto_compaction" = "false", - "enable_single_replica_compaction" = "false" + "replication_allocation" = "tag.location.default: 1" );""" def delta_time = 1000 def wait_for_latest_op_on_table_finish = { table_name, OpTimeout -> @@ -116,99 +72,24 @@ suite("test_mv_select") { assertTrue(useTime <= OpTimeout, "wait_for_latest_op_on_table_finish timeout") } sql """ALTER TABLE SkuUniqDailyCounter - ADD ROLLUP rollup_index_shopid(ShopId, - DateCode, - ProductId, - BrandCid, - BigCid, - FirstCid, - SecondCid, - ThirdCid, - FourthCid, - Price, - CosRatio, - PmtOrders, - PmtViews, - PmtUsers, - VideoPrice, - LivePrice, - LiveSales, - LiveOrders, - LiveGmv, - VideoSales, - VideoGmv, - SubtractSales, - SubtractGmv, - FinalSales, - FinalGmv, - RoomIds, - LiveUids, - VideoAwemeIds, - VideoUids, - Uids, - PlayCount, - LikeCount, - SelfViews, - FinalViews, - SelfSales, - LiveUsers, - LivePriceArray, - VideoPriceArray, - NearlyLivePrice, - NearlyVideoPrice, - NaturalSales, - NaturalGmv + ADD ROLLUP rollup_index_shopid( + Dc, + Bc, + Pd, + Fs ); """ wait_for_latest_op_on_table_finish("SkuUniqDailyCounter",60000); sql """ALTER TABLE SkuUniqDailyCounter - ADD ROLLUP rollup_index_brandcid(BrandCid, - DateCode, - ProductId, - ShopId, - BigCid, - FirstCid, - SecondCid, - ThirdCid, - FourthCid, - Price, - CosRatio, - PmtOrders, - PmtViews, - PmtUsers, - VideoPrice, - LivePrice, - LiveSales, - LiveOrders, - LiveGmv, - VideoSales, - VideoGmv, - SubtractSales, - SubtractGmv, - FinalSales, - FinalGmv, - RoomIds, - LiveUids, - VideoAwemeIds, - VideoUids, - Uids, - PlayCount, - LikeCount, - SelfViews, - FinalViews, - SelfSales, - LiveUsers, - LivePriceArray, - VideoPriceArray, - NearlyLivePrice, - NearlyVideoPrice, - NaturalSales, - NaturalGmv + ADD ROLLUP rollup_index_brandcid( + Bc, + Dc, + Pd, + Fs );""" wait_for_latest_op_on_table_finish("SkuUniqDailyCounter",60000); explain { - sql ("""select ProductId,sum(FinalSales) Sales,Sum(FinalGmv) Gmv ,SUM(NaturalSales) NaturalSales,sum(NaturalGmv) NaturalGmv ,BITMAP_UNION_COUNT(Uids) as Users from SkuUniqDailyCounter where BrandCid=742502946 and DateCode >=20240315 and DateCode <= 20240328 Group by ProductId order by Gmv DESC LIMIT 10;""") + sql ("""select sum(Fs) Sales from SkuUniqDailyCounter where Bc=742502946 and Dc >=20240315 and Dc <= 20240328;""") contains "rollup_index_brandcid" } - } \ No newline at end of file