From 144c94f89dedef6862c36cdaf722774d965df0b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=A7=9C=E5=87=AF?= Date: Thu, 23 May 2024 19:35:14 +0800 Subject: [PATCH 1/7] [Update](inverted index) set default inverted index storage format to v2 --- fe/fe-common/src/main/java/org/apache/doris/common/Config.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java index db2f2700f80318..615e034e56820b 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java @@ -2620,7 +2620,7 @@ public class Config extends ConfigBase { "倒排索引默认存储格式", "Default storage format of inverted index, the default value is V1." }) - public static String inverted_index_storage_format = "V1"; + public static String inverted_index_storage_format = "V2"; @ConfField(description = { "是否开启 Proxy Protocol 支持", From 404fdbc6db6a95f56878dd355c72168b02bc68aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=A7=9C=E5=87=AF?= Date: Thu, 23 May 2024 20:10:22 +0800 Subject: [PATCH 2/7] fix fe ut --- .../analysis/CreateTableAsSelectStmtTest.java | 42 +++++++++---------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateTableAsSelectStmtTest.java b/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateTableAsSelectStmtTest.java index be1ddf7bc4a305..f52a7aea6a2e93 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateTableAsSelectStmtTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateTableAsSelectStmtTest.java @@ -96,7 +96,7 @@ public void testDecimal() throws Exception { + "\"is_being_synced\" = \"false\",\n" + "\"storage_medium\" = \"hdd\",\n" + "\"storage_format\" = \"V2\",\n" - + "\"inverted_index_storage_format\" = \"V1\",\n" + + "\"inverted_index_storage_format\" = \"V2\",\n" + "\"light_schema_change\" = \"true\",\n" + "\"disable_auto_compaction\" = \"false\",\n" + "\"enable_single_replica_compaction\" = \"false\",\n" @@ -122,7 +122,7 @@ public void testDecimal() throws Exception { + "\"is_being_synced\" = \"false\",\n" + "\"storage_medium\" = \"hdd\",\n" + "\"storage_format\" = \"V2\",\n" - + "\"inverted_index_storage_format\" = \"V1\",\n" + + "\"inverted_index_storage_format\" = \"V2\",\n" + "\"light_schema_change\" = \"true\",\n" + "\"disable_auto_compaction\" = \"false\",\n" + "\"enable_single_replica_compaction\" = \"false\",\n" @@ -144,7 +144,7 @@ public void testDecimal() throws Exception { + "\"is_being_synced\" = \"false\",\n" + "\"storage_medium\" = \"hdd\",\n" + "\"storage_format\" = \"V2\",\n" - + "\"inverted_index_storage_format\" = \"V1\",\n" + + "\"inverted_index_storage_format\" = \"V2\",\n" + "\"light_schema_change\" = \"true\",\n" + "\"disable_auto_compaction\" = \"false\",\n" + "\"enable_single_replica_compaction\" = \"false\",\n" @@ -184,7 +184,7 @@ public void testVarchar() throws Exception { + "\"is_being_synced\" = \"false\",\n" + "\"storage_medium\" = \"hdd\",\n" + "\"storage_format\" = \"V2\",\n" - + "\"inverted_index_storage_format\" = \"V1\",\n" + + "\"inverted_index_storage_format\" = \"V2\",\n" + "\"light_schema_change\" = \"true\",\n" + "\"disable_auto_compaction\" = \"false\",\n" + "\"enable_single_replica_compaction\" = \"false\",\n" @@ -213,7 +213,7 @@ public void testFunction() throws Exception { + "\"is_being_synced\" = \"false\",\n" + "\"storage_medium\" = \"hdd\",\n" + "\"storage_format\" = \"V2\",\n" - + "\"inverted_index_storage_format\" = \"V1\",\n" + + "\"inverted_index_storage_format\" = \"V2\",\n" + "\"light_schema_change\" = \"true\",\n" + "\"disable_auto_compaction\" = \"false\",\n" + "\"enable_single_replica_compaction\" = \"false\",\n" @@ -244,7 +244,7 @@ public void testFunction() throws Exception { + "\"is_being_synced\" = \"false\",\n" + "\"storage_medium\" = \"hdd\",\n" + "\"storage_format\" = \"V2\",\n" - + "\"inverted_index_storage_format\" = \"V1\",\n" + + "\"inverted_index_storage_format\" = \"V2\",\n" + "\"light_schema_change\" = \"true\",\n" + "\"disable_auto_compaction\" = \"false\",\n" + "\"enable_single_replica_compaction\" = \"false\",\n" @@ -272,7 +272,7 @@ public void testAlias() throws Exception { + "\"is_being_synced\" = \"false\",\n" + "\"storage_medium\" = \"hdd\",\n" + "\"storage_format\" = \"V2\",\n" - + "\"inverted_index_storage_format\" = \"V1\",\n" + + "\"inverted_index_storage_format\" = \"V2\",\n" + "\"light_schema_change\" = \"true\",\n" + "\"disable_auto_compaction\" = \"false\",\n" + "\"enable_single_replica_compaction\" = \"false\",\n" @@ -296,7 +296,7 @@ public void testAlias() throws Exception { + "\"is_being_synced\" = \"false\",\n" + "\"storage_medium\" = \"hdd\",\n" + "\"storage_format\" = \"V2\",\n" - + "\"inverted_index_storage_format\" = \"V1\",\n" + + "\"inverted_index_storage_format\" = \"V2\",\n" + "\"light_schema_change\" = \"true\",\n" + "\"disable_auto_compaction\" = \"false\",\n" + "\"enable_single_replica_compaction\" = \"false\",\n" @@ -327,7 +327,7 @@ public void testJoin() throws Exception { + "\"is_being_synced\" = \"false\",\n" + "\"storage_medium\" = \"hdd\",\n" + "\"storage_format\" = \"V2\",\n" - + "\"inverted_index_storage_format\" = \"V1\",\n" + + "\"inverted_index_storage_format\" = \"V2\",\n" + "\"light_schema_change\" = \"true\",\n" + "\"disable_auto_compaction\" = \"false\",\n" + "\"enable_single_replica_compaction\" = \"false\",\n" @@ -355,7 +355,7 @@ public void testJoin() throws Exception { + "\"is_being_synced\" = \"false\",\n" + "\"storage_medium\" = \"hdd\",\n" + "\"storage_format\" = \"V2\",\n" - + "\"inverted_index_storage_format\" = \"V1\",\n" + + "\"inverted_index_storage_format\" = \"V2\",\n" + "\"light_schema_change\" = \"true\",\n" + "\"disable_auto_compaction\" = \"false\",\n" + "\"enable_single_replica_compaction\" = \"false\",\n" @@ -387,7 +387,7 @@ public void testName() throws Exception { + "\"is_being_synced\" = \"false\",\n" + "\"storage_medium\" = \"hdd\",\n" + "\"storage_format\" = \"V2\",\n" - + "\"inverted_index_storage_format\" = \"V1\",\n" + + "\"inverted_index_storage_format\" = \"V2\",\n" + "\"light_schema_change\" = \"true\",\n" + "\"disable_auto_compaction\" = \"false\",\n" + "\"enable_single_replica_compaction\" = \"false\",\n" @@ -416,7 +416,7 @@ public void testUnion() throws Exception { + "\"is_being_synced\" = \"false\",\n" + "\"storage_medium\" = \"hdd\",\n" + "\"storage_format\" = \"V2\",\n" - + "\"inverted_index_storage_format\" = \"V1\",\n" + + "\"inverted_index_storage_format\" = \"V2\",\n" + "\"light_schema_change\" = \"true\",\n" + "\"disable_auto_compaction\" = \"false\",\n" + "\"enable_single_replica_compaction\" = \"false\",\n" @@ -444,7 +444,7 @@ public void testCte() throws Exception { + "\"is_being_synced\" = \"false\",\n" + "\"storage_medium\" = \"hdd\",\n" + "\"storage_format\" = \"V2\",\n" - + "\"inverted_index_storage_format\" = \"V1\",\n" + + "\"inverted_index_storage_format\" = \"V2\",\n" + "\"light_schema_change\" = \"true\",\n" + "\"disable_auto_compaction\" = \"false\",\n" + "\"enable_single_replica_compaction\" = \"false\",\n" @@ -468,7 +468,7 @@ public void testCte() throws Exception { + "\"is_being_synced\" = \"false\",\n" + "\"storage_medium\" = \"hdd\",\n" + "\"storage_format\" = \"V2\",\n" - + "\"inverted_index_storage_format\" = \"V1\",\n" + + "\"inverted_index_storage_format\" = \"V2\",\n" + "\"light_schema_change\" = \"true\",\n" + "\"disable_auto_compaction\" = \"false\",\n" + "\"enable_single_replica_compaction\" = \"false\",\n" @@ -499,7 +499,7 @@ public void testPartition() throws Exception { + "\"is_being_synced\" = \"false\",\n" + "\"storage_medium\" = \"hdd\",\n" + "\"storage_format\" = \"V2\",\n" - + "\"inverted_index_storage_format\" = \"V1\",\n" + + "\"inverted_index_storage_format\" = \"V2\",\n" + "\"light_schema_change\" = \"true\",\n" + "\"disable_auto_compaction\" = \"false\",\n" + "\"enable_single_replica_compaction\" = \"false\",\n" @@ -529,7 +529,7 @@ public void testDefaultTimestamp() throws Exception { + "\"is_being_synced\" = \"false\",\n" + "\"storage_medium\" = \"hdd\",\n" + "\"storage_format\" = \"V2\",\n" - + "\"inverted_index_storage_format\" = \"V1\",\n" + + "\"inverted_index_storage_format\" = \"V2\",\n" + "\"light_schema_change\" = \"true\",\n" + "\"disable_auto_compaction\" = \"false\",\n" + "\"enable_single_replica_compaction\" = \"false\",\n" @@ -558,7 +558,7 @@ public void testAggValue() throws Exception { + "\"is_being_synced\" = \"false\",\n" + "\"storage_medium\" = \"hdd\",\n" + "\"storage_format\" = \"V2\",\n" - + "\"inverted_index_storage_format\" = \"V1\",\n" + + "\"inverted_index_storage_format\" = \"V2\",\n" + "\"light_schema_change\" = \"true\",\n" + "\"disable_auto_compaction\" = \"false\",\n" + "\"enable_single_replica_compaction\" = \"false\",\n" @@ -588,7 +588,7 @@ public void testUseKeyType() throws Exception { + "\"is_being_synced\" = \"false\",\n" + "\"storage_medium\" = \"hdd\",\n" + "\"storage_format\" = \"V2\",\n" - + "\"inverted_index_storage_format\" = \"V1\",\n" + + "\"inverted_index_storage_format\" = \"V2\",\n" + "\"enable_unique_key_merge_on_write\" = \"true\",\n" + "\"light_schema_change\" = \"true\",\n" + "\"disable_auto_compaction\" = \"false\",\n" @@ -643,7 +643,7 @@ public void testQuerySchema() throws Exception { + "\"is_being_synced\" = \"false\",\n" + "\"storage_medium\" = \"hdd\",\n" + "\"storage_format\" = \"V2\",\n" - + "\"inverted_index_storage_format\" = \"V1\",\n" + + "\"inverted_index_storage_format\" = \"V2\",\n" + "\"light_schema_change\" = \"true\",\n" + "\"disable_auto_compaction\" = \"false\",\n" + "\"enable_single_replica_compaction\" = \"false\",\n" @@ -665,7 +665,7 @@ public void testQuerySchema() throws Exception { + "\"is_being_synced\" = \"false\",\n" + "\"storage_medium\" = \"hdd\",\n" + "\"storage_format\" = \"V2\",\n" - + "\"inverted_index_storage_format\" = \"V1\",\n" + + "\"inverted_index_storage_format\" = \"V2\",\n" + "\"light_schema_change\" = \"true\",\n" + "\"disable_auto_compaction\" = \"false\",\n" + "\"enable_single_replica_compaction\" = \"false\",\n" @@ -701,7 +701,7 @@ public void testVarcharLength() throws Exception { + "\"is_being_synced\" = \"false\",\n" + "\"storage_medium\" = \"hdd\",\n" + "\"storage_format\" = \"V2\",\n" - + "\"inverted_index_storage_format\" = \"V1\",\n" + + "\"inverted_index_storage_format\" = \"V2\",\n" + "\"light_schema_change\" = \"true\",\n" + "\"disable_auto_compaction\" = \"false\",\n" + "\"enable_single_replica_compaction\" = \"false\",\n" From 1d1c191262c491d373560d15776444ed52a67334 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=A7=9C=E5=87=AF?= Date: Fri, 24 May 2024 17:56:01 +0800 Subject: [PATCH 3/7] fix case --- .../data/show_p0/test_show_create_table_and_views.out | 8 ++++---- .../show_p0/test_show_create_table_and_views_nereids.out | 8 ++++---- .../test_calc_crc_fault_injection.groovy | 6 +++--- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/regression-test/data/show_p0/test_show_create_table_and_views.out b/regression-test/data/show_p0/test_show_create_table_and_views.out index 55b41677ac900a..0bc4f97496ce8f 100644 --- a/regression-test/data/show_p0/test_show_create_table_and_views.out +++ b/regression-test/data/show_p0/test_show_create_table_and_views.out @@ -1,6 +1,6 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !show -- -show_create_table_and_views_table CREATE TABLE `show_create_table_and_views_table` (\n `user_id` LARGEINT NOT NULL,\n `good_id` LARGEINT NOT NULL,\n `cost` BIGINT SUM NULL DEFAULT "0"\n) ENGINE=OLAP\nAGGREGATE KEY(`user_id`, `good_id`)\nCOMMENT 'OLAP'\nPARTITION BY RANGE(`good_id`)\n(PARTITION p1 VALUES [("-170141183460469231731687303715884105728"), ("100")),\nPARTITION p2 VALUES [("100"), ("200")),\nPARTITION p3 VALUES [("200"), ("300")),\nPARTITION p4 VALUES [("300"), ("400")),\nPARTITION p5 VALUES [("400"), ("500")),\nPARTITION p6 VALUES [("500"), ("600")),\nPARTITION p7 VALUES [("600"), (MAXVALUE)))\nDISTRIBUTED BY HASH(`user_id`) BUCKETS 2\nPROPERTIES (\n"replication_allocation" = "tag.location.default: 1",\n"min_load_replica_num" = "-1",\n"is_being_synced" = "false",\n"storage_medium" = "hdd",\n"storage_format" = "V2",\n"inverted_index_storage_format" = "V1",\n"light_schema_change" = "true",\n"disable_auto_compaction" = "false",\n"binlog.enable" = "false",\n"binlog.ttl_seconds" = "86400",\n"binlog.max_bytes" = "9223372036854775807",\n"binlog.max_history_nums" = "9223372036854775807",\n"enable_single_replica_compaction" = "false",\n"group_commit_interval_ms" = "10000",\n"group_commit_data_bytes" = "134217728"\n); +show_create_table_and_views_table CREATE TABLE `show_create_table_and_views_table` (\n `user_id` LARGEINT NOT NULL,\n `good_id` LARGEINT NOT NULL,\n `cost` BIGINT SUM NULL DEFAULT "0"\n) ENGINE=OLAP\nAGGREGATE KEY(`user_id`, `good_id`)\nCOMMENT 'OLAP'\nPARTITION BY RANGE(`good_id`)\n(PARTITION p1 VALUES [("-170141183460469231731687303715884105728"), ("100")),\nPARTITION p2 VALUES [("100"), ("200")),\nPARTITION p3 VALUES [("200"), ("300")),\nPARTITION p4 VALUES [("300"), ("400")),\nPARTITION p5 VALUES [("400"), ("500")),\nPARTITION p6 VALUES [("500"), ("600")),\nPARTITION p7 VALUES [("600"), (MAXVALUE)))\nDISTRIBUTED BY HASH(`user_id`) BUCKETS 2\nPROPERTIES (\n"replication_allocation" = "tag.location.default: 1",\n"min_load_replica_num" = "-1",\n"is_being_synced" = "false",\n"storage_medium" = "hdd",\n"storage_format" = "V2",\n"inverted_index_storage_format" = "V2",\n"light_schema_change" = "true",\n"disable_auto_compaction" = "false",\n"binlog.enable" = "false",\n"binlog.ttl_seconds" = "86400",\n"binlog.max_bytes" = "9223372036854775807",\n"binlog.max_history_nums" = "9223372036854775807",\n"enable_single_replica_compaction" = "false",\n"group_commit_interval_ms" = "10000",\n"group_commit_data_bytes" = "134217728"\n); -- !select -- 1 1 30 @@ -36,11 +36,11 @@ show_create_table_and_views_view CREATE VIEW `show_create_table_and_views_view` 300 1 -- !show -- -show_create_table_and_views_table CREATE TABLE `show_create_table_and_views_table` (\n `user_id` LARGEINT NOT NULL,\n `good_id` LARGEINT NOT NULL,\n `cost` BIGINT SUM NULL DEFAULT "0"\n) ENGINE=OLAP\nAGGREGATE KEY(`user_id`, `good_id`)\nCOMMENT 'OLAP'\nPARTITION BY RANGE(`good_id`)\n(PARTITION p1 VALUES [("-170141183460469231731687303715884105728"), ("100")),\nPARTITION p2 VALUES [("100"), ("200")),\nPARTITION p3 VALUES [("200"), ("300")),\nPARTITION p4 VALUES [("300"), ("400")),\nPARTITION p5 VALUES [("400"), ("500")),\nPARTITION p6 VALUES [("500"), ("600")),\nPARTITION p7 VALUES [("600"), (MAXVALUE)))\nDISTRIBUTED BY HASH(`user_id`) BUCKETS 2\nPROPERTIES (\n"replication_allocation" = "tag.location.default: 1",\n"min_load_replica_num" = "-1",\n"is_being_synced" = "false",\n"storage_medium" = "hdd",\n"storage_format" = "V2",\n"inverted_index_storage_format" = "V1",\n"light_schema_change" = "true",\n"disable_auto_compaction" = "false",\n"binlog.enable" = "false",\n"binlog.ttl_seconds" = "86400",\n"binlog.max_bytes" = "9223372036854775807",\n"binlog.max_history_nums" = "9223372036854775807",\n"enable_single_replica_compaction" = "false",\n"group_commit_interval_ms" = "10000",\n"group_commit_data_bytes" = "134217728"\n); +show_create_table_and_views_table CREATE TABLE `show_create_table_and_views_table` (\n `user_id` LARGEINT NOT NULL,\n `good_id` LARGEINT NOT NULL,\n `cost` BIGINT SUM NULL DEFAULT "0"\n) ENGINE=OLAP\nAGGREGATE KEY(`user_id`, `good_id`)\nCOMMENT 'OLAP'\nPARTITION BY RANGE(`good_id`)\n(PARTITION p1 VALUES [("-170141183460469231731687303715884105728"), ("100")),\nPARTITION p2 VALUES [("100"), ("200")),\nPARTITION p3 VALUES [("200"), ("300")),\nPARTITION p4 VALUES [("300"), ("400")),\nPARTITION p5 VALUES [("400"), ("500")),\nPARTITION p6 VALUES [("500"), ("600")),\nPARTITION p7 VALUES [("600"), (MAXVALUE)))\nDISTRIBUTED BY HASH(`user_id`) BUCKETS 2\nPROPERTIES (\n"replication_allocation" = "tag.location.default: 1",\n"min_load_replica_num" = "-1",\n"is_being_synced" = "false",\n"storage_medium" = "hdd",\n"storage_format" = "V2",\n"inverted_index_storage_format" = "V2",\n"light_schema_change" = "true",\n"disable_auto_compaction" = "false",\n"binlog.enable" = "false",\n"binlog.ttl_seconds" = "86400",\n"binlog.max_bytes" = "9223372036854775807",\n"binlog.max_history_nums" = "9223372036854775807",\n"enable_single_replica_compaction" = "false",\n"group_commit_interval_ms" = "10000",\n"group_commit_data_bytes" = "134217728"\n); -- !show -- -show_create_table_and_views_like CREATE TABLE `show_create_table_and_views_like` (\n `user_id` LARGEINT NOT NULL,\n `good_id` LARGEINT NOT NULL,\n `cost` BIGINT SUM NULL DEFAULT "0"\n) ENGINE=OLAP\nAGGREGATE KEY(`user_id`, `good_id`)\nCOMMENT 'OLAP'\nPARTITION BY RANGE(`good_id`)\n(PARTITION p1 VALUES [("-170141183460469231731687303715884105728"), ("100")),\nPARTITION p2 VALUES [("100"), ("200")),\nPARTITION p3 VALUES [("200"), ("300")),\nPARTITION p4 VALUES [("300"), ("400")),\nPARTITION p5 VALUES [("400"), ("500")),\nPARTITION p6 VALUES [("500"), ("600")),\nPARTITION p7 VALUES [("600"), (MAXVALUE)))\nDISTRIBUTED BY HASH(`user_id`) BUCKETS 2\nPROPERTIES (\n"replication_allocation" = "tag.location.default: 1",\n"min_load_replica_num" = "-1",\n"is_being_synced" = "false",\n"storage_medium" = "hdd",\n"storage_format" = "V2",\n"inverted_index_storage_format" = "V1",\n"light_schema_change" = "true",\n"disable_auto_compaction" = "false",\n"binlog.enable" = "false",\n"binlog.ttl_seconds" = "86400",\n"binlog.max_bytes" = "9223372036854775807",\n"binlog.max_history_nums" = "9223372036854775807",\n"enable_single_replica_compaction" = "false",\n"group_commit_interval_ms" = "10000",\n"group_commit_data_bytes" = "134217728"\n); +show_create_table_and_views_like CREATE TABLE `show_create_table_and_views_like` (\n `user_id` LARGEINT NOT NULL,\n `good_id` LARGEINT NOT NULL,\n `cost` BIGINT SUM NULL DEFAULT "0"\n) ENGINE=OLAP\nAGGREGATE KEY(`user_id`, `good_id`)\nCOMMENT 'OLAP'\nPARTITION BY RANGE(`good_id`)\n(PARTITION p1 VALUES [("-170141183460469231731687303715884105728"), ("100")),\nPARTITION p2 VALUES [("100"), ("200")),\nPARTITION p3 VALUES [("200"), ("300")),\nPARTITION p4 VALUES [("300"), ("400")),\nPARTITION p5 VALUES [("400"), ("500")),\nPARTITION p6 VALUES [("500"), ("600")),\nPARTITION p7 VALUES [("600"), (MAXVALUE)))\nDISTRIBUTED BY HASH(`user_id`) BUCKETS 2\nPROPERTIES (\n"replication_allocation" = "tag.location.default: 1",\n"min_load_replica_num" = "-1",\n"is_being_synced" = "false",\n"storage_medium" = "hdd",\n"storage_format" = "V2",\n"inverted_index_storage_format" = "V2",\n"light_schema_change" = "true",\n"disable_auto_compaction" = "false",\n"binlog.enable" = "false",\n"binlog.ttl_seconds" = "86400",\n"binlog.max_bytes" = "9223372036854775807",\n"binlog.max_history_nums" = "9223372036854775807",\n"enable_single_replica_compaction" = "false",\n"group_commit_interval_ms" = "10000",\n"group_commit_data_bytes" = "134217728"\n); -- !show -- -show_create_table_and_views_like_with_rollup CREATE TABLE `show_create_table_and_views_like_with_rollup` (\n `user_id` LARGEINT NOT NULL,\n `good_id` LARGEINT NOT NULL,\n `cost` BIGINT SUM NULL DEFAULT "0"\n) ENGINE=OLAP\nAGGREGATE KEY(`user_id`, `good_id`)\nCOMMENT 'OLAP'\nPARTITION BY RANGE(`good_id`)\n(PARTITION p1 VALUES [("-170141183460469231731687303715884105728"), ("100")),\nPARTITION p2 VALUES [("100"), ("200")),\nPARTITION p3 VALUES [("200"), ("300")),\nPARTITION p4 VALUES [("300"), ("400")),\nPARTITION p5 VALUES [("400"), ("500")),\nPARTITION p6 VALUES [("500"), ("600")),\nPARTITION p7 VALUES [("600"), (MAXVALUE)))\nDISTRIBUTED BY HASH(`user_id`) BUCKETS 2\nPROPERTIES (\n"replication_allocation" = "tag.location.default: 1",\n"min_load_replica_num" = "-1",\n"is_being_synced" = "false",\n"storage_medium" = "hdd",\n"storage_format" = "V2",\n"inverted_index_storage_format" = "V1",\n"light_schema_change" = "true",\n"disable_auto_compaction" = "false",\n"binlog.enable" = "false",\n"binlog.ttl_seconds" = "86400",\n"binlog.max_bytes" = "9223372036854775807",\n"binlog.max_history_nums" = "9223372036854775807",\n"enable_single_replica_compaction" = "false",\n"group_commit_interval_ms" = "10000",\n"group_commit_data_bytes" = "134217728"\n); +show_create_table_and_views_like_with_rollup CREATE TABLE `show_create_table_and_views_like_with_rollup` (\n `user_id` LARGEINT NOT NULL,\n `good_id` LARGEINT NOT NULL,\n `cost` BIGINT SUM NULL DEFAULT "0"\n) ENGINE=OLAP\nAGGREGATE KEY(`user_id`, `good_id`)\nCOMMENT 'OLAP'\nPARTITION BY RANGE(`good_id`)\n(PARTITION p1 VALUES [("-170141183460469231731687303715884105728"), ("100")),\nPARTITION p2 VALUES [("100"), ("200")),\nPARTITION p3 VALUES [("200"), ("300")),\nPARTITION p4 VALUES [("300"), ("400")),\nPARTITION p5 VALUES [("400"), ("500")),\nPARTITION p6 VALUES [("500"), ("600")),\nPARTITION p7 VALUES [("600"), (MAXVALUE)))\nDISTRIBUTED BY HASH(`user_id`) BUCKETS 2\nPROPERTIES (\n"replication_allocation" = "tag.location.default: 1",\n"min_load_replica_num" = "-1",\n"is_being_synced" = "false",\n"storage_medium" = "hdd",\n"storage_format" = "V2",\n"inverted_index_storage_format" = "V2",\n"light_schema_change" = "true",\n"disable_auto_compaction" = "false",\n"binlog.enable" = "false",\n"binlog.ttl_seconds" = "86400",\n"binlog.max_bytes" = "9223372036854775807",\n"binlog.max_history_nums" = "9223372036854775807",\n"enable_single_replica_compaction" = "false",\n"group_commit_interval_ms" = "10000",\n"group_commit_data_bytes" = "134217728"\n); diff --git a/regression-test/data/show_p0/test_show_create_table_and_views_nereids.out b/regression-test/data/show_p0/test_show_create_table_and_views_nereids.out index 3a5728d3d9e50a..bbd716a7249364 100644 --- a/regression-test/data/show_p0/test_show_create_table_and_views_nereids.out +++ b/regression-test/data/show_p0/test_show_create_table_and_views_nereids.out @@ -1,6 +1,6 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !show -- -show_create_table_and_views_nereids_table CREATE TABLE `show_create_table_and_views_nereids_table` (\n `user_id` LARGEINT NOT NULL,\n `good_id` LARGEINT NOT NULL,\n `cost` BIGINT SUM NULL DEFAULT "0"\n) ENGINE=OLAP\nAGGREGATE KEY(`user_id`, `good_id`)\nCOMMENT 'OLAP'\nPARTITION BY RANGE(`good_id`)\n(PARTITION p1 VALUES [("-170141183460469231731687303715884105728"), ("100")),\nPARTITION p2 VALUES [("100"), ("200")),\nPARTITION p3 VALUES [("200"), ("300")),\nPARTITION p4 VALUES [("300"), ("400")),\nPARTITION p5 VALUES [("400"), ("500")),\nPARTITION p6 VALUES [("500"), ("600")),\nPARTITION p7 VALUES [("600"), (MAXVALUE)))\nDISTRIBUTED BY HASH(`user_id`) BUCKETS 2\nPROPERTIES (\n"replication_allocation" = "tag.location.default: 1",\n"min_load_replica_num" = "-1",\n"is_being_synced" = "false",\n"storage_medium" = "hdd",\n"storage_format" = "V2",\n"inverted_index_storage_format" = "V1",\n"light_schema_change" = "true",\n"disable_auto_compaction" = "false",\n"binlog.enable" = "false",\n"binlog.ttl_seconds" = "86400",\n"binlog.max_bytes" = "9223372036854775807",\n"binlog.max_history_nums" = "9223372036854775807",\n"enable_single_replica_compaction" = "false",\n"group_commit_interval_ms" = "10000",\n"group_commit_data_bytes" = "134217728"\n); +show_create_table_and_views_nereids_table CREATE TABLE `show_create_table_and_views_nereids_table` (\n `user_id` LARGEINT NOT NULL,\n `good_id` LARGEINT NOT NULL,\n `cost` BIGINT SUM NULL DEFAULT "0"\n) ENGINE=OLAP\nAGGREGATE KEY(`user_id`, `good_id`)\nCOMMENT 'OLAP'\nPARTITION BY RANGE(`good_id`)\n(PARTITION p1 VALUES [("-170141183460469231731687303715884105728"), ("100")),\nPARTITION p2 VALUES [("100"), ("200")),\nPARTITION p3 VALUES [("200"), ("300")),\nPARTITION p4 VALUES [("300"), ("400")),\nPARTITION p5 VALUES [("400"), ("500")),\nPARTITION p6 VALUES [("500"), ("600")),\nPARTITION p7 VALUES [("600"), (MAXVALUE)))\nDISTRIBUTED BY HASH(`user_id`) BUCKETS 2\nPROPERTIES (\n"replication_allocation" = "tag.location.default: 1",\n"min_load_replica_num" = "-1",\n"is_being_synced" = "false",\n"storage_medium" = "hdd",\n"storage_format" = "V2",\n"inverted_index_storage_format" = "V2",\n"light_schema_change" = "true",\n"disable_auto_compaction" = "false",\n"binlog.enable" = "false",\n"binlog.ttl_seconds" = "86400",\n"binlog.max_bytes" = "9223372036854775807",\n"binlog.max_history_nums" = "9223372036854775807",\n"enable_single_replica_compaction" = "false",\n"group_commit_interval_ms" = "10000",\n"group_commit_data_bytes" = "134217728"\n); -- !select -- 1 1 30 @@ -36,11 +36,11 @@ show_create_table_and_views_nereids_view CREATE VIEW `show_create_table_and_view 300 1 -- !show -- -show_create_table_and_views_nereids_table CREATE TABLE `show_create_table_and_views_nereids_table` (\n `user_id` LARGEINT NOT NULL,\n `good_id` LARGEINT NOT NULL,\n `cost` BIGINT SUM NULL DEFAULT "0"\n) ENGINE=OLAP\nAGGREGATE KEY(`user_id`, `good_id`)\nCOMMENT 'OLAP'\nPARTITION BY RANGE(`good_id`)\n(PARTITION p1 VALUES [("-170141183460469231731687303715884105728"), ("100")),\nPARTITION p2 VALUES [("100"), ("200")),\nPARTITION p3 VALUES [("200"), ("300")),\nPARTITION p4 VALUES [("300"), ("400")),\nPARTITION p5 VALUES [("400"), ("500")),\nPARTITION p6 VALUES [("500"), ("600")),\nPARTITION p7 VALUES [("600"), (MAXVALUE)))\nDISTRIBUTED BY HASH(`user_id`) BUCKETS 2\nPROPERTIES (\n"replication_allocation" = "tag.location.default: 1",\n"min_load_replica_num" = "-1",\n"is_being_synced" = "false",\n"storage_medium" = "hdd",\n"storage_format" = "V2",\n"inverted_index_storage_format" = "V1",\n"light_schema_change" = "true",\n"disable_auto_compaction" = "false",\n"binlog.enable" = "false",\n"binlog.ttl_seconds" = "86400",\n"binlog.max_bytes" = "9223372036854775807",\n"binlog.max_history_nums" = "9223372036854775807",\n"enable_single_replica_compaction" = "false",\n"group_commit_interval_ms" = "10000",\n"group_commit_data_bytes" = "134217728"\n); +show_create_table_and_views_nereids_table CREATE TABLE `show_create_table_and_views_nereids_table` (\n `user_id` LARGEINT NOT NULL,\n `good_id` LARGEINT NOT NULL,\n `cost` BIGINT SUM NULL DEFAULT "0"\n) ENGINE=OLAP\nAGGREGATE KEY(`user_id`, `good_id`)\nCOMMENT 'OLAP'\nPARTITION BY RANGE(`good_id`)\n(PARTITION p1 VALUES [("-170141183460469231731687303715884105728"), ("100")),\nPARTITION p2 VALUES [("100"), ("200")),\nPARTITION p3 VALUES [("200"), ("300")),\nPARTITION p4 VALUES [("300"), ("400")),\nPARTITION p5 VALUES [("400"), ("500")),\nPARTITION p6 VALUES [("500"), ("600")),\nPARTITION p7 VALUES [("600"), (MAXVALUE)))\nDISTRIBUTED BY HASH(`user_id`) BUCKETS 2\nPROPERTIES (\n"replication_allocation" = "tag.location.default: 1",\n"min_load_replica_num" = "-1",\n"is_being_synced" = "false",\n"storage_medium" = "hdd",\n"storage_format" = "V2",\n"inverted_index_storage_format" = "V2",\n"light_schema_change" = "true",\n"disable_auto_compaction" = "false",\n"binlog.enable" = "false",\n"binlog.ttl_seconds" = "86400",\n"binlog.max_bytes" = "9223372036854775807",\n"binlog.max_history_nums" = "9223372036854775807",\n"enable_single_replica_compaction" = "false",\n"group_commit_interval_ms" = "10000",\n"group_commit_data_bytes" = "134217728"\n); -- !show -- -show_create_table_and_views_nereids_like CREATE TABLE `show_create_table_and_views_nereids_like` (\n `user_id` LARGEINT NOT NULL,\n `good_id` LARGEINT NOT NULL,\n `cost` BIGINT SUM NULL DEFAULT "0"\n) ENGINE=OLAP\nAGGREGATE KEY(`user_id`, `good_id`)\nCOMMENT 'OLAP'\nPARTITION BY RANGE(`good_id`)\n(PARTITION p1 VALUES [("-170141183460469231731687303715884105728"), ("100")),\nPARTITION p2 VALUES [("100"), ("200")),\nPARTITION p3 VALUES [("200"), ("300")),\nPARTITION p4 VALUES [("300"), ("400")),\nPARTITION p5 VALUES [("400"), ("500")),\nPARTITION p6 VALUES [("500"), ("600")),\nPARTITION p7 VALUES [("600"), (MAXVALUE)))\nDISTRIBUTED BY HASH(`user_id`) BUCKETS 2\nPROPERTIES (\n"replication_allocation" = "tag.location.default: 1",\n"min_load_replica_num" = "-1",\n"is_being_synced" = "false",\n"storage_medium" = "hdd",\n"storage_format" = "V2",\n"inverted_index_storage_format" = "V1",\n"light_schema_change" = "true",\n"disable_auto_compaction" = "false",\n"binlog.enable" = "false",\n"binlog.ttl_seconds" = "86400",\n"binlog.max_bytes" = "9223372036854775807",\n"binlog.max_history_nums" = "9223372036854775807",\n"enable_single_replica_compaction" = "false",\n"group_commit_interval_ms" = "10000",\n"group_commit_data_bytes" = "134217728"\n); +show_create_table_and_views_nereids_like CREATE TABLE `show_create_table_and_views_nereids_like` (\n `user_id` LARGEINT NOT NULL,\n `good_id` LARGEINT NOT NULL,\n `cost` BIGINT SUM NULL DEFAULT "0"\n) ENGINE=OLAP\nAGGREGATE KEY(`user_id`, `good_id`)\nCOMMENT 'OLAP'\nPARTITION BY RANGE(`good_id`)\n(PARTITION p1 VALUES [("-170141183460469231731687303715884105728"), ("100")),\nPARTITION p2 VALUES [("100"), ("200")),\nPARTITION p3 VALUES [("200"), ("300")),\nPARTITION p4 VALUES [("300"), ("400")),\nPARTITION p5 VALUES [("400"), ("500")),\nPARTITION p6 VALUES [("500"), ("600")),\nPARTITION p7 VALUES [("600"), (MAXVALUE)))\nDISTRIBUTED BY HASH(`user_id`) BUCKETS 2\nPROPERTIES (\n"replication_allocation" = "tag.location.default: 1",\n"min_load_replica_num" = "-1",\n"is_being_synced" = "false",\n"storage_medium" = "hdd",\n"storage_format" = "V2",\n"inverted_index_storage_format" = "V2",\n"light_schema_change" = "true",\n"disable_auto_compaction" = "false",\n"binlog.enable" = "false",\n"binlog.ttl_seconds" = "86400",\n"binlog.max_bytes" = "9223372036854775807",\n"binlog.max_history_nums" = "9223372036854775807",\n"enable_single_replica_compaction" = "false",\n"group_commit_interval_ms" = "10000",\n"group_commit_data_bytes" = "134217728"\n); -- !show -- -show_create_table_and_views_nereids_like_with_rollup CREATE TABLE `show_create_table_and_views_nereids_like_with_rollup` (\n `user_id` LARGEINT NOT NULL,\n `good_id` LARGEINT NOT NULL,\n `cost` BIGINT SUM NULL DEFAULT "0"\n) ENGINE=OLAP\nAGGREGATE KEY(`user_id`, `good_id`)\nCOMMENT 'OLAP'\nPARTITION BY RANGE(`good_id`)\n(PARTITION p1 VALUES [("-170141183460469231731687303715884105728"), ("100")),\nPARTITION p2 VALUES [("100"), ("200")),\nPARTITION p3 VALUES [("200"), ("300")),\nPARTITION p4 VALUES [("300"), ("400")),\nPARTITION p5 VALUES [("400"), ("500")),\nPARTITION p6 VALUES [("500"), ("600")),\nPARTITION p7 VALUES [("600"), (MAXVALUE)))\nDISTRIBUTED BY HASH(`user_id`) BUCKETS 2\nPROPERTIES (\n"replication_allocation" = "tag.location.default: 1",\n"min_load_replica_num" = "-1",\n"is_being_synced" = "false",\n"storage_medium" = "hdd",\n"storage_format" = "V2",\n"inverted_index_storage_format" = "V1",\n"light_schema_change" = "true",\n"disable_auto_compaction" = "false",\n"binlog.enable" = "false",\n"binlog.ttl_seconds" = "86400",\n"binlog.max_bytes" = "9223372036854775807",\n"binlog.max_history_nums" = "9223372036854775807",\n"enable_single_replica_compaction" = "false",\n"group_commit_interval_ms" = "10000",\n"group_commit_data_bytes" = "134217728"\n); +show_create_table_and_views_nereids_like_with_rollup CREATE TABLE `show_create_table_and_views_nereids_like_with_rollup` (\n `user_id` LARGEINT NOT NULL,\n `good_id` LARGEINT NOT NULL,\n `cost` BIGINT SUM NULL DEFAULT "0"\n) ENGINE=OLAP\nAGGREGATE KEY(`user_id`, `good_id`)\nCOMMENT 'OLAP'\nPARTITION BY RANGE(`good_id`)\n(PARTITION p1 VALUES [("-170141183460469231731687303715884105728"), ("100")),\nPARTITION p2 VALUES [("100"), ("200")),\nPARTITION p3 VALUES [("200"), ("300")),\nPARTITION p4 VALUES [("300"), ("400")),\nPARTITION p5 VALUES [("400"), ("500")),\nPARTITION p6 VALUES [("500"), ("600")),\nPARTITION p7 VALUES [("600"), (MAXVALUE)))\nDISTRIBUTED BY HASH(`user_id`) BUCKETS 2\nPROPERTIES (\n"replication_allocation" = "tag.location.default: 1",\n"min_load_replica_num" = "-1",\n"is_being_synced" = "false",\n"storage_medium" = "hdd",\n"storage_format" = "V2",\n"inverted_index_storage_format" = "V2",\n"light_schema_change" = "true",\n"disable_auto_compaction" = "false",\n"binlog.enable" = "false",\n"binlog.ttl_seconds" = "86400",\n"binlog.max_bytes" = "9223372036854775807",\n"binlog.max_history_nums" = "9223372036854775807",\n"enable_single_replica_compaction" = "false",\n"group_commit_interval_ms" = "10000",\n"group_commit_data_bytes" = "134217728"\n); diff --git a/regression-test/suites/fault_injection_p0/test_calc_crc_fault_injection.groovy b/regression-test/suites/fault_injection_p0/test_calc_crc_fault_injection.groovy index e5cb5526d9f93a..156610ebb804d7 100644 --- a/regression-test/suites/fault_injection_p0/test_calc_crc_fault_injection.groovy +++ b/regression-test/suites/fault_injection_p0/test_calc_crc_fault_injection.groovy @@ -73,7 +73,7 @@ suite("test_calc_crc") { assertEquals("0", parseJson(out_0.trim()).start_version) assertEquals("7", parseJson(out_0.trim()).end_version) assertEquals("7", parseJson(out_0.trim()).rowset_count) - assertEquals("18", parseJson(out_0.trim()).file_count) + assertEquals("12", parseJson(out_0.trim()).file_count) try { GetDebugPoint().enableDebugPointForAllBEs("fault_inject::BetaRowset::calc_local_file_crc") @@ -90,7 +90,7 @@ suite("test_calc_crc") { assertEquals("0", parseJson(out_2.trim()).start_version) assertEquals("7", parseJson(out_2.trim()).end_version) assertEquals("7", parseJson(out_2.trim()).rowset_count) - assertEquals("18", parseJson(out_2.trim()).file_count) + assertEquals("12", parseJson(out_2.trim()).file_count) assertTrue(parseJson(out_0.trim()).crc_value == parseJson(out_2.trim()).crc_value) def (code_3, out_3, err_3) = calc_file_crc_on_tablet_with_end(ip, port, tablet_id, 7) @@ -99,7 +99,7 @@ suite("test_calc_crc") { assertEquals("0", parseJson(out_3.trim()).start_version) assertEquals("7", parseJson(out_3.trim()).end_version) assertEquals("7", parseJson(out_3.trim()).rowset_count) - assertEquals("18", parseJson(out_3.trim()).file_count) + assertEquals("12", parseJson(out_3.trim()).file_count) assertTrue(parseJson(out_2.trim()).crc_value == parseJson(out_3.trim()).crc_value) def (code_4, out_4, err_4) = calc_file_crc_on_tablet_with_start_end(ip, port, tablet_id, 3, 6) From de166e544e48cb42f1f33dcb246508e61b7e964b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=A7=9C=E5=87=AF?= Date: Fri, 24 May 2024 19:47:34 +0800 Subject: [PATCH 4/7] fix case --- .../fault_injection_p0/test_calc_crc_fault_injection.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/regression-test/suites/fault_injection_p0/test_calc_crc_fault_injection.groovy b/regression-test/suites/fault_injection_p0/test_calc_crc_fault_injection.groovy index 156610ebb804d7..9fb9d88e168872 100644 --- a/regression-test/suites/fault_injection_p0/test_calc_crc_fault_injection.groovy +++ b/regression-test/suites/fault_injection_p0/test_calc_crc_fault_injection.groovy @@ -109,7 +109,7 @@ suite("test_calc_crc") { assertEquals("3", parseJson(out_4.trim()).start_version) assertEquals("6", parseJson(out_4.trim()).end_version) assertEquals("4", parseJson(out_4.trim()).rowset_count) - assertEquals("12", parseJson(out_4.trim()).file_count) + assertEquals("8", parseJson(out_4.trim()).file_count) def (code_5, out_5, err_5) = calc_file_crc_on_tablet_with_start_end(ip, port, tablet_id, 5, 9) logger.info("Run calc_file_crc_on_tablet: code=" + code_5 + ", out=" + out_5 + ", err=" + err_5) From 1162906a6e3e97e6f80ffa7745809e361d4c7267 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=A7=9C=E5=87=AF?= Date: Wed, 5 Jun 2024 14:49:31 +0800 Subject: [PATCH 5/7] update index_id usage --- be/src/olap/rowset/segment_v2/inverted_index_desc.cpp | 8 ++++---- be/src/olap/rowset/segment_v2/inverted_index_desc.h | 5 +++-- .../olap/rowset/segment_v2/inverted_index_file_reader.cpp | 2 +- .../olap/rowset/segment_v2/inverted_index_file_writer.cpp | 4 ++-- 4 files changed, 10 insertions(+), 9 deletions(-) diff --git a/be/src/olap/rowset/segment_v2/inverted_index_desc.cpp b/be/src/olap/rowset/segment_v2/inverted_index_desc.cpp index ecce5d30920ec8..bc25534eaaab3e 100644 --- a/be/src/olap/rowset/segment_v2/inverted_index_desc.cpp +++ b/be/src/olap/rowset/segment_v2/inverted_index_desc.cpp @@ -27,21 +27,21 @@ namespace doris::segment_v2 { // {tmp_dir}/{rowset_id}_{seg_id}_{index_id}@{suffix} std::string InvertedIndexDescriptor::get_temporary_index_path(std::string_view tmp_dir_path, std::string_view rowset_id, - int64_t seg_id, uint64_t uuid, + int64_t seg_id, int64_t index_id, std::string_view index_path_suffix) { std::string suffix = index_path_suffix.empty() ? "" : std::string {"@"} + index_path_suffix.data(); - return fmt::format("{}/{}_{}_{}{}", tmp_dir_path, rowset_id, seg_id, uuid, suffix); + return fmt::format("{}/{}_{}_{}{}", tmp_dir_path, rowset_id, seg_id, index_id, suffix); } // InvertedIndexStorageFormat V1 // {prefix}_{index_id}@{suffix}.idx std::string InvertedIndexDescriptor::get_index_path_v1(std::string_view index_path_prefix, - uint64_t uuid, + int64_t index_id, std::string_view index_path_suffix) { std::string suffix = index_path_suffix.empty() ? "" : std::string {"@"} + index_path_suffix.data(); - return fmt::format("{}_{}{}{}", index_path_prefix, uuid, suffix, index_suffix); + return fmt::format("{}_{}{}{}", index_path_prefix, index_id, suffix, index_suffix); } // InvertedIndexStorageFormat V2 diff --git a/be/src/olap/rowset/segment_v2/inverted_index_desc.h b/be/src/olap/rowset/segment_v2/inverted_index_desc.h index bd64bdb5eaa0e2..01f0636f943707 100644 --- a/be/src/olap/rowset/segment_v2/inverted_index_desc.h +++ b/be/src/olap/rowset/segment_v2/inverted_index_desc.h @@ -32,9 +32,10 @@ class InvertedIndexDescriptor { static constexpr std::string_view index_suffix = ".idx"; static std::string get_temporary_index_path(std::string_view tmp_dir_path, std::string_view rowset_id, int64_t seg_id, - uint64_t uuid, std::string_view index_path_suffix); + int64_t index_id, + std::string_view index_path_suffix); // InvertedIndexStorageFormat V1 - static std::string get_index_path_v1(std::string_view index_path_prefix, uint64_t uuid, + static std::string get_index_path_v1(std::string_view index_path_prefix, int64_t index_id, std::string_view index_path_suffix); // InvertedIndexStorageFormat V2 static std::string get_index_path_v2(std::string_view index_path_prefix); diff --git a/be/src/olap/rowset/segment_v2/inverted_index_file_reader.cpp b/be/src/olap/rowset/segment_v2/inverted_index_file_reader.cpp index ed38909002a541..5ef014ea0ab694 100644 --- a/be/src/olap/rowset/segment_v2/inverted_index_file_reader.cpp +++ b/be/src/olap/rowset/segment_v2/inverted_index_file_reader.cpp @@ -73,7 +73,7 @@ Status InvertedIndexFileReader::_init_from_v2(int32_t read_buffer_size) { ReaderFileEntry* entry = nullptr; for (int32_t i = 0; i < numIndices; ++i) { - int64_t indexId = _stream->readInt(); // Read index ID + int64_t indexId = _stream->readLong(); // Read index ID int32_t suffix_length = _stream->readInt(); // Read suffix length std::vector suffix_data(suffix_length); _stream->readBytes(suffix_data.data(), suffix_length); diff --git a/be/src/olap/rowset/segment_v2/inverted_index_file_writer.cpp b/be/src/olap/rowset/segment_v2/inverted_index_file_writer.cpp index 4b5f9dd54cc4f2..4272c2bde23c36 100644 --- a/be/src/olap/rowset/segment_v2/inverted_index_file_writer.cpp +++ b/be/src/olap/rowset/segment_v2/inverted_index_file_writer.cpp @@ -102,7 +102,7 @@ size_t InvertedIndexFileWriter::headerLength() { sizeof(int) * 2; // Account for the size of the version number and number of indices for (const auto& entry : _indices_dirs) { auto suffix = entry.first.second; - header_size += sizeof(int); // index id + header_size += sizeof(int64_t); // index id header_size += 4; // index suffix name size header_size += suffix.length(); // index suffix name header_size += sizeof(int); // index file count @@ -203,7 +203,7 @@ size_t InvertedIndexFileWriter::write() { int32_t file_count = sorted_files.size(); // Write the index ID and the number of files - compound_file_output->writeInt(index_id); + compound_file_output->writeLong(index_id); const auto* index_suffix_str = reinterpret_cast(index_suffix.c_str()); compound_file_output->writeInt(index_suffix.length()); compound_file_output->writeBytes(index_suffix_str, index_suffix.length()); From b169256d24727811c6a3c5b6995cf4f88f2c4e94 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=A7=9C=E5=87=AF?= Date: Wed, 5 Jun 2024 17:04:31 +0800 Subject: [PATCH 6/7] enhance code --- .../segment_v2/inverted_index_file_writer.cpp | 35 ++++++++++--------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/be/src/olap/rowset/segment_v2/inverted_index_file_writer.cpp b/be/src/olap/rowset/segment_v2/inverted_index_file_writer.cpp index 4272c2bde23c36..b7cb9591b883ea 100644 --- a/be/src/olap/rowset/segment_v2/inverted_index_file_writer.cpp +++ b/be/src/olap/rowset/segment_v2/inverted_index_file_writer.cpp @@ -99,22 +99,23 @@ Status InvertedIndexFileWriter::delete_index(const TabletIndex* index_meta) { size_t InvertedIndexFileWriter::headerLength() { size_t header_size = 0; header_size += - sizeof(int) * 2; // Account for the size of the version number and number of indices + sizeof(int32_t) * 2; // Account for the size of the version number and number of indices + for (const auto& entry : _indices_dirs) { - auto suffix = entry.first.second; + const auto& suffix = entry.first.second; header_size += sizeof(int64_t); // index id - header_size += 4; // index suffix name size + header_size += sizeof(int32_t); // index suffix name size header_size += suffix.length(); // index suffix name - header_size += sizeof(int); // index file count + header_size += sizeof(int32_t); // index file count const auto& dir = entry.second; std::vector files; dir->list(&files); - for (auto file : files) { - header_size += 4; // file name size - header_size += file.length(); // file name - header_size += 8; // file offset - header_size += 8; // file size + for (const auto& file : files) { + header_size += sizeof(int32_t); // file name size + header_size += file.length(); // file name + header_size += sizeof(int64_t); // file offset + header_size += sizeof(int64_t); // file size } } return header_size; @@ -192,10 +193,10 @@ size_t InvertedIndexFileWriter::write() { } // sort file list by file length std::vector> sorted_files; - for (auto file : files) { + for (const auto& file : files) { sorted_files.emplace_back(file, dir->fileLength(file.c_str())); } - // TODO: need to optimize + std::sort(sorted_files.begin(), sorted_files.end(), [](const std::pair& a, const std::pair& b) { return (a.second < b.second); }); @@ -204,17 +205,17 @@ size_t InvertedIndexFileWriter::write() { // Write the index ID and the number of files compound_file_output->writeLong(index_id); - const auto* index_suffix_str = reinterpret_cast(index_suffix.c_str()); - compound_file_output->writeInt(index_suffix.length()); - compound_file_output->writeBytes(index_suffix_str, index_suffix.length()); + compound_file_output->writeInt(static_cast(index_suffix.length())); + compound_file_output->writeBytes(reinterpret_cast(index_suffix.data()), + index_suffix.length()); compound_file_output->writeInt(file_count); // Calculate the offset for each file and write the file metadata for (const auto& file : sorted_files) { int64_t file_length = dir->fileLength(file.first.c_str()); - const auto* file_name = reinterpret_cast(file.first.c_str()); - compound_file_output->writeInt(file.first.length()); - compound_file_output->writeBytes(file_name, file.first.length()); + compound_file_output->writeInt(static_cast(file.first.length())); + compound_file_output->writeBytes(reinterpret_cast(file.first.data()), + file.first.length()); compound_file_output->writeLong(current_offset); compound_file_output->writeLong(file_length); From f8ac28bf88644f259afeb0ade6af2b08c8189471 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=A7=9C=E5=87=AF?= Date: Wed, 5 Jun 2024 17:07:33 +0800 Subject: [PATCH 7/7] fix case --- .../data/query_p0/system/test_table_options.out | 12 ++++++------ .../test_calc_crc_fault_injection.groovy | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/regression-test/data/query_p0/system/test_table_options.out b/regression-test/data/query_p0/system/test_table_options.out index 8700b8741ddc1f..8a14e8c546339e 100644 --- a/regression-test/data/query_p0/system/test_table_options.out +++ b/regression-test/data/query_p0/system/test_table_options.out @@ -1,9 +1,9 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !select -- -aggregate_table internal test_table_options_db AGG user_id,date,city,age,sex user_id HASH 1 1 {"min_load_replica_num":"-1","data_sort.col_num":"5","group_commit_interval_ms":"10000","data_sort.sort_type":"LEXICAL","is_being_synced":"false","binlog.enable":"false","binlog.ttl_seconds":"86400","inverted_index_storage_format":"V1","time_series_compaction_empty_rowsets_threshold":"5","default.replication_allocation":"tag.location.default: 1","time_series_compaction_level_threshold":"1","time_series_compaction_time_threshold_seconds":"3600","storage_format":"V2","store_row_column":"false","light_schema_change":"true","enable_unique_key_merge_on_write":"false","in_memory":"false","file_cache_ttl_seconds":"0","group_commit_data_bytes":"134217728","compaction_policy":"size_based","_auto_bucket":"false","binlog.max_history_nums":"9223372036854775807","time_series_compaction_file_count_threshold":"2000","skip_write_index_on_load":"false","disable_auto_compaction":"false","time_series_compaction_goal_size_mbytes":"1024","storage_medium":"HDD","enable_single_replica_compaction":"false","compression":"LZ4F","binlog.max_bytes":"9223372036854775807"} -duplicate_table internal test_table_options_db DUP timestamp,type,error_code type HASH 1 1 {"min_load_replica_num":"-1","data_sort.col_num":"3","group_commit_interval_ms":"10000","data_sort.sort_type":"LEXICAL","is_being_synced":"false","binlog.enable":"false","binlog.ttl_seconds":"86400","inverted_index_storage_format":"V1","time_series_compaction_empty_rowsets_threshold":"5","default.replication_allocation":"tag.location.default: 1","time_series_compaction_level_threshold":"1","time_series_compaction_time_threshold_seconds":"3600","storage_format":"V2","store_row_column":"false","light_schema_change":"true","enable_unique_key_merge_on_write":"false","in_memory":"false","file_cache_ttl_seconds":"0","group_commit_data_bytes":"134217728","compaction_policy":"size_based","_auto_bucket":"false","binlog.max_history_nums":"9223372036854775807","time_series_compaction_file_count_threshold":"2000","skip_write_index_on_load":"false","disable_auto_compaction":"false","time_series_compaction_goal_size_mbytes":"1024","storage_medium":"HDD","enable_single_replica_compaction":"false","compression":"LZ4F","binlog.max_bytes":"9223372036854775807"} -listtable internal test_table_options_db AGG user_id,date,timestamp,city,age,sex user_id HASH 16 3 {"min_load_replica_num":"-1","data_sort.col_num":"6","group_commit_interval_ms":"10000","data_sort.sort_type":"LEXICAL","is_being_synced":"false","binlog.enable":"false","binlog.ttl_seconds":"86400","inverted_index_storage_format":"V1","time_series_compaction_empty_rowsets_threshold":"5","default.replication_allocation":"tag.location.default: 1","time_series_compaction_level_threshold":"1","time_series_compaction_time_threshold_seconds":"3600","storage_format":"V2","store_row_column":"false","light_schema_change":"true","enable_unique_key_merge_on_write":"false","in_memory":"false","file_cache_ttl_seconds":"0","group_commit_data_bytes":"134217728","compaction_policy":"size_based","_auto_bucket":"false","binlog.max_history_nums":"9223372036854775807","time_series_compaction_file_count_threshold":"2000","skip_write_index_on_load":"false","disable_auto_compaction":"false","time_series_compaction_goal_size_mbytes":"1024","storage_medium":"HDD","enable_single_replica_compaction":"false","compression":"LZ4F","binlog.max_bytes":"9223372036854775807"} -randomtable internal test_table_options_db DUP user_id,date,timestamp RANDOM RANDOM 16 1 {"min_load_replica_num":"-1","data_sort.col_num":"3","group_commit_interval_ms":"10000","data_sort.sort_type":"LEXICAL","is_being_synced":"false","binlog.enable":"false","binlog.ttl_seconds":"86400","inverted_index_storage_format":"V1","time_series_compaction_empty_rowsets_threshold":"5","default.replication_allocation":"tag.location.default: 1","time_series_compaction_level_threshold":"1","time_series_compaction_time_threshold_seconds":"3600","storage_format":"V2","store_row_column":"false","light_schema_change":"true","enable_unique_key_merge_on_write":"false","in_memory":"false","file_cache_ttl_seconds":"0","group_commit_data_bytes":"134217728","compaction_policy":"size_based","_auto_bucket":"false","binlog.max_history_nums":"9223372036854775807","time_series_compaction_file_count_threshold":"2000","skip_write_index_on_load":"false","disable_auto_compaction":"false","time_series_compaction_goal_size_mbytes":"1024","storage_medium":"HDD","enable_single_replica_compaction":"false","compression":"LZ4F","binlog.max_bytes":"9223372036854775807"} -rangetable internal test_table_options_db AGG user_id,date,timestamp,city,age,sex user_id HASH 8 3 {"min_load_replica_num":"-1","data_sort.col_num":"6","group_commit_interval_ms":"10000","data_sort.sort_type":"LEXICAL","is_being_synced":"false","binlog.enable":"false","binlog.ttl_seconds":"86400","inverted_index_storage_format":"V1","time_series_compaction_empty_rowsets_threshold":"5","default.replication_allocation":"tag.location.default: 1","time_series_compaction_level_threshold":"1","time_series_compaction_time_threshold_seconds":"3600","storage_format":"V2","store_row_column":"false","light_schema_change":"true","enable_unique_key_merge_on_write":"false","in_memory":"false","file_cache_ttl_seconds":"0","group_commit_data_bytes":"134217728","compaction_policy":"size_based","_auto_bucket":"false","binlog.max_history_nums":"9223372036854775807","time_series_compaction_file_count_threshold":"2000","skip_write_index_on_load":"false","disable_auto_compaction":"false","time_series_compaction_goal_size_mbytes":"1024","storage_medium":"HDD","enable_single_replica_compaction":"false","compression":"LZ4F","binlog.max_bytes":"9223372036854775807"} -unique_table internal test_table_options_db UNI user_id,username user_id HASH 1 1 {"min_load_replica_num":"-1","data_sort.col_num":"2","group_commit_interval_ms":"10000","data_sort.sort_type":"LEXICAL","is_being_synced":"false","binlog.enable":"false","binlog.ttl_seconds":"86400","inverted_index_storage_format":"V1","time_series_compaction_empty_rowsets_threshold":"5","default.replication_allocation":"tag.location.default: 1","time_series_compaction_level_threshold":"1","time_series_compaction_time_threshold_seconds":"3600","storage_format":"V2","store_row_column":"false","light_schema_change":"true","enable_unique_key_merge_on_write":"true","in_memory":"false","file_cache_ttl_seconds":"0","group_commit_data_bytes":"134217728","compaction_policy":"size_based","_auto_bucket":"false","binlog.max_history_nums":"9223372036854775807","time_series_compaction_file_count_threshold":"2000","skip_write_index_on_load":"false","disable_auto_compaction":"false","time_series_compaction_goal_size_mbytes":"1024","storage_medium":"HDD","enable_single_replica_compaction":"false","compression":"LZ4F","binlog.max_bytes":"9223372036854775807"} +aggregate_table internal test_table_options_db AGG user_id,date,city,age,sex user_id HASH 1 1 {"min_load_replica_num":"-1","data_sort.col_num":"5","group_commit_interval_ms":"10000","data_sort.sort_type":"LEXICAL","is_being_synced":"false","binlog.enable":"false","binlog.ttl_seconds":"86400","inverted_index_storage_format":"V2","time_series_compaction_empty_rowsets_threshold":"5","default.replication_allocation":"tag.location.default: 1","time_series_compaction_level_threshold":"1","time_series_compaction_time_threshold_seconds":"3600","storage_format":"V2","store_row_column":"false","light_schema_change":"true","enable_unique_key_merge_on_write":"false","in_memory":"false","file_cache_ttl_seconds":"0","group_commit_data_bytes":"134217728","compaction_policy":"size_based","_auto_bucket":"false","binlog.max_history_nums":"9223372036854775807","time_series_compaction_file_count_threshold":"2000","skip_write_index_on_load":"false","disable_auto_compaction":"false","time_series_compaction_goal_size_mbytes":"1024","storage_medium":"HDD","enable_single_replica_compaction":"false","compression":"LZ4F","binlog.max_bytes":"9223372036854775807"} +duplicate_table internal test_table_options_db DUP timestamp,type,error_code type HASH 1 1 {"min_load_replica_num":"-1","data_sort.col_num":"3","group_commit_interval_ms":"10000","data_sort.sort_type":"LEXICAL","is_being_synced":"false","binlog.enable":"false","binlog.ttl_seconds":"86400","inverted_index_storage_format":"V2","time_series_compaction_empty_rowsets_threshold":"5","default.replication_allocation":"tag.location.default: 1","time_series_compaction_level_threshold":"1","time_series_compaction_time_threshold_seconds":"3600","storage_format":"V2","store_row_column":"false","light_schema_change":"true","enable_unique_key_merge_on_write":"false","in_memory":"false","file_cache_ttl_seconds":"0","group_commit_data_bytes":"134217728","compaction_policy":"size_based","_auto_bucket":"false","binlog.max_history_nums":"9223372036854775807","time_series_compaction_file_count_threshold":"2000","skip_write_index_on_load":"false","disable_auto_compaction":"false","time_series_compaction_goal_size_mbytes":"1024","storage_medium":"HDD","enable_single_replica_compaction":"false","compression":"LZ4F","binlog.max_bytes":"9223372036854775807"} +listtable internal test_table_options_db AGG user_id,date,timestamp,city,age,sex user_id HASH 16 3 {"min_load_replica_num":"-1","data_sort.col_num":"6","group_commit_interval_ms":"10000","data_sort.sort_type":"LEXICAL","is_being_synced":"false","binlog.enable":"false","binlog.ttl_seconds":"86400","inverted_index_storage_format":"V2","time_series_compaction_empty_rowsets_threshold":"5","default.replication_allocation":"tag.location.default: 1","time_series_compaction_level_threshold":"1","time_series_compaction_time_threshold_seconds":"3600","storage_format":"V2","store_row_column":"false","light_schema_change":"true","enable_unique_key_merge_on_write":"false","in_memory":"false","file_cache_ttl_seconds":"0","group_commit_data_bytes":"134217728","compaction_policy":"size_based","_auto_bucket":"false","binlog.max_history_nums":"9223372036854775807","time_series_compaction_file_count_threshold":"2000","skip_write_index_on_load":"false","disable_auto_compaction":"false","time_series_compaction_goal_size_mbytes":"1024","storage_medium":"HDD","enable_single_replica_compaction":"false","compression":"LZ4F","binlog.max_bytes":"9223372036854775807"} +randomtable internal test_table_options_db DUP user_id,date,timestamp RANDOM RANDOM 16 1 {"min_load_replica_num":"-1","data_sort.col_num":"3","group_commit_interval_ms":"10000","data_sort.sort_type":"LEXICAL","is_being_synced":"false","binlog.enable":"false","binlog.ttl_seconds":"86400","inverted_index_storage_format":"V2","time_series_compaction_empty_rowsets_threshold":"5","default.replication_allocation":"tag.location.default: 1","time_series_compaction_level_threshold":"1","time_series_compaction_time_threshold_seconds":"3600","storage_format":"V2","store_row_column":"false","light_schema_change":"true","enable_unique_key_merge_on_write":"false","in_memory":"false","file_cache_ttl_seconds":"0","group_commit_data_bytes":"134217728","compaction_policy":"size_based","_auto_bucket":"false","binlog.max_history_nums":"9223372036854775807","time_series_compaction_file_count_threshold":"2000","skip_write_index_on_load":"false","disable_auto_compaction":"false","time_series_compaction_goal_size_mbytes":"1024","storage_medium":"HDD","enable_single_replica_compaction":"false","compression":"LZ4F","binlog.max_bytes":"9223372036854775807"} +rangetable internal test_table_options_db AGG user_id,date,timestamp,city,age,sex user_id HASH 8 3 {"min_load_replica_num":"-1","data_sort.col_num":"6","group_commit_interval_ms":"10000","data_sort.sort_type":"LEXICAL","is_being_synced":"false","binlog.enable":"false","binlog.ttl_seconds":"86400","inverted_index_storage_format":"V2","time_series_compaction_empty_rowsets_threshold":"5","default.replication_allocation":"tag.location.default: 1","time_series_compaction_level_threshold":"1","time_series_compaction_time_threshold_seconds":"3600","storage_format":"V2","store_row_column":"false","light_schema_change":"true","enable_unique_key_merge_on_write":"false","in_memory":"false","file_cache_ttl_seconds":"0","group_commit_data_bytes":"134217728","compaction_policy":"size_based","_auto_bucket":"false","binlog.max_history_nums":"9223372036854775807","time_series_compaction_file_count_threshold":"2000","skip_write_index_on_load":"false","disable_auto_compaction":"false","time_series_compaction_goal_size_mbytes":"1024","storage_medium":"HDD","enable_single_replica_compaction":"false","compression":"LZ4F","binlog.max_bytes":"9223372036854775807"} +unique_table internal test_table_options_db UNI user_id,username user_id HASH 1 1 {"min_load_replica_num":"-1","data_sort.col_num":"2","group_commit_interval_ms":"10000","data_sort.sort_type":"LEXICAL","is_being_synced":"false","binlog.enable":"false","binlog.ttl_seconds":"86400","inverted_index_storage_format":"V2","time_series_compaction_empty_rowsets_threshold":"5","default.replication_allocation":"tag.location.default: 1","time_series_compaction_level_threshold":"1","time_series_compaction_time_threshold_seconds":"3600","storage_format":"V2","store_row_column":"false","light_schema_change":"true","enable_unique_key_merge_on_write":"true","in_memory":"false","file_cache_ttl_seconds":"0","group_commit_data_bytes":"134217728","compaction_policy":"size_based","_auto_bucket":"false","binlog.max_history_nums":"9223372036854775807","time_series_compaction_file_count_threshold":"2000","skip_write_index_on_load":"false","disable_auto_compaction":"false","time_series_compaction_goal_size_mbytes":"1024","storage_medium":"HDD","enable_single_replica_compaction":"false","compression":"LZ4F","binlog.max_bytes":"9223372036854775807"} diff --git a/regression-test/suites/fault_injection_p0/test_calc_crc_fault_injection.groovy b/regression-test/suites/fault_injection_p0/test_calc_crc_fault_injection.groovy index 9fb9d88e168872..f4971f3fe569ab 100644 --- a/regression-test/suites/fault_injection_p0/test_calc_crc_fault_injection.groovy +++ b/regression-test/suites/fault_injection_p0/test_calc_crc_fault_injection.groovy @@ -118,7 +118,7 @@ suite("test_calc_crc") { assertEquals("5", parseJson(out_5.trim()).start_version) assertEquals("7", parseJson(out_5.trim()).end_version) assertEquals("3", parseJson(out_5.trim()).rowset_count) - assertEquals("9", parseJson(out_5.trim()).file_count) + assertEquals("6", parseJson(out_5.trim()).file_count) def (code_6, out_6, err_6) = calc_file_crc_on_tablet(ip, port, 123) logger.info("Run calc_file_crc_on_tablet: code=" + code_6 + ", out=" + out_6 + ", err=" + err_6)