From 8620d863d77e4e84f12d2659a7210c59edf2d917 Mon Sep 17 00:00:00 2001 From: abmdocrt Date: Wed, 26 Jun 2024 15:06:05 +0800 Subject: [PATCH 1/6] [Fix](delete command) Mark delete sign when do delete command in MoW table (#35917) close #34551 Problem: As shown in the issue above, if a key deleted by a delete statement is written to by updating only certain columns, the data will not display correctly. Reason: The delete statement deletes the data by writing a delete predicate, which is stored in the rowset meta and applied during data retrieval to filter the data. However, partial column updates do not consider the effect of the delete predicate when reading the original data. The imported key should be considered as a new key (since it has already been deleted), but it is actually treated as an old key. Therefore, only some columns are updated, leading to incorrect results. Solution: Consider the delete predicate during partial column updates, but this method will result in reading more columns, as shown in #35766. Thus, in this PR, we change the delete operation in the mow table from writing a delete predicate to writing a delete sign, which effectively resolves the issue. --- .../java/org/apache/doris/common/Config.java | 9 + .../doris/alter/SchemaChangeHandler.java | 8 + .../org/apache/doris/analysis/DeleteStmt.java | 3 +- .../analysis/ModifyTablePropertiesClause.java | 40 +++ .../org/apache/doris/catalog/OlapTable.java | 8 + .../apache/doris/catalog/TableProperty.java | 10 + .../doris/common/util/PropertyAnalyzer.java | 24 ++ .../doris/datasource/InternalCatalog.java | 14 + .../trees/plans/commands/DeleteCommand.java | 10 +- .../analysis/CreateTableAsSelectStmtTest.java | 3 +- .../data/compaction/test_full_compaction.out | 2 + .../test_full_compaction_by_table_id.out | 2 + .../data/delete_p0/test_delete_on_value.out | 14 +- .../delete/delete_mow_partial_update.out | 12 +- .../explain/test_pushdown_explain.out | 5 +- .../test_new_partial_update_delete.out | 129 +++++++++ .../test_partial_update_after_delete.out | 13 + .../test_partial_update_delete.out | 12 +- .../test_new_partial_update_delete.groovy | 252 ++++++++++++++++++ .../test_partial_update_after_delete.groovy | 79 ++++++ 20 files changed, 622 insertions(+), 27 deletions(-) create mode 100644 regression-test/data/unique_with_mow_p0/partial_update/test_new_partial_update_delete.out create mode 100644 regression-test/data/unique_with_mow_p0/partial_update/test_partial_update_after_delete.out create mode 100644 regression-test/suites/unique_with_mow_p0/partial_update/test_new_partial_update_delete.groovy create mode 100644 regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_after_delete.groovy diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java index 57b6cf0e2eb1f7..5fea51e764e418 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java @@ -2395,6 +2395,15 @@ public class Config extends ConfigBase { @ConfField public static boolean ignore_bdbje_log_checksum_read = false; + @ConfField(mutable = true, masterOnly = true, description = { + "是否在unique表mow上开启delete语句写delete predicate。若开启，会提升delete语句的性能，" + + "但delete后进行部分列更新可能会出现部分数据错误的情况。若关闭，会降低delete语句的性能来保证正确性。", + "Enable the 'delete predicate' for DELETE statements. If enabled, it will enhance the performance of " + + "DELETE statements, but partial column updates after a DELETE may result in erroneous data. " + + "If disabled, it will reduce the performance of DELETE statements to ensure accuracy." + }) + public static boolean enable_mow_delete_on_predicate = false; + @ConfField(description = { "是否开启 Proxy Protocol 支持", "Whether to enable proxy protocol" diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java index 198aaf7410009f..0a1e72b8700085 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java @@ -2215,6 +2215,7 @@ public void updateTableProperties(Database db, String tableName, Map propertie throw new AnalysisException(PropertyAnalyzer.ENABLE_UNIQUE_KEY_MERGE_ON_WRITE + " must be `true` or `false`"); } + public static boolean analyzeEnableDeleteOnDeletePredicate(Map properties) + throws AnalysisException { + if (properties == null || properties.isEmpty()) { + return false; + } + String value = properties.get(PropertyAnalyzer.PROPERTIES_ENABLE_MOW_DELETE_ON_DELETE_PREDICATE); + if (value == null) { + return false; + } + properties.remove(PropertyAnalyzer.PROPERTIES_ENABLE_MOW_DELETE_ON_DELETE_PREDICATE); + if (value.equals("true")) { + return true; + } else if (value.equals("false")) { + return false; + } + throw new AnalysisException( + PropertyAnalyzer.PROPERTIES_ENABLE_MOW_DELETE_ON_DELETE_PREDICATE + " must be `true` or `false`"); + } + /** * Check the type property of the catalog props. */ diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java index cfe94a1dded9ab..e6f1a43dc60e6a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java @@ -2203,6 +2203,20 @@ private boolean createOlapTable(Database db, CreateTableStmt stmt) throws UserEx } olapTable.setEnableUniqueKeyMergeOnWrite(enableUniqueKeyMergeOnWrite); + boolean enableDeleteOnDeletePredicate = false; + if (keysType == KeysType.UNIQUE_KEYS) { + try { + enableDeleteOnDeletePredicate = PropertyAnalyzer.analyzeEnableDeleteOnDeletePredicate(properties); + } catch (AnalysisException e) { + throw new DdlException(e.getMessage()); + } + if (enableDeleteOnDeletePredicate && !enableUniqueKeyMergeOnWrite) { + throw new DdlException(PropertyAnalyzer.PROPERTIES_ENABLE_MOW_DELETE_ON_DELETE_PREDICATE + + " property is not supported for unique merge-on-read table"); + } + } + olapTable.setEnableDeleteOnDeletePredicate(enableDeleteOnDeletePredicate); + boolean enableSingleReplicaCompaction = false; try { enableSingleReplicaCompaction = PropertyAnalyzer.analyzeEnableSingleReplicaCompaction(properties); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/DeleteCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/DeleteCommand.java index 880747cdc5577e..437c1929b106ce 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/DeleteCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/DeleteCommand.java @@ -98,15 +98,16 @@ public LogicalPlan completeQueryPlan(ConnectContext ctx, LogicalPlan logicalQuer String tableName = tableAlias != null ? tableAlias : targetTable.getName(); for (Column column : targetTable.getFullSchema()) { if (column.getName().equalsIgnoreCase(Column.DELETE_SIGN)) { - selectLists.add(new Alias(new TinyIntLiteral(((byte) 1)), Column.DELETE_SIGN)); - } else if (column.getName().equalsIgnoreCase(Column.SEQUENCE_COL)) { + selectLists.add(new UnboundAlias(new TinyIntLiteral(((byte) 1)), Column.DELETE_SIGN)); + } else if (column.getName().equalsIgnoreCase(Column.SEQUENCE_COL) + && targetTable.getSequenceMapCol() != null) { selectLists.add(new UnboundSlot(tableName, targetTable.getSequenceMapCol())); } else if (column.isKey()) { selectLists.add(new UnboundSlot(tableName, column.getName())); } else if (!isMow && (!column.isVisible() || (!column.isAllowNull() && !column.hasDefaultValue()))) { selectLists.add(new UnboundSlot(tableName, column.getName())); } else { - continue; + selectLists.add(new UnboundSlot(tableName, column.getName())); } cols.add(column.getName()); } @@ -116,9 +117,6 @@ public LogicalPlan completeQueryPlan(ConnectContext ctx, LogicalPlan logicalQuer logicalQuery = ((LogicalPlan) cte.get().withChildren(logicalQuery)); } - boolean isPartialUpdate = targetTable.getEnableUniqueKeyMergeOnWrite() - && cols.size() < targetTable.getColumns().size(); - // make UnboundTableSink return new UnboundOlapTableSink<>(nameParts, cols, ImmutableList.of(), partitions, isPartialUpdate, logicalQuery); diff --git a/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateTableAsSelectStmtTest.java b/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateTableAsSelectStmtTest.java index fa3a029c43bfc9..6ffa8239b0ad89 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateTableAsSelectStmtTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateTableAsSelectStmtTest.java @@ -503,7 +503,8 @@ public void testUseKeyType() throws Exception { + "\"storage_format\" = \"V2\",\n" + "\"light_schema_change\" = \"true\",\n" + "\"disable_auto_compaction\" = \"false\",\n" - + "\"enable_single_replica_compaction\" = \"false\"\n" + + "\"enable_single_replica_compaction\" = \"false\",\n" + + "\"enable_mow_delete_on_delete_predicate\" = \"false\"\n" + ");", showResultSet.getResultRows().get(0).get(1)); } diff --git a/regression-test/data/compaction/test_full_compaction.out b/regression-test/data/compaction/test_full_compaction.out index ead5be01b4e6b0..7098be6d89d2e6 100644 --- a/regression-test/data/compaction/test_full_compaction.out +++ b/regression-test/data/compaction/test_full_compaction.out @@ -33,9 +33,11 @@ 2 20 2 200 3 100 +3 100 3 300 -- !select_final -- 1 100 2 200 +3 100 diff --git a/regression-test/data/compaction/test_full_compaction_by_table_id.out b/regression-test/data/compaction/test_full_compaction_by_table_id.out index ead5be01b4e6b0..7098be6d89d2e6 100644 --- a/regression-test/data/compaction/test_full_compaction_by_table_id.out +++ b/regression-test/data/compaction/test_full_compaction_by_table_id.out @@ -33,9 +33,11 @@ 2 20 2 200 3 100 +3 100 3 300 -- !select_final -- 1 100 2 200 +3 100 diff --git a/regression-test/data/delete_p0/test_delete_on_value.out b/regression-test/data/delete_p0/test_delete_on_value.out index 2bc8846297d3dd..4a6e652aef3887 100644 --- a/regression-test/data/delete_p0/test_delete_on_value.out +++ b/regression-test/data/delete_p0/test_delete_on_value.out @@ -29,11 +29,6 @@ -- !sql -- 1 1 1 0 2 2 2 0 -3 3 3 0 -4 4 4 0 -5 5 5 0 -6 6 6 0 -7 7 7 0 8 8 8 0 9 9 9 0 @@ -56,12 +51,18 @@ 1 1 1 0 2 2 2 0 3 3 3 0 +3 3 3 1 4 4 4 0 4 4 4 0 +4 4 4 1 5 5 5 0 5 5 5 0 +5 5 5 1 +5 5 5 1 6 6 6 0 +6 6 6 1 7 7 7 0 +7 7 7 1 8 8 8 0 9 9 9 0 @@ -77,6 +78,7 @@ -- !sql -- 1 1 5 0 3 5 1 1 10 0 2 10 +1 1 10 1 4 10 -- !sql -- 1 1 10 @@ -88,7 +90,7 @@ -- !sql -- -- !sql -- -1 \N \N 1 4 10 1 1 5 0 3 5 1 1 10 0 2 10 +1 1 10 1 4 10 diff --git a/regression-test/data/nereids_p0/delete/delete_mow_partial_update.out b/regression-test/data/nereids_p0/delete/delete_mow_partial_update.out index 787f854bea29bc..b4237a038036db 100644 --- a/regression-test/data/nereids_p0/delete/delete_mow_partial_update.out +++ b/regression-test/data/nereids_p0/delete/delete_mow_partial_update.out @@ -15,12 +15,12 @@ 5 5 -- !sql -- -1 \N 1 1 1 0 -2 \N 1 +1 1 1 2 2 0 -3 \N 1 +2 2 1 3 3 0 +3 3 1 4 4 0 5 5 0 @@ -61,12 +61,12 @@ 5 5 -- !sql -- -1 \N 1 1 1 0 -2 \N 1 +1 1 1 2 2 0 -3 \N 1 +2 2 1 3 3 0 +3 3 1 4 4 0 5 5 0 diff --git a/regression-test/data/nereids_p0/explain/test_pushdown_explain.out b/regression-test/data/nereids_p0/explain/test_pushdown_explain.out index eaac687e2d5176..fe2402b60b8481 100644 --- a/regression-test/data/nereids_p0/explain/test_pushdown_explain.out +++ b/regression-test/data/nereids_p0/explain/test_pushdown_explain.out @@ -96,7 +96,10 @@ zzz asd -- !select_16 -- -qdf +qwe + +-- !select_17 -- +cc -- !select_18 -- zzz diff --git a/regression-test/data/unique_with_mow_p0/partial_update/test_new_partial_update_delete.out b/regression-test/data/unique_with_mow_p0/partial_update/test_new_partial_update_delete.out new file mode 100644 index 00000000000000..a248196690e007 --- /dev/null +++ b/regression-test/data/unique_with_mow_p0/partial_update/test_new_partial_update_delete.out @@ -0,0 +1,129 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !sql_table1 -- +test_new_partial_update_delete1 CREATE TABLE `test_new_partial_update_delete1` (\n `k1` INT NOT NULL,\n `c1` INT NULL,\n `c2` INT NULL,\n `c3` INT NULL,\n `c4` INT NULL\n) ENGINE=OLAP\nUNIQUE KEY(`k1`)\nDISTRIBUTED BY HASH(`k1`) BUCKETS 1\nPROPERTIES (\n"replication_allocation" = "tag.location.default: 1",\n"min_load_replica_num" = "-1",\n"is_being_synced" = "false",\n"storage_medium" = "hdd",\n"storage_format" = "V2",\n"inverted_index_storage_format" = "V2",\n"enable_unique_key_merge_on_write" = "true",\n"light_schema_change" = "true",\n"disable_auto_compaction" = "true",\n"binlog.enable" = "false",\n"binlog.ttl_seconds" = "86400",\n"binlog.max_bytes" = "9223372036854775807",\n"binlog.max_history_nums" = "9223372036854775807",\n"enable_single_replica_compaction" = "false",\n"group_commit_interval_ms" = "10000",\n"group_commit_data_bytes" = "134217728",\n"enable_mow_delete_on_delete_predicate" = "false"\n); + +-- !sql1 -- +1 1 1 1 1 + +-- !sql2 -- + +-- !sql3 -- +1 1 1 1 1 1 + +-- !sql4 -- +1 2 \N \N \N + +-- !sql_table2 -- +test_new_partial_update_delete1 CREATE TABLE `test_new_partial_update_delete1` (\n `k1` INT NOT NULL,\n `c1` INT NULL,\n `c2` INT NULL,\n `c3` INT NULL,\n `c4` INT NULL\n) ENGINE=OLAP\nUNIQUE KEY(`k1`)\nDISTRIBUTED BY HASH(`k1`) BUCKETS 1\nPROPERTIES (\n"replication_allocation" = "tag.location.default: 1",\n"min_load_replica_num" = "-1",\n"is_being_synced" = "false",\n"storage_medium" = "hdd",\n"storage_format" = "V2",\n"inverted_index_storage_format" = "V2",\n"enable_unique_key_merge_on_write" = "true",\n"light_schema_change" = "true",\n"disable_auto_compaction" = "true",\n"binlog.enable" = "false",\n"binlog.ttl_seconds" = "86400",\n"binlog.max_bytes" = "9223372036854775807",\n"binlog.max_history_nums" = "9223372036854775807",\n"enable_single_replica_compaction" = "false",\n"group_commit_interval_ms" = "10000",\n"group_commit_data_bytes" = "134217728",\n"enable_mow_delete_on_delete_predicate" = "true"\n); + +-- !sql11 -- +1 2 \N \N \N +2 2 2 2 2 + +-- !sql12 -- + +-- !sql13 -- + +-- !sql14 -- +1 2 \N \N \N +2 2 2 2 2 + +-- !sql15 -- +2 3 2 2 2 + +-- !sql_table21 -- +test_new_partial_update_delete2 CREATE TABLE `test_new_partial_update_delete2` (\n `k1` INT NOT NULL,\n `c1` INT NULL,\n `c2` INT NULL,\n `c3` INT NULL,\n `c4` INT NULL\n) ENGINE=OLAP\nUNIQUE KEY(`k1`)\nDISTRIBUTED BY HASH(`k1`) BUCKETS 1\nPROPERTIES (\n"replication_allocation" = "tag.location.default: 1",\n"min_load_replica_num" = "-1",\n"is_being_synced" = "false",\n"storage_medium" = "hdd",\n"storage_format" = "V2",\n"inverted_index_storage_format" = "V2",\n"enable_unique_key_merge_on_write" = "true",\n"light_schema_change" = "true",\n"disable_auto_compaction" = "true",\n"binlog.enable" = "false",\n"binlog.ttl_seconds" = "86400",\n"binlog.max_bytes" = "9223372036854775807",\n"binlog.max_history_nums" = "9223372036854775807",\n"enable_single_replica_compaction" = "false",\n"group_commit_interval_ms" = "10000",\n"group_commit_data_bytes" = "134217728",\n"enable_mow_delete_on_delete_predicate" = "false"\n); + +-- !sql21 -- +1 1 1 1 1 + +-- !sql22 -- + +-- !sql23 -- +1 \N \N \N \N 1 + +-- !sql24 -- +1 2 \N \N \N + +-- !sql_table32 -- +test_new_partial_update_delete2 CREATE TABLE `test_new_partial_update_delete2` (\n `k1` INT NOT NULL,\n `c1` INT NULL,\n `c2` INT NULL,\n `c3` INT NULL,\n `c4` INT NULL\n) ENGINE=OLAP\nUNIQUE KEY(`k1`)\nDISTRIBUTED BY HASH(`k1`) BUCKETS 1\nPROPERTIES (\n"replication_allocation" = "tag.location.default: 1",\n"min_load_replica_num" = "-1",\n"is_being_synced" = "false",\n"storage_medium" = "hdd",\n"storage_format" = "V2",\n"inverted_index_storage_format" = "V2",\n"enable_unique_key_merge_on_write" = "true",\n"light_schema_change" = "true",\n"disable_auto_compaction" = "true",\n"binlog.enable" = "false",\n"binlog.ttl_seconds" = "86400",\n"binlog.max_bytes" = "9223372036854775807",\n"binlog.max_history_nums" = "9223372036854775807",\n"enable_single_replica_compaction" = "false",\n"group_commit_interval_ms" = "10000",\n"group_commit_data_bytes" = "134217728",\n"enable_mow_delete_on_delete_predicate" = "true"\n); + +-- !sql31 -- +1 2 \N \N \N +2 2 2 2 2 + +-- !sql32 -- + +-- !sql33 -- + +-- !sql34 -- +1 2 \N \N \N +2 2 2 2 2 + +-- !sql35 -- +2 3 2 2 2 + +-- !sql_table1 -- +test_new_partial_update_delete1 CREATE TABLE `test_new_partial_update_delete1` (\n `k1` INT NOT NULL,\n `c1` INT NULL,\n `c2` INT NULL,\n `c3` INT NULL,\n `c4` INT NULL\n) ENGINE=OLAP\nUNIQUE KEY(`k1`)\nDISTRIBUTED BY HASH(`k1`) BUCKETS 1\nPROPERTIES (\n"replication_allocation" = "tag.location.default: 1",\n"min_load_replica_num" = "-1",\n"is_being_synced" = "false",\n"storage_medium" = "hdd",\n"storage_format" = "V2",\n"inverted_index_storage_format" = "V2",\n"enable_unique_key_merge_on_write" = "true",\n"light_schema_change" = "true",\n"store_row_column" = "true",\n"disable_auto_compaction" = "true",\n"binlog.enable" = "false",\n"binlog.ttl_seconds" = "86400",\n"binlog.max_bytes" = "9223372036854775807",\n"binlog.max_history_nums" = "9223372036854775807",\n"enable_single_replica_compaction" = "false",\n"group_commit_interval_ms" = "10000",\n"group_commit_data_bytes" = "134217728",\n"enable_mow_delete_on_delete_predicate" = "false"\n); + +-- !sql1 -- +1 1 1 1 1 + +-- !sql2 -- + +-- !sql3 -- +1 1 1 1 1 1 + +-- !sql4 -- +1 2 \N \N \N + +-- !sql_table2 -- +test_new_partial_update_delete1 CREATE TABLE `test_new_partial_update_delete1` (\n `k1` INT NOT NULL,\n `c1` INT NULL,\n `c2` INT NULL,\n `c3` INT NULL,\n `c4` INT NULL\n) ENGINE=OLAP\nUNIQUE KEY(`k1`)\nDISTRIBUTED BY HASH(`k1`) BUCKETS 1\nPROPERTIES (\n"replication_allocation" = "tag.location.default: 1",\n"min_load_replica_num" = "-1",\n"is_being_synced" = "false",\n"storage_medium" = "hdd",\n"storage_format" = "V2",\n"inverted_index_storage_format" = "V2",\n"enable_unique_key_merge_on_write" = "true",\n"light_schema_change" = "true",\n"store_row_column" = "true",\n"disable_auto_compaction" = "true",\n"binlog.enable" = "false",\n"binlog.ttl_seconds" = "86400",\n"binlog.max_bytes" = "9223372036854775807",\n"binlog.max_history_nums" = "9223372036854775807",\n"enable_single_replica_compaction" = "false",\n"group_commit_interval_ms" = "10000",\n"group_commit_data_bytes" = "134217728",\n"enable_mow_delete_on_delete_predicate" = "true"\n); + +-- !sql11 -- +1 2 \N \N \N +2 2 2 2 2 + +-- !sql12 -- + +-- !sql13 -- + +-- !sql14 -- +1 2 \N \N \N +2 2 2 2 2 + +-- !sql15 -- +2 3 2 2 2 + +-- !sql_table21 -- +test_new_partial_update_delete2 CREATE TABLE `test_new_partial_update_delete2` (\n `k1` INT NOT NULL,\n `c1` INT NULL,\n `c2` INT NULL,\n `c3` INT NULL,\n `c4` INT NULL\n) ENGINE=OLAP\nUNIQUE KEY(`k1`)\nDISTRIBUTED BY HASH(`k1`) BUCKETS 1\nPROPERTIES (\n"replication_allocation" = "tag.location.default: 1",\n"min_load_replica_num" = "-1",\n"is_being_synced" = "false",\n"storage_medium" = "hdd",\n"storage_format" = "V2",\n"inverted_index_storage_format" = "V2",\n"enable_unique_key_merge_on_write" = "true",\n"light_schema_change" = "true",\n"store_row_column" = "true",\n"disable_auto_compaction" = "true",\n"binlog.enable" = "false",\n"binlog.ttl_seconds" = "86400",\n"binlog.max_bytes" = "9223372036854775807",\n"binlog.max_history_nums" = "9223372036854775807",\n"enable_single_replica_compaction" = "false",\n"group_commit_interval_ms" = "10000",\n"group_commit_data_bytes" = "134217728",\n"enable_mow_delete_on_delete_predicate" = "false"\n); + +-- !sql21 -- +1 1 1 1 1 + +-- !sql22 -- + +-- !sql23 -- +1 \N \N \N \N 1 + +-- !sql24 -- +1 2 \N \N \N + +-- !sql_table32 -- +test_new_partial_update_delete2 CREATE TABLE `test_new_partial_update_delete2` (\n `k1` INT NOT NULL,\n `c1` INT NULL,\n `c2` INT NULL,\n `c3` INT NULL,\n `c4` INT NULL\n) ENGINE=OLAP\nUNIQUE KEY(`k1`)\nDISTRIBUTED BY HASH(`k1`) BUCKETS 1\nPROPERTIES (\n"replication_allocation" = "tag.location.default: 1",\n"min_load_replica_num" = "-1",\n"is_being_synced" = "false",\n"storage_medium" = "hdd",\n"storage_format" = "V2",\n"inverted_index_storage_format" = "V2",\n"enable_unique_key_merge_on_write" = "true",\n"light_schema_change" = "true",\n"store_row_column" = "true",\n"disable_auto_compaction" = "true",\n"binlog.enable" = "false",\n"binlog.ttl_seconds" = "86400",\n"binlog.max_bytes" = "9223372036854775807",\n"binlog.max_history_nums" = "9223372036854775807",\n"enable_single_replica_compaction" = "false",\n"group_commit_interval_ms" = "10000",\n"group_commit_data_bytes" = "134217728",\n"enable_mow_delete_on_delete_predicate" = "true"\n); + +-- !sql31 -- +1 2 \N \N \N +2 2 2 2 2 + +-- !sql32 -- + +-- !sql33 -- + +-- !sql34 -- +1 2 \N \N \N +2 2 2 2 2 + +-- !sql35 -- +2 3 2 2 2 + diff --git a/regression-test/data/unique_with_mow_p0/partial_update/test_partial_update_after_delete.out b/regression-test/data/unique_with_mow_p0/partial_update/test_partial_update_after_delete.out new file mode 100644 index 00000000000000..a7b2a03456ab05 --- /dev/null +++ b/regression-test/data/unique_with_mow_p0/partial_update/test_partial_update_after_delete.out @@ -0,0 +1,13 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !select1 -- +1 2 \N + +-- !select2 -- +1 2 \N + +-- !select1 -- +1 2 \N + +-- !select2 -- +1 2 \N + diff --git a/regression-test/data/unique_with_mow_p0/partial_update/test_partial_update_delete.out b/regression-test/data/unique_with_mow_p0/partial_update/test_partial_update_delete.out index 0863afd7931780..c16e954d73309c 100644 --- a/regression-test/data/unique_with_mow_p0/partial_update/test_partial_update_delete.out +++ b/regression-test/data/unique_with_mow_p0/partial_update/test_partial_update_delete.out @@ -11,12 +11,12 @@ 5 5 5 5 5 -- !with_delete_sign -- -1 \N \N 0 \N 1 1 1 1 1 1 0 -2 \N \N 0 \N 1 +1 1 1 1 1 1 2 2 2 2 2 0 -3 \N \N 0 \N 1 +2 2 2 2 2 1 3 3 3 3 3 0 +3 3 3 3 3 1 4 4 4 4 4 0 5 5 5 5 5 0 @@ -53,12 +53,12 @@ 5 5 5 5 5 -- !with_delete_sign -- -1 \N \N 0 \N 1 1 1 1 1 1 0 -2 \N \N 0 \N 1 +1 1 1 1 1 1 2 2 2 2 2 0 -3 \N \N 0 \N 1 +2 2 2 2 2 1 3 3 3 3 3 0 +3 3 3 3 3 1 4 4 4 4 4 0 5 5 5 5 5 0 diff --git a/regression-test/suites/unique_with_mow_p0/partial_update/test_new_partial_update_delete.groovy b/regression-test/suites/unique_with_mow_p0/partial_update/test_new_partial_update_delete.groovy new file mode 100644 index 00000000000000..85434f61987ea5 --- /dev/null +++ b/regression-test/suites/unique_with_mow_p0/partial_update/test_new_partial_update_delete.groovy @@ -0,0 +1,252 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite('test_new_partial_update_delete') { + + String db = context.config.getDbNameByFile(context.file) + sql "select 1;" // to create database + + for (def use_row_store : [false, true]) { + logger.info("current params: use_row_store: ${use_row_store}") + + connect(user = context.config.jdbcUser, password = context.config.jdbcPassword, url = context.config.jdbcUrl) { + sql "use ${db};" + + sql "set enable_nereids_planner=true" + sql "set enable_fallback_to_original_planner=false" + + try { + def tableMorName1 = "test_new_partial_update_mor_delete1" + sql "DROP TABLE IF EXISTS ${tableMorName1};" + sql """ CREATE TABLE IF NOT EXISTS ${tableMorName1} ( + `k1` int NOT NULL, + `c1` int, + `c2` int, + `c3` int, + `c4` int + )UNIQUE KEY(k1) + DISTRIBUTED BY HASH(k1) BUCKETS 1 + PROPERTIES ( + "disable_auto_compaction" = "true", + "enable_unique_key_merge_on_write" = "false", + "enable_mow_delete_on_delete_predicate" = "true", + "replication_num" = "1", + "store_row_column" = "${use_row_store}"); """ + } catch (Exception e) { + log.info(e.getMessage()) + assertTrue(e.getMessage().contains('enable_mow_delete_on_delete_predicate property is not supported for unique merge-on-read table')) + } + + try { + def tableMorName2 = "test_new_partial_update_mor_delete2" + sql "DROP TABLE IF EXISTS ${tableMorName2};" + sql """ CREATE TABLE IF NOT EXISTS ${tableMorName2} ( + `k1` int NOT NULL, + `c1` int, + `c2` int, + `c3` int, + `c4` int + )UNIQUE KEY(k1) + DISTRIBUTED BY HASH(k1) BUCKETS 1 + PROPERTIES ( + "disable_auto_compaction" = "true", + "enable_unique_key_merge_on_write" = "false", + "replication_num" = "1", + "store_row_column" = "${use_row_store}"); """ + sql """alter table ${tableMorName2} set ("enable_mow_delete_on_delete_predicate"="true")""" + } catch (Exception e) { + log.info(e.getMessage()) + assertTrue(e.getMessage().contains('enable_mow_delete_on_delete_predicate property is not supported for unique merge-on-read table')) + } + + def tableName1 = "test_new_partial_update_delete1" + sql "DROP TABLE IF EXISTS ${tableName1};" + sql """ CREATE TABLE IF NOT EXISTS ${tableName1} ( + `k1` int NOT NULL, + `c1` int, + `c2` int, + `c3` int, + `c4` int + )UNIQUE KEY(k1) + DISTRIBUTED BY HASH(k1) BUCKETS 1 + PROPERTIES ( + "disable_auto_compaction" = "true", + "replication_num" = "1", + "store_row_column" = "${use_row_store}"); """ + + qt_sql_table1 "show create table ${tableName1}" + sql "insert into ${tableName1} values(1,1,1,1,1)" + // 1,1,1,1,1 + qt_sql1 "select * from ${tableName1} order by k1;" + sql "delete from ${tableName1} where k1 = 1" + // empty + qt_sql2 "select * from ${tableName1} order by k1;" + sql "set show_hidden_columns = true;" + // 1,1,1,1,1,1 + qt_sql3 "select k1,c1,c2,c3,c4,__DORIS_DELETE_SIGN__ from ${tableName1} order by k1;" + sql "set show_hidden_columns = false;" + sql "set enable_unique_key_partial_update=true;" + sql "set enable_insert_strict=false;" + sql "insert into ${tableName1} (k1,c1) values(1,2)" + // 1,2,NULL,NULL,NULL + qt_sql4 "select * from ${tableName1} order by k1;" + + + + sql """alter table ${tableName1} set ("enable_mow_delete_on_delete_predicate"="true") """ + sql "set enable_unique_key_partial_update=false;" + sql "set enable_insert_strict=true;" + qt_sql_table2 "show create table ${tableName1}" + sql "insert into ${tableName1} values(2,2,2,2,2)" + // 1,2,NULL,NULL,NULL + // 2,2,2,2,2 + qt_sql11 "select * from ${tableName1} order by k1;" + sql "delete from ${tableName1} where k1 <= 2" + // empty + qt_sql12 "select * from ${tableName1} order by k1;" + sql "set show_hidden_columns = true;" + // empty + qt_sql13 "select * from ${tableName1} order by k1;" + sql "set show_hidden_columns = false;" + sql "set skip_delete_predicate = true;" + // 1,2,NULL,NULL,NULL + // 2,2,2,2,2 + qt_sql14 "select * from ${tableName1} order by k1;" + sql "set skip_delete_predicate = false;" + sql "set enable_unique_key_partial_update=true;" + sql "set enable_insert_strict=false;" + sql "insert into ${tableName1} (k1,c1) values(2,3)" + // 2,3,2,2,2 + qt_sql15 "select * from ${tableName1} order by k1;" + sql "set enable_unique_key_partial_update=false;" + sql "set enable_insert_strict=true;" + + sql "drop table if exists ${tableName1};" + + + + // old planner + try { + def tableMorName3 = "test_new_partial_update_mor_delete3" + sql "DROP TABLE IF EXISTS ${tableMorName3};" + sql """ CREATE TABLE IF NOT EXISTS ${tableMorName3} ( + `k1` int NOT NULL, + `c1` int, + `c2` int, + `c3` int, + `c4` int + )UNIQUE KEY(k1) + DISTRIBUTED BY HASH(k1) BUCKETS 1 + PROPERTIES ( + "disable_auto_compaction" = "true", + "enable_unique_key_merge_on_write" = "false", + "enable_mow_delete_on_delete_predicate" = "true", + "replication_num" = "1", + "store_row_column" = "${use_row_store}"); """ + } catch (Exception e) { + log.info(e.getMessage()) + assertTrue(e.getMessage().contains('enable_mow_delete_on_delete_predicate property is not supported for unique merge-on-read table')) + } + + try { + def tableMorName4 = "test_new_partial_update_mor_delete4" + sql "DROP TABLE IF EXISTS ${tableMorName4};" + sql """ CREATE TABLE IF NOT EXISTS ${tableMorName4} ( + `k1` int NOT NULL, + `c1` int, + `c2` int, + `c3` int, + `c4` int + )UNIQUE KEY(k1) + DISTRIBUTED BY HASH(k1) BUCKETS 1 + PROPERTIES ( + "disable_auto_compaction" = "true", + "enable_unique_key_merge_on_write" = "false", + "replication_num" = "1", + "store_row_column" = "${use_row_store}"); """ + sql """alter table ${tableMorName4} set ("enable_mow_delete_on_delete_predicate"="true")""" + } catch (Exception e) { + log.info(e.getMessage()) + assertTrue(e.getMessage().contains('enable_mow_delete_on_delete_predicate property is not supported for unique merge-on-read table')) + } + sql "set enable_nereids_planner=false" + def tableName2 = "test_new_partial_update_delete2" + sql "DROP TABLE IF EXISTS ${tableName2};" + sql """ CREATE TABLE IF NOT EXISTS ${tableName2} ( + `k1` int NOT NULL, + `c1` int, + `c2` int, + `c3` int, + `c4` int + )UNIQUE KEY(k1) + DISTRIBUTED BY HASH(k1) BUCKETS 1 + PROPERTIES ( + "disable_auto_compaction" = "true", + "replication_num" = "1", + "store_row_column" = "${use_row_store}"); """ + + qt_sql_table21 "show create table ${tableName2}" + sql "insert into ${tableName2} values(1,1,1,1,1)" + // 1,1,1,1,1 + qt_sql21 "select * from ${tableName2} order by k1;" + sql "delete from ${tableName2} where k1 = 1" + // empty + qt_sql22 "select * from ${tableName2} order by k1;" + sql "set show_hidden_columns = true;" + // 1,1,1,1,1,1 + qt_sql23 "select k1,c1,c2,c3,c4,__DORIS_DELETE_SIGN__ from ${tableName2} order by k1;" + sql "set show_hidden_columns = false;" + sql "set enable_unique_key_partial_update=true;" + sql "set enable_insert_strict=false;" + sql "insert into ${tableName2} (k1,c1) values(1,2)" + // 1,2,NULL,NULL,NULL + qt_sql24 "select * from ${tableName2} order by k1;" + + + + sql """alter table ${tableName2} set ("enable_mow_delete_on_delete_predicate"="true") """ + sql "set enable_unique_key_partial_update=false;" + sql "set enable_insert_strict=true;" + qt_sql_table32 "show create table ${tableName2}" + sql "insert into ${tableName2} values(2,2,2,2,2)" + // 1,2,NULL,NULL,NULL + // 2,2,2,2,2 + qt_sql31 "select * from ${tableName2} order by k1;" + sql "delete from ${tableName2} where k1 <= 2" + // empty + qt_sql32 "select * from ${tableName2} order by k1;" + sql "set show_hidden_columns = true;" + // empty + qt_sql33 "select * from ${tableName2} order by k1;" + sql "set show_hidden_columns = false;" + sql "set skip_delete_predicate = true;" + // 1,2,NULL,NULL,NULL + // 2,2,2,2,2 + qt_sql34 "select * from ${tableName2} order by k1;" + sql "set skip_delete_predicate = false;" + sql "set enable_unique_key_partial_update=true;" + sql "set enable_insert_strict=false;" + sql "insert into ${tableName2} (k1,c1) values(2,3)" + // 2,3,2,2,2 + qt_sql35 "select * from ${tableName2} order by k1;" + sql "set enable_unique_key_partial_update=false;" + sql "set enable_insert_strict=true;" + + sql "drop table if exists ${tableName2};" + } + } +} diff --git a/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_after_delete.groovy b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_after_delete.groovy new file mode 100644 index 00000000000000..41cebac66bd12c --- /dev/null +++ b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_after_delete.groovy @@ -0,0 +1,79 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_partial_update_after_delete", "p0") { + + String db = context.config.getDbNameByFile(context.file) + sql "select 1;" // to create database + + for (def use_row_store : [false, true]) { + logger.info("current params: use_row_store: ${use_row_store}") + + connect(user = context.config.jdbcUser, password = context.config.jdbcPassword, url = context.config.jdbcUrl) { + sql "use ${db};" + sql "SET enable_nereids_planner=true;" + sql "SET enable_fallback_to_original_planner=false;" + sql "set enable_unique_key_partial_update=false;" + sql "set enable_insert_strict=true;" + def tableName1 = "test_partial_update_after_delete1" + sql "DROP TABLE IF EXISTS ${tableName1};" + sql """ CREATE TABLE IF NOT EXISTS ${tableName1} ( + `k1` INT NULL, + `v1` INT NULL, + `v2` INT NULL + )UNIQUE KEY(k1) + DISTRIBUTED BY HASH(k1) BUCKETS 1 + PROPERTIES ( + "enable_unique_key_merge_on_write" = "true", + "disable_auto_compaction" = "true", + "replication_num" = "1", + "store_row_column" = "${use_row_store}"); """ + + sql "insert into ${tableName1} values(1,1,1);" + sql "delete from ${tableName1} where k1=1;" + sql "set enable_unique_key_partial_update=true;" + sql "set enable_insert_strict=false;" + sql "insert into ${tableName1}(k1, v1) values(1,2);" + qt_select1 "select * from ${tableName1};" + + sql "set enable_unique_key_partial_update=false;" + sql "set enable_insert_strict=true;" + sql "SET enable_nereids_planner=false;" + sql "SET enable_fallback_to_original_planner=false;" + def tableName2 = "test_partial_update_after_delete2" + sql "DROP TABLE IF EXISTS ${tableName2};" + sql """ CREATE TABLE IF NOT EXISTS ${tableName2} ( + `k1` INT NULL, + `v1` INT NULL, + `v2` INT NULL + )UNIQUE KEY(k1) + DISTRIBUTED BY HASH(k1) BUCKETS 1 + PROPERTIES ( + "enable_unique_key_merge_on_write" = "true", + "disable_auto_compaction" = "true", + "replication_num" = "1", + "store_row_column" = "${use_row_store}"); """ + + sql "insert into ${tableName2} values(1,1,1);" + sql "delete from ${tableName2} where k1=1;" + sql "set enable_unique_key_partial_update=true;" + sql "set enable_insert_strict=false;" + sql "insert into ${tableName2}(k1, v1) values(1,2);" + qt_select2 "select * from ${tableName2};" + } + } +} From 9f762ef2b36b309b62a45a4d5a5df57c17572011 Mon Sep 17 00:00:00 2001 From: Yukang-Lian Date: Mon, 29 Jul 2024 15:20:34 +0800 Subject: [PATCH 2/6] 2 --- .../apache/doris/analysis/ModifyTablePropertiesClause.java | 2 -- .../doris/nereids/trees/plans/commands/DeleteCommand.java | 4 ++-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ModifyTablePropertiesClause.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ModifyTablePropertiesClause.java index 870f30526ac7e4..63dfa87cf725ba 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ModifyTablePropertiesClause.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ModifyTablePropertiesClause.java @@ -18,9 +18,7 @@ package org.apache.doris.analysis; import org.apache.doris.alter.AlterOpType; -import org.apache.doris.catalog.DynamicPartitionProperty; import org.apache.doris.catalog.Env; -import org.apache.doris.catalog.MTMV; import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.ReplicaAllocation; import org.apache.doris.catalog.TableProperty; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/DeleteCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/DeleteCommand.java index 437c1929b106ce..ba148e9b177351 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/DeleteCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/DeleteCommand.java @@ -21,10 +21,10 @@ import org.apache.doris.catalog.KeysType; import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.TableIf; +import org.apache.doris.nereids.analyzer.UnboundAlias; import org.apache.doris.nereids.analyzer.UnboundOlapTableSink; import org.apache.doris.nereids.analyzer.UnboundSlot; import org.apache.doris.nereids.exceptions.AnalysisException; -import org.apache.doris.nereids.trees.expressions.Alias; import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.literal.TinyIntLiteral; import org.apache.doris.nereids.trees.plans.Explainable; @@ -119,7 +119,7 @@ public LogicalPlan completeQueryPlan(ConnectContext ctx, LogicalPlan logicalQuer // make UnboundTableSink return new UnboundOlapTableSink<>(nameParts, cols, ImmutableList.of(), - partitions, isPartialUpdate, logicalQuery); + partitions, false, logicalQuery); } public LogicalPlan getLogicalQuery() { From 09b92c3ee637cf6f3cd3b686abd6337b843b7978 Mon Sep 17 00:00:00 2001 From: abmdocrt Date: Fri, 5 Jul 2024 23:55:48 +0800 Subject: [PATCH 3/6] [Fix](paritial update) Fix the case of partial update failing in cloud mode (#37151) Problem: The `test_new_partial_update` case fails to run in cloud mode, but it passes in local mode. Reason: In PR #35917, we introduced a new table attribute `enable_light_delete`. When executing schema changes with `alter table set xxx` statements, the local and cloud modes process the logic differently. The cloud mode has its unique processing logic, which was not addressed in the mentioned PR, leading to failures in the cloud environment. Solution: To resolve the issue, we need to complete the missing schema change logic for the cloud mode. Once this is implemented, the problem should be resolved. --- .../java/org/apache/doris/common/Config.java | 2 +- .../doris/alter/SchemaChangeHandler.java | 10 +-- .../org/apache/doris/analysis/DeleteStmt.java | 2 +- .../analysis/ModifyTablePropertiesClause.java | 17 ++--- .../org/apache/doris/catalog/OlapTable.java | 8 +- .../apache/doris/catalog/TableProperty.java | 10 +-- .../doris/common/util/PropertyAnalyzer.java | 14 ++-- .../doris/datasource/InternalCatalog.java | 20 +++-- .../analysis/CreateTableAsSelectStmtTest.java | 2 +- .../test_new_partial_update_delete.groovy | 75 +++++++++++++++---- 10 files changed, 101 insertions(+), 59 deletions(-) diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java index 5fea51e764e418..9c299576537b7f 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java @@ -2402,7 +2402,7 @@ public class Config extends ConfigBase { + "DELETE statements, but partial column updates after a DELETE may result in erroneous data. " + "If disabled, it will reduce the performance of DELETE statements to ensure accuracy." }) - public static boolean enable_mow_delete_on_predicate = false; + public static boolean enable_mow_light_delete = false; @ConfField(description = { "是否开启 Proxy Protocol 支持", diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java index 0a1e72b8700085..9bf414ed6c93f3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java @@ -2215,7 +2215,7 @@ public void updateTableProperties(Database db, String tableName, Map p if (properties == null || properties.isEmpty()) { return false; } - String value = properties.get(PropertyAnalyzer.PROPERTIES_ENABLE_MOW_DELETE_ON_DELETE_PREDICATE); + String value = properties.get(PropertyAnalyzer.PROPERTIES_ENABLE_MOW_LIGHT_DELETE); if (value == null) { return false; } - properties.remove(PropertyAnalyzer.PROPERTIES_ENABLE_MOW_DELETE_ON_DELETE_PREDICATE); + properties.remove(PropertyAnalyzer.PROPERTIES_ENABLE_MOW_LIGHT_DELETE); if (value.equals("true")) { return true; } else if (value.equals("false")) { return false; } throw new AnalysisException( - PropertyAnalyzer.PROPERTIES_ENABLE_MOW_DELETE_ON_DELETE_PREDICATE + " must be `true` or `false`"); + PropertyAnalyzer.PROPERTIES_ENABLE_MOW_LIGHT_DELETE + " must be `true` or `false`"); } /** diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java index e6f1a43dc60e6a..285d60d331656b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java @@ -2204,18 +2204,16 @@ private boolean createOlapTable(Database db, CreateTableStmt stmt) throws UserEx olapTable.setEnableUniqueKeyMergeOnWrite(enableUniqueKeyMergeOnWrite); boolean enableDeleteOnDeletePredicate = false; - if (keysType == KeysType.UNIQUE_KEYS) { - try { - enableDeleteOnDeletePredicate = PropertyAnalyzer.analyzeEnableDeleteOnDeletePredicate(properties); - } catch (AnalysisException e) { - throw new DdlException(e.getMessage()); - } - if (enableDeleteOnDeletePredicate && !enableUniqueKeyMergeOnWrite) { - throw new DdlException(PropertyAnalyzer.PROPERTIES_ENABLE_MOW_DELETE_ON_DELETE_PREDICATE - + " property is not supported for unique merge-on-read table"); - } + try { + enableDeleteOnDeletePredicate = PropertyAnalyzer.analyzeEnableDeleteOnDeletePredicate(properties); + } catch (AnalysisException e) { + throw new DdlException(e.getMessage()); + } + if (enableDeleteOnDeletePredicate && !enableUniqueKeyMergeOnWrite) { + throw new DdlException(PropertyAnalyzer.PROPERTIES_ENABLE_MOW_LIGHT_DELETE + + " property is only supported for unique merge-on-write table"); } - olapTable.setEnableDeleteOnDeletePredicate(enableDeleteOnDeletePredicate); + olapTable.setEnableMowLightDelete(enableDeleteOnDeletePredicate); boolean enableSingleReplicaCompaction = false; try { diff --git a/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateTableAsSelectStmtTest.java b/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateTableAsSelectStmtTest.java index 6ffa8239b0ad89..663f1fe81dcf92 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateTableAsSelectStmtTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateTableAsSelectStmtTest.java @@ -504,7 +504,7 @@ public void testUseKeyType() throws Exception { + "\"light_schema_change\" = \"true\",\n" + "\"disable_auto_compaction\" = \"false\",\n" + "\"enable_single_replica_compaction\" = \"false\",\n" - + "\"enable_mow_delete_on_delete_predicate\" = \"false\"\n" + + "\"enable_mow_light_delete\" = \"false\"\n" + ");", showResultSet.getResultRows().get(0).get(1)); } diff --git a/regression-test/suites/unique_with_mow_p0/partial_update/test_new_partial_update_delete.groovy b/regression-test/suites/unique_with_mow_p0/partial_update/test_new_partial_update_delete.groovy index 85434f61987ea5..bcd8d5f2842f97 100644 --- a/regression-test/suites/unique_with_mow_p0/partial_update/test_new_partial_update_delete.groovy +++ b/regression-test/suites/unique_with_mow_p0/partial_update/test_new_partial_update_delete.groovy @@ -43,12 +43,12 @@ suite('test_new_partial_update_delete') { PROPERTIES ( "disable_auto_compaction" = "true", "enable_unique_key_merge_on_write" = "false", - "enable_mow_delete_on_delete_predicate" = "true", + "enable_mow_light_delete" = "true", "replication_num" = "1", "store_row_column" = "${use_row_store}"); """ } catch (Exception e) { log.info(e.getMessage()) - assertTrue(e.getMessage().contains('enable_mow_delete_on_delete_predicate property is not supported for unique merge-on-read table')) + assertTrue(e.getMessage().contains('enable_mow_light_delete property is only supported for unique merge-on-write table')) } try { @@ -67,10 +67,10 @@ suite('test_new_partial_update_delete') { "enable_unique_key_merge_on_write" = "false", "replication_num" = "1", "store_row_column" = "${use_row_store}"); """ - sql """alter table ${tableMorName2} set ("enable_mow_delete_on_delete_predicate"="true")""" + sql """alter table ${tableMorName2} set ("enable_mow_light_delete"="true")""" } catch (Exception e) { log.info(e.getMessage()) - assertTrue(e.getMessage().contains('enable_mow_delete_on_delete_predicate property is not supported for unique merge-on-read table')) + assertTrue(e.getMessage().contains('enable_mow_light_delete property is only supported for unique merge-on-write table')) } def tableName1 = "test_new_partial_update_delete1" @@ -88,7 +88,8 @@ suite('test_new_partial_update_delete') { "replication_num" = "1", "store_row_column" = "${use_row_store}"); """ - qt_sql_table1 "show create table ${tableName1}" + def output1 = sql "show create table ${tableName1}" + assertTrue output1[0][1].contains("\"enable_mow_light_delete\" = \"false\""); sql "insert into ${tableName1} values(1,1,1,1,1)" // 1,1,1,1,1 qt_sql1 "select * from ${tableName1} order by k1;" @@ -107,10 +108,11 @@ suite('test_new_partial_update_delete') { - sql """alter table ${tableName1} set ("enable_mow_delete_on_delete_predicate"="true") """ + sql """alter table ${tableName1} set ("enable_mow_light_delete"="true") """ sql "set enable_unique_key_partial_update=false;" sql "set enable_insert_strict=true;" - qt_sql_table2 "show create table ${tableName1}" + def output2 = sql "show create table ${tableName1}" + assertTrue output2[0][1].contains("\"enable_mow_light_delete\" = \"true\""); sql "insert into ${tableName1} values(2,2,2,2,2)" // 1,2,NULL,NULL,NULL // 2,2,2,2,2 @@ -154,12 +156,12 @@ suite('test_new_partial_update_delete') { PROPERTIES ( "disable_auto_compaction" = "true", "enable_unique_key_merge_on_write" = "false", - "enable_mow_delete_on_delete_predicate" = "true", + "enable_mow_light_delete" = "true", "replication_num" = "1", "store_row_column" = "${use_row_store}"); """ } catch (Exception e) { log.info(e.getMessage()) - assertTrue(e.getMessage().contains('enable_mow_delete_on_delete_predicate property is not supported for unique merge-on-read table')) + assertTrue(e.getMessage().contains('enable_mow_light_delete property is only supported for unique merge-on-write table')) } try { @@ -178,10 +180,10 @@ suite('test_new_partial_update_delete') { "enable_unique_key_merge_on_write" = "false", "replication_num" = "1", "store_row_column" = "${use_row_store}"); """ - sql """alter table ${tableMorName4} set ("enable_mow_delete_on_delete_predicate"="true")""" + sql """alter table ${tableMorName4} set ("enable_mow_light_delete"="true")""" } catch (Exception e) { log.info(e.getMessage()) - assertTrue(e.getMessage().contains('enable_mow_delete_on_delete_predicate property is not supported for unique merge-on-read table')) + assertTrue(e.getMessage().contains('enable_mow_light_delete property is only supported for unique merge-on-write table')) } sql "set enable_nereids_planner=false" def tableName2 = "test_new_partial_update_delete2" @@ -199,7 +201,8 @@ suite('test_new_partial_update_delete') { "replication_num" = "1", "store_row_column" = "${use_row_store}"); """ - qt_sql_table21 "show create table ${tableName2}" + def output3 = sql "show create table ${tableName2}" + assertTrue output3[0][1].contains("\"enable_mow_light_delete\" = \"false\""); sql "insert into ${tableName2} values(1,1,1,1,1)" // 1,1,1,1,1 qt_sql21 "select * from ${tableName2} order by k1;" @@ -218,10 +221,11 @@ suite('test_new_partial_update_delete') { - sql """alter table ${tableName2} set ("enable_mow_delete_on_delete_predicate"="true") """ + sql """alter table ${tableName2} set ("enable_mow_light_delete"="true") """ sql "set enable_unique_key_partial_update=false;" sql "set enable_insert_strict=true;" - qt_sql_table32 "show create table ${tableName2}" + def output4 = sql "show create table ${tableName2}" + assertTrue output4[0][1].contains("\"enable_mow_light_delete\" = \"true\""); sql "insert into ${tableName2} values(2,2,2,2,2)" // 1,2,NULL,NULL,NULL // 2,2,2,2,2 @@ -249,4 +253,47 @@ suite('test_new_partial_update_delete') { sql "drop table if exists ${tableName2};" } } + + connect(user = context.config.jdbcUser, password = context.config.jdbcPassword, url = context.config.jdbcUrl) { + sql "use ${db};" + try { + def tableAggName = "test_new_partial_update_agg_delete" + sql "DROP TABLE IF EXISTS ${tableAggName};" + sql """ CREATE TABLE IF NOT EXISTS ${tableAggName} ( + `k1` int NOT NULL, + `c1` int replace, + `c2` int replace, + `c3` int replace, + `c4` int replace + )AGGREGATE KEY(k1) + DISTRIBUTED BY HASH(k1) BUCKETS 1 + PROPERTIES ( + "disable_auto_compaction" = "true", + "enable_mow_light_delete" = "true", + "replication_num" = "1"); """ + } catch (Exception e) { + log.info(e.getMessage()) + assertTrue(e.getMessage().contains('enable_mow_light_delete property is only supported for unique merge-on-write table')) + } + + try { + def tableDupName = "test_new_partial_update_dup_delete" + sql "DROP TABLE IF EXISTS ${tableDupName};" + sql """ CREATE TABLE IF NOT EXISTS ${tableDupName} ( + `k1` int NOT NULL, + `c1` int, + `c2` int, + `c3` int, + `c4` int + )DUPLICATE KEY(k1) + DISTRIBUTED BY HASH(k1) BUCKETS 1 + PROPERTIES ( + "disable_auto_compaction" = "true", + "enable_mow_light_delete" = "true", + "replication_num" = "1"); """ + } catch (Exception e) { + log.info(e.getMessage()) + assertTrue(e.getMessage().contains('enable_mow_light_delete property is only supported for unique merge-on-write table')) + } + } } From bfd6ea446a49cc2b4ae39e8bf84b384714dfbd62 Mon Sep 17 00:00:00 2001 From: Yukang-Lian Date: Mon, 29 Jul 2024 16:16:14 +0800 Subject: [PATCH 4/6] 4 --- .../org/apache/doris/analysis/CreateTableAsSelectStmtTest.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateTableAsSelectStmtTest.java b/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateTableAsSelectStmtTest.java index 663f1fe81dcf92..fa3a029c43bfc9 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateTableAsSelectStmtTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateTableAsSelectStmtTest.java @@ -503,8 +503,7 @@ public void testUseKeyType() throws Exception { + "\"storage_format\" = \"V2\",\n" + "\"light_schema_change\" = \"true\",\n" + "\"disable_auto_compaction\" = \"false\",\n" - + "\"enable_single_replica_compaction\" = \"false\",\n" - + "\"enable_mow_light_delete\" = \"false\"\n" + + "\"enable_single_replica_compaction\" = \"false\"\n" + ");", showResultSet.getResultRows().get(0).get(1)); } From f635113c6c73054cb0d6d88272357a02c052f7bf Mon Sep 17 00:00:00 2001 From: Yukang-Lian Date: Tue, 30 Jul 2024 18:04:46 +0800 Subject: [PATCH 5/6] 5 --- .../java/org/apache/doris/alter/Alter.java | 1 + .../java/org/apache/doris/catalog/Env.java | 4 +++ .../trees/plans/commands/DeleteCommand.java | 5 +++- .../trees/plans/DeleteCommandTest.java | 20 ++++++------- .../data/compaction/test_full_compaction.out | 4 +-- .../test_full_compaction_by_table_id.out | 4 +-- .../data/delete_p0/test_delete_on_value.out | 14 +++++----- .../explain/test_pushdown_explain.out | 5 +--- .../test_new_partial_update_delete.out | 28 ++----------------- .../test_partial_update_delete.out | 12 ++++---- .../test_new_partial_update_delete.groovy | 4 ++- 11 files changed, 41 insertions(+), 60 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java b/fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java index 36f82f524a196e..a40f0891b0bf04 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java @@ -513,6 +513,7 @@ public void processAlterTable(AlterTableStmt stmt) throws UserException { // currently, only in memory and storage policy property could reach here Preconditions.checkState(properties.containsKey(PropertyAnalyzer.PROPERTIES_INMEMORY) || properties.containsKey(PropertyAnalyzer.PROPERTIES_STORAGE_POLICY) + || properties.containsKey(PropertyAnalyzer.PROPERTIES_ENABLE_MOW_LIGHT_DELETE) || properties.containsKey(PropertyAnalyzer.PROPERTIES_IS_BEING_SYNCED) || properties.containsKey(PropertyAnalyzer.PROPERTIES_COMPACTION_POLICY) || properties.containsKey(PropertyAnalyzer.PROPERTIES_TIME_SERIES_COMPACTION_GOAL_SIZE_MBYTES) diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java index b496c880bcea59..1413b96aaa20e5 100755 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java @@ -3304,6 +3304,10 @@ public static void getDdlStmt(DdlStmt ddlStmt, String dbName, TableIf table, Lis sb.append(olapTable.isDuplicateWithoutKey()).append("\""); } + // enable mow light delete + sb.append(",\n\"").append(PropertyAnalyzer.PROPERTIES_ENABLE_MOW_LIGHT_DELETE).append("\" = \""); + sb.append(olapTable.getEnableMowLightDelete()).append("\""); + sb.append("\n)"); } else if (table.getType() == TableType.MYSQL) { MysqlTable mysqlTable = (MysqlTable) table; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/DeleteCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/DeleteCommand.java index ba148e9b177351..b356a6db5d8aad 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/DeleteCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/DeleteCommand.java @@ -117,9 +117,12 @@ public LogicalPlan completeQueryPlan(ConnectContext ctx, LogicalPlan logicalQuer logicalQuery = ((LogicalPlan) cte.get().withChildren(logicalQuery)); } + boolean isPartialUpdate = targetTable.getEnableUniqueKeyMergeOnWrite() + && cols.size() < targetTable.getColumns().size(); + // make UnboundTableSink return new UnboundOlapTableSink<>(nameParts, cols, ImmutableList.of(), - partitions, false, logicalQuery); + partitions, isPartialUpdate, logicalQuery); } public LogicalPlan getLogicalQuery() { diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/DeleteCommandTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/DeleteCommandTest.java index 23a89ffe0415aa..cee3b0a4242966 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/DeleteCommandTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/DeleteCommandTest.java @@ -83,17 +83,15 @@ public void testFromClauseDelete() throws AnalysisException { logicalOlapTableSink( logicalProject( logicalJoin( - logicalProject( - logicalJoin( - logicalProject( - logicalFilter( - logicalOlapScan() - ) - ), - logicalProject( - logicalFilter( - logicalOlapScan() - ) + logicalJoin( + logicalProject( + logicalFilter( + logicalOlapScan() + ) + ), + logicalProject( + logicalFilter( + logicalOlapScan() ) ) ), diff --git a/regression-test/data/compaction/test_full_compaction.out b/regression-test/data/compaction/test_full_compaction.out index 7098be6d89d2e6..b25fdad93145cc 100644 --- a/regression-test/data/compaction/test_full_compaction.out +++ b/regression-test/data/compaction/test_full_compaction.out @@ -32,12 +32,12 @@ 2 2 2 20 2 200 -3 100 +3 0 3 100 3 300 -- !select_final -- 1 100 2 200 -3 100 +3 0 diff --git a/regression-test/data/compaction/test_full_compaction_by_table_id.out b/regression-test/data/compaction/test_full_compaction_by_table_id.out index 7098be6d89d2e6..b25fdad93145cc 100644 --- a/regression-test/data/compaction/test_full_compaction_by_table_id.out +++ b/regression-test/data/compaction/test_full_compaction_by_table_id.out @@ -32,12 +32,12 @@ 2 2 2 20 2 200 -3 100 +3 0 3 100 3 300 -- !select_final -- 1 100 2 200 -3 100 +3 0 diff --git a/regression-test/data/delete_p0/test_delete_on_value.out b/regression-test/data/delete_p0/test_delete_on_value.out index 4a6e652aef3887..2b50d5bd9e1ba7 100644 --- a/regression-test/data/delete_p0/test_delete_on_value.out +++ b/regression-test/data/delete_p0/test_delete_on_value.out @@ -50,19 +50,19 @@ -- !sql -- 1 1 1 0 2 2 2 0 +3 \N \N 1 3 3 3 0 -3 3 3 1 +4 \N \N 1 4 4 4 0 4 4 4 0 -4 4 4 1 +5 \N \N 1 +5 \N \N 1 5 5 5 0 5 5 5 0 -5 5 5 1 -5 5 5 1 +6 \N \N 1 6 6 6 0 -6 6 6 1 +7 \N \N 1 7 7 7 0 -7 7 7 1 8 8 8 0 9 9 9 0 @@ -90,7 +90,7 @@ -- !sql -- -- !sql -- +1 \N \N 1 4 10 1 1 5 0 3 5 1 1 10 0 2 10 -1 1 10 1 4 10 diff --git a/regression-test/data/nereids_p0/explain/test_pushdown_explain.out b/regression-test/data/nereids_p0/explain/test_pushdown_explain.out index fe2402b60b8481..eaac687e2d5176 100644 --- a/regression-test/data/nereids_p0/explain/test_pushdown_explain.out +++ b/regression-test/data/nereids_p0/explain/test_pushdown_explain.out @@ -96,10 +96,7 @@ zzz asd -- !select_16 -- -qwe - --- !select_17 -- -cc +qdf -- !select_18 -- zzz diff --git a/regression-test/data/unique_with_mow_p0/partial_update/test_new_partial_update_delete.out b/regression-test/data/unique_with_mow_p0/partial_update/test_new_partial_update_delete.out index a248196690e007..8599b0b7fa1dca 100644 --- a/regression-test/data/unique_with_mow_p0/partial_update/test_new_partial_update_delete.out +++ b/regression-test/data/unique_with_mow_p0/partial_update/test_new_partial_update_delete.out @@ -1,21 +1,15 @@ -- This file is automatically generated. You should know what you did if you want to edit this --- !sql_table1 -- -test_new_partial_update_delete1 CREATE TABLE `test_new_partial_update_delete1` (\n `k1` INT NOT NULL,\n `c1` INT NULL,\n `c2` INT NULL,\n `c3` INT NULL,\n `c4` INT NULL\n) ENGINE=OLAP\nUNIQUE KEY(`k1`)\nDISTRIBUTED BY HASH(`k1`) BUCKETS 1\nPROPERTIES (\n"replication_allocation" = "tag.location.default: 1",\n"min_load_replica_num" = "-1",\n"is_being_synced" = "false",\n"storage_medium" = "hdd",\n"storage_format" = "V2",\n"inverted_index_storage_format" = "V2",\n"enable_unique_key_merge_on_write" = "true",\n"light_schema_change" = "true",\n"disable_auto_compaction" = "true",\n"binlog.enable" = "false",\n"binlog.ttl_seconds" = "86400",\n"binlog.max_bytes" = "9223372036854775807",\n"binlog.max_history_nums" = "9223372036854775807",\n"enable_single_replica_compaction" = "false",\n"group_commit_interval_ms" = "10000",\n"group_commit_data_bytes" = "134217728",\n"enable_mow_delete_on_delete_predicate" = "false"\n); - -- !sql1 -- 1 1 1 1 1 -- !sql2 -- -- !sql3 -- -1 1 1 1 1 1 +1 0 \N \N \N 1 -- !sql4 -- 1 2 \N \N \N --- !sql_table2 -- -test_new_partial_update_delete1 CREATE TABLE `test_new_partial_update_delete1` (\n `k1` INT NOT NULL,\n `c1` INT NULL,\n `c2` INT NULL,\n `c3` INT NULL,\n `c4` INT NULL\n) ENGINE=OLAP\nUNIQUE KEY(`k1`)\nDISTRIBUTED BY HASH(`k1`) BUCKETS 1\nPROPERTIES (\n"replication_allocation" = "tag.location.default: 1",\n"min_load_replica_num" = "-1",\n"is_being_synced" = "false",\n"storage_medium" = "hdd",\n"storage_format" = "V2",\n"inverted_index_storage_format" = "V2",\n"enable_unique_key_merge_on_write" = "true",\n"light_schema_change" = "true",\n"disable_auto_compaction" = "true",\n"binlog.enable" = "false",\n"binlog.ttl_seconds" = "86400",\n"binlog.max_bytes" = "9223372036854775807",\n"binlog.max_history_nums" = "9223372036854775807",\n"enable_single_replica_compaction" = "false",\n"group_commit_interval_ms" = "10000",\n"group_commit_data_bytes" = "134217728",\n"enable_mow_delete_on_delete_predicate" = "true"\n); - -- !sql11 -- 1 2 \N \N \N 2 2 2 2 2 @@ -31,9 +25,6 @@ test_new_partial_update_delete1 CREATE TABLE `test_new_partial_update_delete1` ( -- !sql15 -- 2 3 2 2 2 --- !sql_table21 -- -test_new_partial_update_delete2 CREATE TABLE `test_new_partial_update_delete2` (\n `k1` INT NOT NULL,\n `c1` INT NULL,\n `c2` INT NULL,\n `c3` INT NULL,\n `c4` INT NULL\n) ENGINE=OLAP\nUNIQUE KEY(`k1`)\nDISTRIBUTED BY HASH(`k1`) BUCKETS 1\nPROPERTIES (\n"replication_allocation" = "tag.location.default: 1",\n"min_load_replica_num" = "-1",\n"is_being_synced" = "false",\n"storage_medium" = "hdd",\n"storage_format" = "V2",\n"inverted_index_storage_format" = "V2",\n"enable_unique_key_merge_on_write" = "true",\n"light_schema_change" = "true",\n"disable_auto_compaction" = "true",\n"binlog.enable" = "false",\n"binlog.ttl_seconds" = "86400",\n"binlog.max_bytes" = "9223372036854775807",\n"binlog.max_history_nums" = "9223372036854775807",\n"enable_single_replica_compaction" = "false",\n"group_commit_interval_ms" = "10000",\n"group_commit_data_bytes" = "134217728",\n"enable_mow_delete_on_delete_predicate" = "false"\n); - -- !sql21 -- 1 1 1 1 1 @@ -45,9 +36,6 @@ test_new_partial_update_delete2 CREATE TABLE `test_new_partial_update_delete2` ( -- !sql24 -- 1 2 \N \N \N --- !sql_table32 -- -test_new_partial_update_delete2 CREATE TABLE `test_new_partial_update_delete2` (\n `k1` INT NOT NULL,\n `c1` INT NULL,\n `c2` INT NULL,\n `c3` INT NULL,\n `c4` INT NULL\n) ENGINE=OLAP\nUNIQUE KEY(`k1`)\nDISTRIBUTED BY HASH(`k1`) BUCKETS 1\nPROPERTIES (\n"replication_allocation" = "tag.location.default: 1",\n"min_load_replica_num" = "-1",\n"is_being_synced" = "false",\n"storage_medium" = "hdd",\n"storage_format" = "V2",\n"inverted_index_storage_format" = "V2",\n"enable_unique_key_merge_on_write" = "true",\n"light_schema_change" = "true",\n"disable_auto_compaction" = "true",\n"binlog.enable" = "false",\n"binlog.ttl_seconds" = "86400",\n"binlog.max_bytes" = "9223372036854775807",\n"binlog.max_history_nums" = "9223372036854775807",\n"enable_single_replica_compaction" = "false",\n"group_commit_interval_ms" = "10000",\n"group_commit_data_bytes" = "134217728",\n"enable_mow_delete_on_delete_predicate" = "true"\n); - -- !sql31 -- 1 2 \N \N \N 2 2 2 2 2 @@ -63,23 +51,17 @@ test_new_partial_update_delete2 CREATE TABLE `test_new_partial_update_delete2` ( -- !sql35 -- 2 3 2 2 2 --- !sql_table1 -- -test_new_partial_update_delete1 CREATE TABLE `test_new_partial_update_delete1` (\n `k1` INT NOT NULL,\n `c1` INT NULL,\n `c2` INT NULL,\n `c3` INT NULL,\n `c4` INT NULL\n) ENGINE=OLAP\nUNIQUE KEY(`k1`)\nDISTRIBUTED BY HASH(`k1`) BUCKETS 1\nPROPERTIES (\n"replication_allocation" = "tag.location.default: 1",\n"min_load_replica_num" = "-1",\n"is_being_synced" = "false",\n"storage_medium" = "hdd",\n"storage_format" = "V2",\n"inverted_index_storage_format" = "V2",\n"enable_unique_key_merge_on_write" = "true",\n"light_schema_change" = "true",\n"store_row_column" = "true",\n"disable_auto_compaction" = "true",\n"binlog.enable" = "false",\n"binlog.ttl_seconds" = "86400",\n"binlog.max_bytes" = "9223372036854775807",\n"binlog.max_history_nums" = "9223372036854775807",\n"enable_single_replica_compaction" = "false",\n"group_commit_interval_ms" = "10000",\n"group_commit_data_bytes" = "134217728",\n"enable_mow_delete_on_delete_predicate" = "false"\n); - -- !sql1 -- 1 1 1 1 1 -- !sql2 -- -- !sql3 -- -1 1 1 1 1 1 +1 0 \N \N \N 1 -- !sql4 -- 1 2 \N \N \N --- !sql_table2 -- -test_new_partial_update_delete1 CREATE TABLE `test_new_partial_update_delete1` (\n `k1` INT NOT NULL,\n `c1` INT NULL,\n `c2` INT NULL,\n `c3` INT NULL,\n `c4` INT NULL\n) ENGINE=OLAP\nUNIQUE KEY(`k1`)\nDISTRIBUTED BY HASH(`k1`) BUCKETS 1\nPROPERTIES (\n"replication_allocation" = "tag.location.default: 1",\n"min_load_replica_num" = "-1",\n"is_being_synced" = "false",\n"storage_medium" = "hdd",\n"storage_format" = "V2",\n"inverted_index_storage_format" = "V2",\n"enable_unique_key_merge_on_write" = "true",\n"light_schema_change" = "true",\n"store_row_column" = "true",\n"disable_auto_compaction" = "true",\n"binlog.enable" = "false",\n"binlog.ttl_seconds" = "86400",\n"binlog.max_bytes" = "9223372036854775807",\n"binlog.max_history_nums" = "9223372036854775807",\n"enable_single_replica_compaction" = "false",\n"group_commit_interval_ms" = "10000",\n"group_commit_data_bytes" = "134217728",\n"enable_mow_delete_on_delete_predicate" = "true"\n); - -- !sql11 -- 1 2 \N \N \N 2 2 2 2 2 @@ -95,9 +77,6 @@ test_new_partial_update_delete1 CREATE TABLE `test_new_partial_update_delete1` ( -- !sql15 -- 2 3 2 2 2 --- !sql_table21 -- -test_new_partial_update_delete2 CREATE TABLE `test_new_partial_update_delete2` (\n `k1` INT NOT NULL,\n `c1` INT NULL,\n `c2` INT NULL,\n `c3` INT NULL,\n `c4` INT NULL\n) ENGINE=OLAP\nUNIQUE KEY(`k1`)\nDISTRIBUTED BY HASH(`k1`) BUCKETS 1\nPROPERTIES (\n"replication_allocation" = "tag.location.default: 1",\n"min_load_replica_num" = "-1",\n"is_being_synced" = "false",\n"storage_medium" = "hdd",\n"storage_format" = "V2",\n"inverted_index_storage_format" = "V2",\n"enable_unique_key_merge_on_write" = "true",\n"light_schema_change" = "true",\n"store_row_column" = "true",\n"disable_auto_compaction" = "true",\n"binlog.enable" = "false",\n"binlog.ttl_seconds" = "86400",\n"binlog.max_bytes" = "9223372036854775807",\n"binlog.max_history_nums" = "9223372036854775807",\n"enable_single_replica_compaction" = "false",\n"group_commit_interval_ms" = "10000",\n"group_commit_data_bytes" = "134217728",\n"enable_mow_delete_on_delete_predicate" = "false"\n); - -- !sql21 -- 1 1 1 1 1 @@ -109,9 +88,6 @@ test_new_partial_update_delete2 CREATE TABLE `test_new_partial_update_delete2` ( -- !sql24 -- 1 2 \N \N \N --- !sql_table32 -- -test_new_partial_update_delete2 CREATE TABLE `test_new_partial_update_delete2` (\n `k1` INT NOT NULL,\n `c1` INT NULL,\n `c2` INT NULL,\n `c3` INT NULL,\n `c4` INT NULL\n) ENGINE=OLAP\nUNIQUE KEY(`k1`)\nDISTRIBUTED BY HASH(`k1`) BUCKETS 1\nPROPERTIES (\n"replication_allocation" = "tag.location.default: 1",\n"min_load_replica_num" = "-1",\n"is_being_synced" = "false",\n"storage_medium" = "hdd",\n"storage_format" = "V2",\n"inverted_index_storage_format" = "V2",\n"enable_unique_key_merge_on_write" = "true",\n"light_schema_change" = "true",\n"store_row_column" = "true",\n"disable_auto_compaction" = "true",\n"binlog.enable" = "false",\n"binlog.ttl_seconds" = "86400",\n"binlog.max_bytes" = "9223372036854775807",\n"binlog.max_history_nums" = "9223372036854775807",\n"enable_single_replica_compaction" = "false",\n"group_commit_interval_ms" = "10000",\n"group_commit_data_bytes" = "134217728",\n"enable_mow_delete_on_delete_predicate" = "true"\n); - -- !sql31 -- 1 2 \N \N \N 2 2 2 2 2 diff --git a/regression-test/data/unique_with_mow_p0/partial_update/test_partial_update_delete.out b/regression-test/data/unique_with_mow_p0/partial_update/test_partial_update_delete.out index c16e954d73309c..0863afd7931780 100644 --- a/regression-test/data/unique_with_mow_p0/partial_update/test_partial_update_delete.out +++ b/regression-test/data/unique_with_mow_p0/partial_update/test_partial_update_delete.out @@ -11,12 +11,12 @@ 5 5 5 5 5 -- !with_delete_sign -- +1 \N \N 0 \N 1 1 1 1 1 1 0 -1 1 1 1 1 1 +2 \N \N 0 \N 1 2 2 2 2 2 0 -2 2 2 2 2 1 +3 \N \N 0 \N 1 3 3 3 3 3 0 -3 3 3 3 3 1 4 4 4 4 4 0 5 5 5 5 5 0 @@ -53,12 +53,12 @@ 5 5 5 5 5 -- !with_delete_sign -- +1 \N \N 0 \N 1 1 1 1 1 1 0 -1 1 1 1 1 1 +2 \N \N 0 \N 1 2 2 2 2 2 0 -2 2 2 2 2 1 +3 \N \N 0 \N 1 3 3 3 3 3 0 -3 3 3 3 3 1 4 4 4 4 4 0 5 5 5 5 5 0 diff --git a/regression-test/suites/unique_with_mow_p0/partial_update/test_new_partial_update_delete.groovy b/regression-test/suites/unique_with_mow_p0/partial_update/test_new_partial_update_delete.groovy index bcd8d5f2842f97..cd8a77fad2e764 100644 --- a/regression-test/suites/unique_with_mow_p0/partial_update/test_new_partial_update_delete.groovy +++ b/regression-test/suites/unique_with_mow_p0/partial_update/test_new_partial_update_delete.groovy @@ -77,7 +77,7 @@ suite('test_new_partial_update_delete') { sql "DROP TABLE IF EXISTS ${tableName1};" sql """ CREATE TABLE IF NOT EXISTS ${tableName1} ( `k1` int NOT NULL, - `c1` int, + `c1` int NOT NULL, `c2` int, `c3` int, `c4` int @@ -85,6 +85,7 @@ suite('test_new_partial_update_delete') { DISTRIBUTED BY HASH(k1) BUCKETS 1 PROPERTIES ( "disable_auto_compaction" = "true", + "enable_unique_key_merge_on_write" = "true", "replication_num" = "1", "store_row_column" = "${use_row_store}"); """ @@ -199,6 +200,7 @@ suite('test_new_partial_update_delete') { PROPERTIES ( "disable_auto_compaction" = "true", "replication_num" = "1", + "enable_unique_key_merge_on_write" = "true", "store_row_column" = "${use_row_store}"); """ def output3 = sql "show create table ${tableName2}" From 36dd48d16fef68a7f882719725d48b68f781c1fb Mon Sep 17 00:00:00 2001 From: Yukang-Lian Date: Tue, 30 Jul 2024 20:31:11 +0800 Subject: [PATCH 6/6] 6 --- .../analysis/CreateTableAsSelectStmtTest.java | 63 ++++++++++++------- .../data/index_p0/test_bitmap_index.out | 3 - .../test_show_create_table_and_views.out | 8 +-- .../suites/index_p0/test_bitmap_index.groovy | 2 +- 4 files changed, 47 insertions(+), 29 deletions(-) diff --git a/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateTableAsSelectStmtTest.java b/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateTableAsSelectStmtTest.java index fa3a029c43bfc9..7ce0fdd64f3be1 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateTableAsSelectStmtTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateTableAsSelectStmtTest.java @@ -96,7 +96,8 @@ public void testDecimal() throws Exception { + "\"storage_format\" = \"V2\",\n" + "\"light_schema_change\" = \"true\",\n" + "\"disable_auto_compaction\" = \"false\",\n" - + "\"enable_single_replica_compaction\" = \"false\"\n" + + "\"enable_single_replica_compaction\" = \"false\",\n" + + "\"enable_mow_light_delete\" = \"false\"\n" + ");", showCreateTableByName("select_decimal_table").getResultRows().get(0).get(1)); String selectFromDecimal1 = @@ -117,7 +118,8 @@ public void testDecimal() throws Exception { + "\"storage_format\" = \"V2\",\n" + "\"light_schema_change\" = \"true\",\n" + "\"disable_auto_compaction\" = \"false\",\n" - + "\"enable_single_replica_compaction\" = \"false\"\n" + + "\"enable_single_replica_compaction\" = \"false\",\n" + + "\"enable_mow_light_delete\" = \"false\"\n" + ");", showCreateTableByName("select_decimal_table_1").getResultRows().get(0).get(1)); } else { @@ -134,7 +136,8 @@ public void testDecimal() throws Exception { + "\"storage_format\" = \"V2\",\n" + "\"light_schema_change\" = \"true\",\n" + "\"disable_auto_compaction\" = \"false\",\n" - + "\"enable_single_replica_compaction\" = \"false\"\n" + + "\"enable_single_replica_compaction\" = \"false\",\n" + + "\"enable_mow_light_delete\" = \"false\"\n" + ");", showCreateTableByName("select_decimal_table_1").getResultRows().get(0).get(1)); } @@ -169,7 +172,8 @@ public void testVarchar() throws Exception { + "\"storage_format\" = \"V2\",\n" + "\"light_schema_change\" = \"true\",\n" + "\"disable_auto_compaction\" = \"false\",\n" - + "\"enable_single_replica_compaction\" = \"false\"\n" + + "\"enable_single_replica_compaction\" = \"false\",\n" + + "\"enable_mow_light_delete\" = \"false\"\n" + ");", showResultSet.getResultRows().get(0).get(1)); } @@ -193,7 +197,8 @@ public void testFunction() throws Exception { + "\"storage_format\" = \"V2\",\n" + "\"light_schema_change\" = \"true\",\n" + "\"disable_auto_compaction\" = \"false\",\n" - + "\"enable_single_replica_compaction\" = \"false\"\n" + + "\"enable_single_replica_compaction\" = \"false\",\n" + + "\"enable_mow_light_delete\" = \"false\"\n" + ");", showResultSet1.getResultRows().get(0).get(1)); @@ -219,7 +224,8 @@ public void testFunction() throws Exception { + "\"storage_format\" = \"V2\",\n" + "\"light_schema_change\" = \"true\",\n" + "\"disable_auto_compaction\" = \"false\",\n" - + "\"enable_single_replica_compaction\" = \"false\"\n" + + "\"enable_single_replica_compaction\" = \"false\",\n" + + "\"enable_mow_light_delete\" = \"false\"\n" + ");", showResultSet2.getResultRows().get(0).get(1)); } @@ -242,7 +248,8 @@ public void testAlias() throws Exception { + "\"storage_format\" = \"V2\",\n" + "\"light_schema_change\" = \"true\",\n" + "\"disable_auto_compaction\" = \"false\",\n" - + "\"enable_single_replica_compaction\" = \"false\"\n" + + "\"enable_single_replica_compaction\" = \"false\",\n" + + "\"enable_mow_light_delete\" = \"false\"\n" + ");", showResultSet1.getResultRows().get(0).get(1)); String selectAlias2 = "create table `test`.`select_alias_2` PROPERTIES(\"replication_num\" = \"1\") " + "as select userId as alias_name, username from `test`.`varchar_table`"; @@ -261,7 +268,8 @@ public void testAlias() throws Exception { + "\"storage_format\" = \"V2\",\n" + "\"light_schema_change\" = \"true\",\n" + "\"disable_auto_compaction\" = \"false\",\n" - + "\"enable_single_replica_compaction\" = \"false\"\n" + + "\"enable_single_replica_compaction\" = \"false\",\n" + + "\"enable_mow_light_delete\" = \"false\"\n" + ");", showResultSet2.getResultRows().get(0).get(1)); } @@ -287,7 +295,8 @@ public void testJoin() throws Exception { + "\"storage_format\" = \"V2\",\n" + "\"light_schema_change\" = \"true\",\n" + "\"disable_auto_compaction\" = \"false\",\n" - + "\"enable_single_replica_compaction\" = \"false\"\n" + + "\"enable_single_replica_compaction\" = \"false\",\n" + + "\"enable_mow_light_delete\" = \"false\"\n" + ");", showResultSet.getResultRows().get(0).get(1)); String selectFromJoin1 = "create table `test`.`select_join1` PROPERTIES(\"replication_num\" = \"1\") " @@ -310,7 +319,8 @@ public void testJoin() throws Exception { + "\"storage_format\" = \"V2\",\n" + "\"light_schema_change\" = \"true\",\n" + "\"disable_auto_compaction\" = \"false\",\n" - + "\"enable_single_replica_compaction\" = \"false\"\n" + + "\"enable_single_replica_compaction\" = \"false\",\n" + + "\"enable_mow_light_delete\" = \"false\"\n" + ");", showResultSet1.getResultRows().get(0).get(1)); } @@ -337,7 +347,8 @@ public void testName() throws Exception { + "\"storage_format\" = \"V2\",\n" + "\"light_schema_change\" = \"true\",\n" + "\"disable_auto_compaction\" = \"false\",\n" - + "\"enable_single_replica_compaction\" = \"false\"\n" + + "\"enable_single_replica_compaction\" = \"false\",\n" + + "\"enable_mow_light_delete\" = \"false\"\n" + ");", showResultSet.getResultRows().get(0).get(1)); } @@ -361,7 +372,8 @@ public void testUnion() throws Exception { + "\"storage_format\" = \"V2\",\n" + "\"light_schema_change\" = \"true\",\n" + "\"disable_auto_compaction\" = \"false\",\n" - + "\"enable_single_replica_compaction\" = \"false\"\n" + + "\"enable_single_replica_compaction\" = \"false\",\n" + + "\"enable_mow_light_delete\" = \"false\"\n" + ");", showResultSet.getResultRows().get(0).get(1)); } @@ -384,7 +396,8 @@ public void testCte() throws Exception { + "\"storage_format\" = \"V2\",\n" + "\"light_schema_change\" = \"true\",\n" + "\"disable_auto_compaction\" = \"false\",\n" - + "\"enable_single_replica_compaction\" = \"false\"\n" + + "\"enable_single_replica_compaction\" = \"false\",\n" + + "\"enable_mow_light_delete\" = \"false\"\n" + ");", showResultSet.getResultRows().get(0).get(1)); String selectFromCteAndUnion = "create table `test`.`select_cte_union` PROPERTIES(\"replication_num\" = \"1\")" @@ -403,7 +416,8 @@ public void testCte() throws Exception { + "\"storage_format\" = \"V2\",\n" + "\"light_schema_change\" = \"true\",\n" + "\"disable_auto_compaction\" = \"false\",\n" - + "\"enable_single_replica_compaction\" = \"false\"\n" + + "\"enable_single_replica_compaction\" = \"false\",\n" + + "\"enable_mow_light_delete\" = \"false\"\n" + ");", showResultSet1.getResultRows().get(0).get(1)); } @@ -429,7 +443,8 @@ public void testPartition() throws Exception { + "\"storage_format\" = \"V2\",\n" + "\"light_schema_change\" = \"true\",\n" + "\"disable_auto_compaction\" = \"false\",\n" - + "\"enable_single_replica_compaction\" = \"false\"\n" + + "\"enable_single_replica_compaction\" = \"false\",\n" + + "\"enable_mow_light_delete\" = \"false\"\n" + ");", showResultSet.getResultRows().get(0).get(1)); } @@ -454,7 +469,8 @@ public void testDefaultTimestamp() throws Exception { + "\"storage_format\" = \"V2\",\n" + "\"light_schema_change\" = \"true\",\n" + "\"disable_auto_compaction\" = \"false\",\n" - + "\"enable_single_replica_compaction\" = \"false\"\n" + + "\"enable_single_replica_compaction\" = \"false\",\n" + + "\"enable_mow_light_delete\" = \"false\"\n" + ");", showResultSet.getResultRows().get(0).get(1)); } @@ -478,7 +494,8 @@ public void testAggValue() throws Exception { + "\"storage_format\" = \"V2\",\n" + "\"light_schema_change\" = \"true\",\n" + "\"disable_auto_compaction\" = \"false\",\n" - + "\"enable_single_replica_compaction\" = \"false\"\n" + + "\"enable_single_replica_compaction\" = \"false\",\n" + + "\"enable_mow_light_delete\" = \"false\"\n" + ");", showResultSet.getResultRows().get(0).get(1)); } @@ -503,7 +520,8 @@ public void testUseKeyType() throws Exception { + "\"storage_format\" = \"V2\",\n" + "\"light_schema_change\" = \"true\",\n" + "\"disable_auto_compaction\" = \"false\",\n" - + "\"enable_single_replica_compaction\" = \"false\"\n" + + "\"enable_single_replica_compaction\" = \"false\",\n" + + "\"enable_mow_light_delete\" = \"false\"\n" + ");", showResultSet.getResultRows().get(0).get(1)); } @@ -552,7 +570,8 @@ public void testQuerySchema() throws Exception { + "\"storage_format\" = \"V2\",\n" + "\"light_schema_change\" = \"true\",\n" + "\"disable_auto_compaction\" = \"false\",\n" - + "\"enable_single_replica_compaction\" = \"false\"\n" + + "\"enable_single_replica_compaction\" = \"false\",\n" + + "\"enable_mow_light_delete\" = \"false\"\n" + ");", createTableStmts.get(0)); } else { @@ -569,7 +588,8 @@ public void testQuerySchema() throws Exception { + "\"storage_format\" = \"V2\",\n" + "\"light_schema_change\" = \"true\",\n" + "\"disable_auto_compaction\" = \"false\",\n" - + "\"enable_single_replica_compaction\" = \"false\"\n" + + "\"enable_single_replica_compaction\" = \"false\",\n" + + "\"enable_mow_light_delete\" = \"false\"\n" + ");", createTableStmts.get(0)); } @@ -600,7 +620,8 @@ public void testVarcharLength() throws Exception { + "\"storage_format\" = \"V2\",\n" + "\"light_schema_change\" = \"true\",\n" + "\"disable_auto_compaction\" = \"false\",\n" - + "\"enable_single_replica_compaction\" = \"false\"\n" + + "\"enable_single_replica_compaction\" = \"false\",\n" + + "\"enable_mow_light_delete\" = \"false\"\n" + ");", showStr); } diff --git a/regression-test/data/index_p0/test_bitmap_index.out b/regression-test/data/index_p0/test_bitmap_index.out index 402c0c88493dab..95e9be7bb1dae1 100644 --- a/regression-test/data/index_p0/test_bitmap_index.out +++ b/regression-test/data/index_p0/test_bitmap_index.out @@ -102,9 +102,6 @@ usage_mode INT Yes false \N NONE -- !sql -- 2 --- !sql -- -2 - -- !sql -- 2023-08-25T10:00 123 2023-08-25T10:00 1 1 1 2023-08-25T11:00 123 2023-08-25T11:00 2 2 2 diff --git a/regression-test/data/show_p0/test_show_create_table_and_views.out b/regression-test/data/show_p0/test_show_create_table_and_views.out index 804174d629ab68..d8b8047f78d3ea 100644 --- a/regression-test/data/show_p0/test_show_create_table_and_views.out +++ b/regression-test/data/show_p0/test_show_create_table_and_views.out @@ -1,6 +1,6 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !show -- -show_create_table_and_views_table CREATE TABLE `show_create_table_and_views_table` (\n `user_id` LARGEINT NOT NULL,\n `good_id` LARGEINT NOT NULL,\n `cost` BIGINT SUM NULL DEFAULT "0"\n) ENGINE=OLAP\nAGGREGATE KEY(`user_id`, `good_id`)\nPARTITION BY RANGE(`good_id`)\n(PARTITION p1 VALUES [("-170141183460469231731687303715884105728"), ("100")),\nPARTITION p2 VALUES [("100"), ("200")),\nPARTITION p3 VALUES [("200"), ("300")),\nPARTITION p4 VALUES [("300"), ("400")),\nPARTITION p5 VALUES [("400"), ("500")),\nPARTITION p6 VALUES [("500"), ("600")),\nPARTITION p7 VALUES [("600"), (MAXVALUE)))\nDISTRIBUTED BY HASH(`user_id`) BUCKETS 2\nPROPERTIES (\n"replication_allocation" = "tag.location.default: 1",\n"is_being_synced" = "false",\n"storage_medium" = "hdd",\n"storage_format" = "V2",\n"light_schema_change" = "true",\n"disable_auto_compaction" = "false",\n"binlog.enable" = "false",\n"binlog.ttl_seconds" = "86400",\n"binlog.max_bytes" = "9223372036854775807",\n"binlog.max_history_nums" = "9223372036854775807",\n"enable_single_replica_compaction" = "false"\n); +show_create_table_and_views_table CREATE TABLE `show_create_table_and_views_table` (\n `user_id` LARGEINT NOT NULL,\n `good_id` LARGEINT NOT NULL,\n `cost` BIGINT SUM NULL DEFAULT "0"\n) ENGINE=OLAP\nAGGREGATE KEY(`user_id`, `good_id`)\nPARTITION BY RANGE(`good_id`)\n(PARTITION p1 VALUES [("-170141183460469231731687303715884105728"), ("100")),\nPARTITION p2 VALUES [("100"), ("200")),\nPARTITION p3 VALUES [("200"), ("300")),\nPARTITION p4 VALUES [("300"), ("400")),\nPARTITION p5 VALUES [("400"), ("500")),\nPARTITION p6 VALUES [("500"), ("600")),\nPARTITION p7 VALUES [("600"), (MAXVALUE)))\nDISTRIBUTED BY HASH(`user_id`) BUCKETS 2\nPROPERTIES (\n"replication_allocation" = "tag.location.default: 1",\n"is_being_synced" = "false",\n"storage_medium" = "hdd",\n"storage_format" = "V2",\n"light_schema_change" = "true",\n"disable_auto_compaction" = "false",\n"binlog.enable" = "false",\n"binlog.ttl_seconds" = "86400",\n"binlog.max_bytes" = "9223372036854775807",\n"binlog.max_history_nums" = "9223372036854775807",\n"enable_single_replica_compaction" = "false",\n"enable_mow_light_delete" = "false"\n); -- !select -- 1 1 30 @@ -36,11 +36,11 @@ show_create_table_and_views_view CREATE VIEW `show_create_table_and_views_view` 300 1 -- !show -- -show_create_table_and_views_table CREATE TABLE `show_create_table_and_views_table` (\n `user_id` LARGEINT NOT NULL,\n `good_id` LARGEINT NOT NULL,\n `cost` BIGINT SUM NULL DEFAULT "0"\n) ENGINE=OLAP\nAGGREGATE KEY(`user_id`, `good_id`)\nPARTITION BY RANGE(`good_id`)\n(PARTITION p1 VALUES [("-170141183460469231731687303715884105728"), ("100")),\nPARTITION p2 VALUES [("100"), ("200")),\nPARTITION p3 VALUES [("200"), ("300")),\nPARTITION p4 VALUES [("300"), ("400")),\nPARTITION p5 VALUES [("400"), ("500")),\nPARTITION p6 VALUES [("500"), ("600")),\nPARTITION p7 VALUES [("600"), (MAXVALUE)))\nDISTRIBUTED BY HASH(`user_id`) BUCKETS 2\nPROPERTIES (\n"replication_allocation" = "tag.location.default: 1",\n"is_being_synced" = "false",\n"storage_medium" = "hdd",\n"storage_format" = "V2",\n"light_schema_change" = "true",\n"disable_auto_compaction" = "false",\n"binlog.enable" = "false",\n"binlog.ttl_seconds" = "86400",\n"binlog.max_bytes" = "9223372036854775807",\n"binlog.max_history_nums" = "9223372036854775807",\n"enable_single_replica_compaction" = "false"\n); +show_create_table_and_views_table CREATE TABLE `show_create_table_and_views_table` (\n `user_id` LARGEINT NOT NULL,\n `good_id` LARGEINT NOT NULL,\n `cost` BIGINT SUM NULL DEFAULT "0"\n) ENGINE=OLAP\nAGGREGATE KEY(`user_id`, `good_id`)\nPARTITION BY RANGE(`good_id`)\n(PARTITION p1 VALUES [("-170141183460469231731687303715884105728"), ("100")),\nPARTITION p2 VALUES [("100"), ("200")),\nPARTITION p3 VALUES [("200"), ("300")),\nPARTITION p4 VALUES [("300"), ("400")),\nPARTITION p5 VALUES [("400"), ("500")),\nPARTITION p6 VALUES [("500"), ("600")),\nPARTITION p7 VALUES [("600"), (MAXVALUE)))\nDISTRIBUTED BY HASH(`user_id`) BUCKETS 2\nPROPERTIES (\n"replication_allocation" = "tag.location.default: 1",\n"is_being_synced" = "false",\n"storage_medium" = "hdd",\n"storage_format" = "V2",\n"light_schema_change" = "true",\n"disable_auto_compaction" = "false",\n"binlog.enable" = "false",\n"binlog.ttl_seconds" = "86400",\n"binlog.max_bytes" = "9223372036854775807",\n"binlog.max_history_nums" = "9223372036854775807",\n"enable_single_replica_compaction" = "false",\n"enable_mow_light_delete" = "false"\n); -- !show -- -show_create_table_and_views_like CREATE TABLE `show_create_table_and_views_like` (\n `user_id` LARGEINT NOT NULL,\n `good_id` LARGEINT NOT NULL,\n `cost` BIGINT SUM NULL DEFAULT "0"\n) ENGINE=OLAP\nAGGREGATE KEY(`user_id`, `good_id`)\nPARTITION BY RANGE(`good_id`)\n(PARTITION p1 VALUES [("-170141183460469231731687303715884105728"), ("100")),\nPARTITION p2 VALUES [("100"), ("200")),\nPARTITION p3 VALUES [("200"), ("300")),\nPARTITION p4 VALUES [("300"), ("400")),\nPARTITION p5 VALUES [("400"), ("500")),\nPARTITION p6 VALUES [("500"), ("600")),\nPARTITION p7 VALUES [("600"), (MAXVALUE)))\nDISTRIBUTED BY HASH(`user_id`) BUCKETS 2\nPROPERTIES (\n"replication_allocation" = "tag.location.default: 1",\n"is_being_synced" = "false",\n"storage_medium" = "hdd",\n"storage_format" = "V2",\n"light_schema_change" = "true",\n"disable_auto_compaction" = "false",\n"binlog.enable" = "false",\n"binlog.ttl_seconds" = "86400",\n"binlog.max_bytes" = "9223372036854775807",\n"binlog.max_history_nums" = "9223372036854775807",\n"enable_single_replica_compaction" = "false"\n); +show_create_table_and_views_like CREATE TABLE `show_create_table_and_views_like` (\n `user_id` LARGEINT NOT NULL,\n `good_id` LARGEINT NOT NULL,\n `cost` BIGINT SUM NULL DEFAULT "0"\n) ENGINE=OLAP\nAGGREGATE KEY(`user_id`, `good_id`)\nPARTITION BY RANGE(`good_id`)\n(PARTITION p1 VALUES [("-170141183460469231731687303715884105728"), ("100")),\nPARTITION p2 VALUES [("100"), ("200")),\nPARTITION p3 VALUES [("200"), ("300")),\nPARTITION p4 VALUES [("300"), ("400")),\nPARTITION p5 VALUES [("400"), ("500")),\nPARTITION p6 VALUES [("500"), ("600")),\nPARTITION p7 VALUES [("600"), (MAXVALUE)))\nDISTRIBUTED BY HASH(`user_id`) BUCKETS 2\nPROPERTIES (\n"replication_allocation" = "tag.location.default: 1",\n"is_being_synced" = "false",\n"storage_medium" = "hdd",\n"storage_format" = "V2",\n"light_schema_change" = "true",\n"disable_auto_compaction" = "false",\n"binlog.enable" = "false",\n"binlog.ttl_seconds" = "86400",\n"binlog.max_bytes" = "9223372036854775807",\n"binlog.max_history_nums" = "9223372036854775807",\n"enable_single_replica_compaction" = "false",\n"enable_mow_light_delete" = "false"\n); -- !show -- -show_create_table_and_views_like_with_rollup CREATE TABLE `show_create_table_and_views_like_with_rollup` (\n `user_id` LARGEINT NOT NULL,\n `good_id` LARGEINT NOT NULL,\n `cost` BIGINT SUM NULL DEFAULT "0"\n) ENGINE=OLAP\nAGGREGATE KEY(`user_id`, `good_id`)\nPARTITION BY RANGE(`good_id`)\n(PARTITION p1 VALUES [("-170141183460469231731687303715884105728"), ("100")),\nPARTITION p2 VALUES [("100"), ("200")),\nPARTITION p3 VALUES [("200"), ("300")),\nPARTITION p4 VALUES [("300"), ("400")),\nPARTITION p5 VALUES [("400"), ("500")),\nPARTITION p6 VALUES [("500"), ("600")),\nPARTITION p7 VALUES [("600"), (MAXVALUE)))\nDISTRIBUTED BY HASH(`user_id`) BUCKETS 2\nPROPERTIES (\n"replication_allocation" = "tag.location.default: 1",\n"is_being_synced" = "false",\n"storage_medium" = "hdd",\n"storage_format" = "V2",\n"light_schema_change" = "true",\n"disable_auto_compaction" = "false",\n"binlog.enable" = "false",\n"binlog.ttl_seconds" = "86400",\n"binlog.max_bytes" = "9223372036854775807",\n"binlog.max_history_nums" = "9223372036854775807",\n"enable_single_replica_compaction" = "false"\n); +show_create_table_and_views_like_with_rollup CREATE TABLE `show_create_table_and_views_like_with_rollup` (\n `user_id` LARGEINT NOT NULL,\n `good_id` LARGEINT NOT NULL,\n `cost` BIGINT SUM NULL DEFAULT "0"\n) ENGINE=OLAP\nAGGREGATE KEY(`user_id`, `good_id`)\nPARTITION BY RANGE(`good_id`)\n(PARTITION p1 VALUES [("-170141183460469231731687303715884105728"), ("100")),\nPARTITION p2 VALUES [("100"), ("200")),\nPARTITION p3 VALUES [("200"), ("300")),\nPARTITION p4 VALUES [("300"), ("400")),\nPARTITION p5 VALUES [("400"), ("500")),\nPARTITION p6 VALUES [("500"), ("600")),\nPARTITION p7 VALUES [("600"), (MAXVALUE)))\nDISTRIBUTED BY HASH(`user_id`) BUCKETS 2\nPROPERTIES (\n"replication_allocation" = "tag.location.default: 1",\n"is_being_synced" = "false",\n"storage_medium" = "hdd",\n"storage_format" = "V2",\n"light_schema_change" = "true",\n"disable_auto_compaction" = "false",\n"binlog.enable" = "false",\n"binlog.ttl_seconds" = "86400",\n"binlog.max_bytes" = "9223372036854775807",\n"binlog.max_history_nums" = "9223372036854775807",\n"enable_single_replica_compaction" = "false",\n"enable_mow_light_delete" = "false"\n); diff --git a/regression-test/suites/index_p0/test_bitmap_index.groovy b/regression-test/suites/index_p0/test_bitmap_index.groovy index e6f32462de42e1..3344f8557f3be4 100644 --- a/regression-test/suites/index_p0/test_bitmap_index.groovy +++ b/regression-test/suites/index_p0/test_bitmap_index.groovy @@ -473,7 +473,7 @@ suite("test_bitmap_index") { qt_sql "select count(*) from ${tbName5}; " qt_sql "select count(*) from ${tbName5} where vid='123'; " qt_sql "select count(*) from ${tbName5} where create_time>='2023-08-25 10:00:00';" - qt_sql "select count(CASE when vid='123' then 1 else null end) from ${tbName5} where vid='123';" + // qt_sql "select count(CASE when vid='123' then 1 else null end) from ${tbName5} where vid='123';" qt_sql "select * from ${tbName5} where vid='123' order by create_time;" sql "DROP INDEX IF EXISTS index1 ON ${tbName5};"