diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java index 109a0376a316e8..cdb69177afaf43 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java @@ -2635,7 +2635,7 @@ public class Config extends ConfigBase { + "DELETE statements, but partial column updates after a DELETE may result in erroneous data. " + "If disabled, it will reduce the performance of DELETE statements to ensure accuracy." }) - public static boolean enable_mow_delete_on_predicate = false; + public static boolean enable_mow_light_delete = false; @ConfField(description = { "是否开启 Proxy Protocol 支持", diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/CloudRollupJobV2.java b/fe/fe-core/src/main/java/org/apache/doris/alter/CloudRollupJobV2.java index 688c2cd17cd0e0..4f2a241e08981c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/CloudRollupJobV2.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/CloudRollupJobV2.java @@ -196,20 +196,21 @@ private void createRollupReplicaForPartition(OlapTable tbl) throws Exception { ((CloudInternalCatalog) Env.getCurrentInternalCatalog()) .createTabletMetaBuilder(tableId, rollupIndexId, partitionId, rollupTablet, tabletType, rollupSchemaHash, - rollupKeysType, rollupShortKeyColumnCount, tbl.getCopiedBfColumns(), - tbl.getBfFpp(), null, rollupSchema, - tbl.getDataSortInfo(), tbl.getCompressionType(), tbl.getStoragePolicy(), - tbl.isInMemory(), true, - tbl.getName(), tbl.getTTLSeconds(), - tbl.getEnableUniqueKeyMergeOnWrite(), tbl.storeRowColumn(), - tbl.getBaseSchemaVersion(), tbl.getCompactionPolicy(), - tbl.getTimeSeriesCompactionGoalSizeMbytes(), - tbl.getTimeSeriesCompactionFileCountThreshold(), - tbl.getTimeSeriesCompactionTimeThresholdSeconds(), - tbl.getTimeSeriesCompactionEmptyRowsetsThreshold(), - tbl.getTimeSeriesCompactionLevelThreshold(), - tbl.disableAutoCompaction(), - tbl.getRowStoreColumnsUniqueIds(rowStoreColumns)); + rollupKeysType, rollupShortKeyColumnCount, tbl.getCopiedBfColumns(), + tbl.getBfFpp(), null, rollupSchema, + tbl.getDataSortInfo(), tbl.getCompressionType(), tbl.getStoragePolicy(), + tbl.isInMemory(), true, + tbl.getName(), tbl.getTTLSeconds(), + tbl.getEnableUniqueKeyMergeOnWrite(), tbl.storeRowColumn(), + tbl.getBaseSchemaVersion(), tbl.getCompactionPolicy(), + tbl.getTimeSeriesCompactionGoalSizeMbytes(), + tbl.getTimeSeriesCompactionFileCountThreshold(), + tbl.getTimeSeriesCompactionTimeThresholdSeconds(), + tbl.getTimeSeriesCompactionEmptyRowsetsThreshold(), + tbl.getTimeSeriesCompactionLevelThreshold(), + tbl.disableAutoCompaction(), + tbl.getRowStoreColumnsUniqueIds(rowStoreColumns), + tbl.getEnableMowLightDelete()); requestBuilder.addTabletMetas(builder); } // end for rollupTablets ((CloudInternalCatalog) Env.getCurrentInternalCatalog()) diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/CloudSchemaChangeJobV2.java b/fe/fe-core/src/main/java/org/apache/doris/alter/CloudSchemaChangeJobV2.java index 3968f2d274f41e..6a4584c733bb0e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/CloudSchemaChangeJobV2.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/CloudSchemaChangeJobV2.java @@ -213,21 +213,24 @@ private void createShadowIndexReplicaForPartition(OlapTable tbl) throws Exceptio for (Tablet shadowTablet : shadowIdx.getTablets()) { OlapFile.TabletMetaCloudPB.Builder builder = ((CloudInternalCatalog) Env.getCurrentInternalCatalog()) - .createTabletMetaBuilder(tableId, shadowIdxId, - partitionId, shadowTablet, tbl.getPartitionInfo().getTabletType(partitionId), - shadowSchemaHash, originKeysType, shadowShortKeyColumnCount, bfColumns, - bfFpp, tabletIndexes, shadowSchema, tbl.getDataSortInfo(), tbl.getCompressionType(), - tbl.getStoragePolicy(), tbl.isInMemory(), true, - tbl.getName(), tbl.getTTLSeconds(), - tbl.getEnableUniqueKeyMergeOnWrite(), tbl.storeRowColumn(), - shadowSchemaVersion, tbl.getCompactionPolicy(), - tbl.getTimeSeriesCompactionGoalSizeMbytes(), - tbl.getTimeSeriesCompactionFileCountThreshold(), - tbl.getTimeSeriesCompactionTimeThresholdSeconds(), - tbl.getTimeSeriesCompactionEmptyRowsetsThreshold(), - tbl.getTimeSeriesCompactionLevelThreshold(), - tbl.disableAutoCompaction(), - tbl.getRowStoreColumnsUniqueIds(rowStoreColumns)); + .createTabletMetaBuilder(tableId, shadowIdxId, + partitionId, shadowTablet, + tbl.getPartitionInfo().getTabletType(partitionId), + shadowSchemaHash, originKeysType, shadowShortKeyColumnCount, bfColumns, + bfFpp, tabletIndexes, shadowSchema, tbl.getDataSortInfo(), + tbl.getCompressionType(), + tbl.getStoragePolicy(), tbl.isInMemory(), true, + tbl.getName(), tbl.getTTLSeconds(), + tbl.getEnableUniqueKeyMergeOnWrite(), tbl.storeRowColumn(), + shadowSchemaVersion, tbl.getCompactionPolicy(), + tbl.getTimeSeriesCompactionGoalSizeMbytes(), + tbl.getTimeSeriesCompactionFileCountThreshold(), + tbl.getTimeSeriesCompactionTimeThresholdSeconds(), + tbl.getTimeSeriesCompactionEmptyRowsetsThreshold(), + tbl.getTimeSeriesCompactionLevelThreshold(), + tbl.disableAutoCompaction(), + tbl.getRowStoreColumnsUniqueIds(rowStoreColumns), + tbl.getEnableMowLightDelete()); requestBuilder.addTabletMetas(builder); } // end for rollupTablets ((CloudInternalCatalog) Env.getCurrentInternalCatalog()) diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java index b3b04bce21a0e6..a03c6809ba572b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java @@ -2258,7 +2258,7 @@ public void updateTableProperties(Database db, String tableName, Map rowStoreColumns = - tbl.getTableProperty().getCopiedRowStoreColumns(); + tbl.getTableProperty().getCopiedRowStoreColumns(); for (Tablet tablet : index.getTablets()) { OlapFile.TabletMetaCloudPB.Builder builder = createTabletMetaBuilder(tbl.getId(), indexId, partitionId, tablet, tabletType, schemaHash, keysType, shortKeyColumnCount, @@ -169,7 +169,8 @@ protected Partition createPartitionWithIndices(long dbId, OlapTable tbl, long pa tbl.getTimeSeriesCompactionEmptyRowsetsThreshold(), tbl.getTimeSeriesCompactionLevelThreshold(), tbl.disableAutoCompaction(), - tbl.getRowStoreColumnsUniqueIds(rowStoreColumns)); + tbl.getRowStoreColumnsUniqueIds(rowStoreColumns), + tbl.getEnableMowLightDelete()); requestBuilder.addTabletMetas(builder); } if (!storageVaultIdSet && ((CloudEnv) Env.getCurrentEnv()).getEnableStorageVault()) { @@ -216,7 +217,7 @@ public OlapFile.TabletMetaCloudPB.Builder createTabletMetaBuilder(long tableId, Long timeSeriesCompactionGoalSizeMbytes, Long timeSeriesCompactionFileCountThreshold, Long timeSeriesCompactionTimeThresholdSeconds, Long timeSeriesCompactionEmptyRowsetsThreshold, Long timeSeriesCompactionLevelThreshold, boolean disableAutoCompaction, - List rowStoreColumnUniqueIds) throws DdlException { + List rowStoreColumnUniqueIds, boolean enableMowLightDelete) throws DdlException { OlapFile.TabletMetaCloudPB.Builder builder = OlapFile.TabletMetaCloudPB.newBuilder(); builder.setTableId(tableId); builder.setIndexId(indexId); @@ -333,6 +334,7 @@ public OlapFile.TabletMetaCloudPB.Builder createTabletMetaBuilder(long tableId, schemaBuilder.addAllRowStoreColumnUniqueIds(rowStoreColumnUniqueIds); } schemaBuilder.setDisableAutoCompaction(disableAutoCompaction); + schemaBuilder.setEnableMowLightDelete(enableMowLightDelete); OlapFile.TabletSchemaCloudPB schema = schemaBuilder.build(); builder.setSchema(schema); diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java index 29ed40676b3903..32e6edc14eeb80 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java @@ -209,10 +209,10 @@ public class PropertyAnalyzer { public static final int PROPERTIES_GROUP_COMMIT_DATA_BYTES_DEFAULT_VALUE = Config.group_commit_data_bytes_default_value; - public static final String PROPERTIES_ENABLE_MOW_DELETE_ON_DELETE_PREDICATE = - "enable_mow_delete_on_delete_predicate"; - public static final boolean PROPERTIES_ENABLE_MOW_DELETE_ON_DELETE_PREDICATE_DEFAULT_VALUE - = Config.enable_mow_delete_on_predicate; + public static final String PROPERTIES_ENABLE_MOW_LIGHT_DELETE = + "enable_mow_light_delete"; + public static final boolean PROPERTIES_ENABLE_MOW_LIGHT_DELETE_DEFAULT_VALUE + = Config.enable_mow_light_delete; // compaction policy public static final String SIZE_BASED_COMPACTION_POLICY = "size_based"; @@ -1423,18 +1423,18 @@ public static boolean analyzeEnableDeleteOnDeletePredicate(Map p if (properties == null || properties.isEmpty()) { return false; } - String value = properties.get(PropertyAnalyzer.PROPERTIES_ENABLE_MOW_DELETE_ON_DELETE_PREDICATE); + String value = properties.get(PropertyAnalyzer.PROPERTIES_ENABLE_MOW_LIGHT_DELETE); if (value == null) { return false; } - properties.remove(PropertyAnalyzer.PROPERTIES_ENABLE_MOW_DELETE_ON_DELETE_PREDICATE); + properties.remove(PropertyAnalyzer.PROPERTIES_ENABLE_MOW_LIGHT_DELETE); if (value.equals("true")) { return true; } else if (value.equals("false")) { return false; } throw new AnalysisException( - PropertyAnalyzer.PROPERTIES_ENABLE_MOW_DELETE_ON_DELETE_PREDICATE + " must be `true` or `false`"); + PropertyAnalyzer.PROPERTIES_ENABLE_MOW_LIGHT_DELETE + " must be `true` or `false`"); } /** diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java index 68d7db6256cce9..436642189c8fbf 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java @@ -2457,18 +2457,16 @@ private boolean createOlapTable(Database db, CreateTableStmt stmt) throws UserEx olapTable.setEnableUniqueKeyMergeOnWrite(enableUniqueKeyMergeOnWrite); boolean enableDeleteOnDeletePredicate = false; - if (keysType == KeysType.UNIQUE_KEYS) { - try { - enableDeleteOnDeletePredicate = PropertyAnalyzer.analyzeEnableDeleteOnDeletePredicate(properties); - } catch (AnalysisException e) { - throw new DdlException(e.getMessage()); - } - if (enableDeleteOnDeletePredicate && !enableUniqueKeyMergeOnWrite) { - throw new DdlException(PropertyAnalyzer.PROPERTIES_ENABLE_MOW_DELETE_ON_DELETE_PREDICATE - + " property is not supported for unique merge-on-read table"); - } + try { + enableDeleteOnDeletePredicate = PropertyAnalyzer.analyzeEnableDeleteOnDeletePredicate(properties); + } catch (AnalysisException e) { + throw new DdlException(e.getMessage()); + } + if (enableDeleteOnDeletePredicate && !enableUniqueKeyMergeOnWrite) { + throw new DdlException(PropertyAnalyzer.PROPERTIES_ENABLE_MOW_LIGHT_DELETE + + " property is only supported for unique merge-on-write table"); } - olapTable.setEnableDeleteOnDeletePredicate(enableDeleteOnDeletePredicate); + olapTable.setEnableMowLightDelete(enableDeleteOnDeletePredicate); boolean enableSingleReplicaCompaction = false; try { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/DeleteFromCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/DeleteFromCommand.java index cbfa94be0503ff..fe5552fed65e3e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/DeleteFromCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/DeleteFromCommand.java @@ -167,7 +167,7 @@ public void run(ConnectContext ctx, StmtExecutor executor) throws Exception { } if (olapTable.getKeysType() == KeysType.UNIQUE_KEYS && olapTable.getEnableUniqueKeyMergeOnWrite() - && !olapTable.getEnableDeleteOnDeletePredicate()) { + && !olapTable.getEnableMowLightDelete()) { new DeleteFromUsingCommand(nameParts, tableAlias, isTempPart, partitions, logicalQuery, Optional.empty()).run(ctx, executor); return; diff --git a/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateTableAsSelectStmtTest.java b/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateTableAsSelectStmtTest.java index d3c7bdf95a5fc2..5e6c53b18ad2d9 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateTableAsSelectStmtTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateTableAsSelectStmtTest.java @@ -578,7 +578,7 @@ public void testUseKeyType() throws Exception { + "\"enable_single_replica_compaction\" = \"false\",\n" + "\"group_commit_interval_ms\" = \"10000\",\n" + "\"group_commit_data_bytes\" = \"134217728\",\n" - + "\"enable_mow_delete_on_delete_predicate\" = \"false\"\n" + + "\"enable_mow_light_delete\" = \"false\"\n" + ");", showResultSet.getResultRows().get(0).get(1)); } diff --git a/gensrc/proto/cloud.proto b/gensrc/proto/cloud.proto index ab69deaa212adb..5179ea52a9e0d3 100644 --- a/gensrc/proto/cloud.proto +++ b/gensrc/proto/cloud.proto @@ -497,6 +497,7 @@ message TabletMetaInfoPB { // For update tablet meta optional int64 time_series_compaction_empty_rowsets_threshold = 11; optional int64 time_series_compaction_level_threshold = 12; optional bool disable_auto_compaction = 13; + optional bool enable_mow_light_delete = 14; } message TabletCompactionJobPB { diff --git a/gensrc/proto/olap_file.proto b/gensrc/proto/olap_file.proto index 1f8f88801be40f..f2fbf09122df36 100644 --- a/gensrc/proto/olap_file.proto +++ b/gensrc/proto/olap_file.proto @@ -411,6 +411,7 @@ message TabletSchemaCloudPB { optional InvertedIndexStorageFormatPB inverted_index_storage_format = 25 [default=V1]; // column unique ids for row store columns repeated int32 row_store_column_unique_ids = 26; + optional bool enable_mow_light_delete = 27 [default=false]; optional bool is_dynamic_schema = 100 [default=false]; } diff --git a/regression-test/data/query_p0/system/test_table_options.out b/regression-test/data/query_p0/system/test_table_options.out index 71b7de92f837c2..974c3bc2c7e817 100644 --- a/regression-test/data/query_p0/system/test_table_options.out +++ b/regression-test/data/query_p0/system/test_table_options.out @@ -1,9 +1,9 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !select -- -aggregate_table internal test_table_options_db AGG user_id,date,city,age,sex user_id HASH 1 1 {"min_load_replica_num":"-1","data_sort.col_num":"5","group_commit_interval_ms":"10000","data_sort.sort_type":"LEXICAL","is_being_synced":"false","binlog.enable":"false","binlog.ttl_seconds":"86400","inverted_index_storage_format":"V2","time_series_compaction_empty_rowsets_threshold":"5","default.replication_allocation":"tag.location.default: 1","time_series_compaction_level_threshold":"1","time_series_compaction_time_threshold_seconds":"3600","storage_format":"V2","store_row_column":"false","light_schema_change":"true","enable_unique_key_merge_on_write":"false","in_memory":"false","file_cache_ttl_seconds":"0","group_commit_data_bytes":"134217728","enable_mow_delete_on_delete_predicate":"false","compaction_policy":"size_based","_auto_bucket":"false","binlog.max_history_nums":"9223372036854775807","time_series_compaction_file_count_threshold":"2000","skip_write_index_on_load":"false","disable_auto_compaction":"false","time_series_compaction_goal_size_mbytes":"1024","storage_medium":"HDD","enable_single_replica_compaction":"false","compression":"LZ4F","binlog.max_bytes":"9223372036854775807"} -duplicate_table internal test_table_options_db DUP timestamp,type,error_code type HASH 1 1 {"min_load_replica_num":"-1","data_sort.col_num":"3","group_commit_interval_ms":"10000","data_sort.sort_type":"LEXICAL","is_being_synced":"false","binlog.enable":"false","binlog.ttl_seconds":"86400","inverted_index_storage_format":"V2","time_series_compaction_empty_rowsets_threshold":"5","default.replication_allocation":"tag.location.default: 1","time_series_compaction_level_threshold":"1","time_series_compaction_time_threshold_seconds":"3600","storage_format":"V2","store_row_column":"false","light_schema_change":"true","enable_unique_key_merge_on_write":"false","in_memory":"false","file_cache_ttl_seconds":"0","group_commit_data_bytes":"134217728","enable_mow_delete_on_delete_predicate":"false","compaction_policy":"size_based","_auto_bucket":"false","binlog.max_history_nums":"9223372036854775807","time_series_compaction_file_count_threshold":"2000","skip_write_index_on_load":"false","disable_auto_compaction":"false","time_series_compaction_goal_size_mbytes":"1024","storage_medium":"HDD","enable_single_replica_compaction":"false","compression":"LZ4F","binlog.max_bytes":"9223372036854775807"} -listtable internal test_table_options_db AGG user_id,date,timestamp,city,age,sex user_id HASH 16 3 {"min_load_replica_num":"-1","data_sort.col_num":"6","group_commit_interval_ms":"10000","data_sort.sort_type":"LEXICAL","is_being_synced":"false","binlog.enable":"false","binlog.ttl_seconds":"86400","inverted_index_storage_format":"V2","time_series_compaction_empty_rowsets_threshold":"5","default.replication_allocation":"tag.location.default: 1","time_series_compaction_level_threshold":"1","time_series_compaction_time_threshold_seconds":"3600","storage_format":"V2","store_row_column":"false","light_schema_change":"true","enable_unique_key_merge_on_write":"false","in_memory":"false","file_cache_ttl_seconds":"0","group_commit_data_bytes":"134217728","enable_mow_delete_on_delete_predicate":"false","compaction_policy":"size_based","_auto_bucket":"false","binlog.max_history_nums":"9223372036854775807","time_series_compaction_file_count_threshold":"2000","skip_write_index_on_load":"false","disable_auto_compaction":"false","time_series_compaction_goal_size_mbytes":"1024","storage_medium":"HDD","enable_single_replica_compaction":"false","compression":"LZ4F","binlog.max_bytes":"9223372036854775807"} -randomtable internal test_table_options_db DUP user_id,date,timestamp RANDOM RANDOM 16 1 {"min_load_replica_num":"-1","data_sort.col_num":"3","group_commit_interval_ms":"10000","data_sort.sort_type":"LEXICAL","is_being_synced":"false","binlog.enable":"false","binlog.ttl_seconds":"86400","inverted_index_storage_format":"V2","time_series_compaction_empty_rowsets_threshold":"5","default.replication_allocation":"tag.location.default: 1","time_series_compaction_level_threshold":"1","time_series_compaction_time_threshold_seconds":"3600","storage_format":"V2","store_row_column":"false","light_schema_change":"true","enable_unique_key_merge_on_write":"false","in_memory":"false","file_cache_ttl_seconds":"0","group_commit_data_bytes":"134217728","enable_mow_delete_on_delete_predicate":"false","compaction_policy":"size_based","_auto_bucket":"false","binlog.max_history_nums":"9223372036854775807","time_series_compaction_file_count_threshold":"2000","skip_write_index_on_load":"false","disable_auto_compaction":"false","time_series_compaction_goal_size_mbytes":"1024","storage_medium":"HDD","enable_single_replica_compaction":"false","compression":"LZ4F","binlog.max_bytes":"9223372036854775807"} -rangetable internal test_table_options_db AGG user_id,date,timestamp,city,age,sex user_id HASH 8 3 {"min_load_replica_num":"-1","data_sort.col_num":"6","group_commit_interval_ms":"10000","data_sort.sort_type":"LEXICAL","is_being_synced":"false","binlog.enable":"false","binlog.ttl_seconds":"86400","inverted_index_storage_format":"V2","time_series_compaction_empty_rowsets_threshold":"5","default.replication_allocation":"tag.location.default: 1","time_series_compaction_level_threshold":"1","time_series_compaction_time_threshold_seconds":"3600","storage_format":"V2","store_row_column":"false","light_schema_change":"true","enable_unique_key_merge_on_write":"false","in_memory":"false","file_cache_ttl_seconds":"0","group_commit_data_bytes":"134217728","enable_mow_delete_on_delete_predicate":"false","compaction_policy":"size_based","_auto_bucket":"false","binlog.max_history_nums":"9223372036854775807","time_series_compaction_file_count_threshold":"2000","skip_write_index_on_load":"false","disable_auto_compaction":"false","time_series_compaction_goal_size_mbytes":"1024","storage_medium":"HDD","enable_single_replica_compaction":"false","compression":"LZ4F","binlog.max_bytes":"9223372036854775807"} -unique_table internal test_table_options_db UNI user_id,username user_id HASH 1 1 {"min_load_replica_num":"-1","data_sort.col_num":"2","group_commit_interval_ms":"10000","data_sort.sort_type":"LEXICAL","is_being_synced":"false","binlog.enable":"false","binlog.ttl_seconds":"86400","inverted_index_storage_format":"V2","time_series_compaction_empty_rowsets_threshold":"5","default.replication_allocation":"tag.location.default: 1","time_series_compaction_level_threshold":"1","time_series_compaction_time_threshold_seconds":"3600","storage_format":"V2","store_row_column":"false","light_schema_change":"true","enable_unique_key_merge_on_write":"true","in_memory":"false","file_cache_ttl_seconds":"0","group_commit_data_bytes":"134217728","enable_mow_delete_on_delete_predicate":"false","compaction_policy":"size_based","_auto_bucket":"false","binlog.max_history_nums":"9223372036854775807","time_series_compaction_file_count_threshold":"2000","skip_write_index_on_load":"false","disable_auto_compaction":"false","time_series_compaction_goal_size_mbytes":"1024","storage_medium":"HDD","enable_single_replica_compaction":"false","compression":"LZ4F","binlog.max_bytes":"9223372036854775807"} +aggregate_table internal test_table_options_db AGG user_id,date,city,age,sex user_id HASH 1 1 {"min_load_replica_num":"-1","data_sort.col_num":"5","group_commit_interval_ms":"10000","data_sort.sort_type":"LEXICAL","is_being_synced":"false","binlog.enable":"false","enable_mow_light_delete":"false","binlog.ttl_seconds":"86400","inverted_index_storage_format":"V2","time_series_compaction_empty_rowsets_threshold":"5","default.replication_allocation":"tag.location.default: 1","time_series_compaction_level_threshold":"1","time_series_compaction_time_threshold_seconds":"3600","storage_format":"V2","store_row_column":"false","light_schema_change":"true","enable_unique_key_merge_on_write":"false","in_memory":"false","file_cache_ttl_seconds":"0","group_commit_data_bytes":"134217728","compaction_policy":"size_based","_auto_bucket":"false","binlog.max_history_nums":"9223372036854775807","time_series_compaction_file_count_threshold":"2000","skip_write_index_on_load":"false","disable_auto_compaction":"false","time_series_compaction_goal_size_mbytes":"1024","storage_medium":"HDD","enable_single_replica_compaction":"false","compression":"LZ4F","binlog.max_bytes":"9223372036854775807"} +duplicate_table internal test_table_options_db DUP timestamp,type,error_code type HASH 1 1 {"min_load_replica_num":"-1","data_sort.col_num":"3","group_commit_interval_ms":"10000","data_sort.sort_type":"LEXICAL","is_being_synced":"false","binlog.enable":"false","enable_mow_light_delete":"false","binlog.ttl_seconds":"86400","inverted_index_storage_format":"V2","time_series_compaction_empty_rowsets_threshold":"5","default.replication_allocation":"tag.location.default: 1","time_series_compaction_level_threshold":"1","time_series_compaction_time_threshold_seconds":"3600","storage_format":"V2","store_row_column":"false","light_schema_change":"true","enable_unique_key_merge_on_write":"false","in_memory":"false","file_cache_ttl_seconds":"0","group_commit_data_bytes":"134217728","compaction_policy":"size_based","_auto_bucket":"false","binlog.max_history_nums":"9223372036854775807","time_series_compaction_file_count_threshold":"2000","skip_write_index_on_load":"false","disable_auto_compaction":"false","time_series_compaction_goal_size_mbytes":"1024","storage_medium":"HDD","enable_single_replica_compaction":"false","compression":"LZ4F","binlog.max_bytes":"9223372036854775807"} +listtable internal test_table_options_db AGG user_id,date,timestamp,city,age,sex user_id HASH 16 3 {"min_load_replica_num":"-1","data_sort.col_num":"6","group_commit_interval_ms":"10000","data_sort.sort_type":"LEXICAL","is_being_synced":"false","binlog.enable":"false","enable_mow_light_delete":"false","binlog.ttl_seconds":"86400","inverted_index_storage_format":"V2","time_series_compaction_empty_rowsets_threshold":"5","default.replication_allocation":"tag.location.default: 1","time_series_compaction_level_threshold":"1","time_series_compaction_time_threshold_seconds":"3600","storage_format":"V2","store_row_column":"false","light_schema_change":"true","enable_unique_key_merge_on_write":"false","in_memory":"false","file_cache_ttl_seconds":"0","group_commit_data_bytes":"134217728","compaction_policy":"size_based","_auto_bucket":"false","binlog.max_history_nums":"9223372036854775807","time_series_compaction_file_count_threshold":"2000","skip_write_index_on_load":"false","disable_auto_compaction":"false","time_series_compaction_goal_size_mbytes":"1024","storage_medium":"HDD","enable_single_replica_compaction":"false","compression":"LZ4F","binlog.max_bytes":"9223372036854775807"} +randomtable internal test_table_options_db DUP user_id,date,timestamp RANDOM RANDOM 16 1 {"min_load_replica_num":"-1","data_sort.col_num":"3","group_commit_interval_ms":"10000","data_sort.sort_type":"LEXICAL","is_being_synced":"false","binlog.enable":"false","enable_mow_light_delete":"false","binlog.ttl_seconds":"86400","inverted_index_storage_format":"V2","time_series_compaction_empty_rowsets_threshold":"5","default.replication_allocation":"tag.location.default: 1","time_series_compaction_level_threshold":"1","time_series_compaction_time_threshold_seconds":"3600","storage_format":"V2","store_row_column":"false","light_schema_change":"true","enable_unique_key_merge_on_write":"false","in_memory":"false","file_cache_ttl_seconds":"0","group_commit_data_bytes":"134217728","compaction_policy":"size_based","_auto_bucket":"false","binlog.max_history_nums":"9223372036854775807","time_series_compaction_file_count_threshold":"2000","skip_write_index_on_load":"false","disable_auto_compaction":"false","time_series_compaction_goal_size_mbytes":"1024","storage_medium":"HDD","enable_single_replica_compaction":"false","compression":"LZ4F","binlog.max_bytes":"9223372036854775807"} +rangetable internal test_table_options_db AGG user_id,date,timestamp,city,age,sex user_id HASH 8 3 {"min_load_replica_num":"-1","data_sort.col_num":"6","group_commit_interval_ms":"10000","data_sort.sort_type":"LEXICAL","is_being_synced":"false","binlog.enable":"false","enable_mow_light_delete":"false","binlog.ttl_seconds":"86400","inverted_index_storage_format":"V2","time_series_compaction_empty_rowsets_threshold":"5","default.replication_allocation":"tag.location.default: 1","time_series_compaction_level_threshold":"1","time_series_compaction_time_threshold_seconds":"3600","storage_format":"V2","store_row_column":"false","light_schema_change":"true","enable_unique_key_merge_on_write":"false","in_memory":"false","file_cache_ttl_seconds":"0","group_commit_data_bytes":"134217728","compaction_policy":"size_based","_auto_bucket":"false","binlog.max_history_nums":"9223372036854775807","time_series_compaction_file_count_threshold":"2000","skip_write_index_on_load":"false","disable_auto_compaction":"false","time_series_compaction_goal_size_mbytes":"1024","storage_medium":"HDD","enable_single_replica_compaction":"false","compression":"LZ4F","binlog.max_bytes":"9223372036854775807"} +unique_table internal test_table_options_db UNI user_id,username user_id HASH 1 1 {"min_load_replica_num":"-1","data_sort.col_num":"2","group_commit_interval_ms":"10000","data_sort.sort_type":"LEXICAL","is_being_synced":"false","binlog.enable":"false","enable_mow_light_delete":"false","binlog.ttl_seconds":"86400","inverted_index_storage_format":"V2","time_series_compaction_empty_rowsets_threshold":"5","default.replication_allocation":"tag.location.default: 1","time_series_compaction_level_threshold":"1","time_series_compaction_time_threshold_seconds":"3600","storage_format":"V2","store_row_column":"false","light_schema_change":"true","enable_unique_key_merge_on_write":"true","in_memory":"false","file_cache_ttl_seconds":"0","group_commit_data_bytes":"134217728","compaction_policy":"size_based","_auto_bucket":"false","binlog.max_history_nums":"9223372036854775807","time_series_compaction_file_count_threshold":"2000","skip_write_index_on_load":"false","disable_auto_compaction":"false","time_series_compaction_goal_size_mbytes":"1024","storage_medium":"HDD","enable_single_replica_compaction":"false","compression":"LZ4F","binlog.max_bytes":"9223372036854775807"} diff --git a/regression-test/suites/unique_with_mow_p0/partial_update/test_new_partial_update_delete.groovy b/regression-test/suites/unique_with_mow_p0/partial_update/test_new_partial_update_delete.groovy index 015af68b9aab3e..bcd8d5f2842f97 100644 --- a/regression-test/suites/unique_with_mow_p0/partial_update/test_new_partial_update_delete.groovy +++ b/regression-test/suites/unique_with_mow_p0/partial_update/test_new_partial_update_delete.groovy @@ -43,12 +43,12 @@ suite('test_new_partial_update_delete') { PROPERTIES ( "disable_auto_compaction" = "true", "enable_unique_key_merge_on_write" = "false", - "enable_mow_delete_on_delete_predicate" = "true", + "enable_mow_light_delete" = "true", "replication_num" = "1", "store_row_column" = "${use_row_store}"); """ } catch (Exception e) { log.info(e.getMessage()) - assertTrue(e.getMessage().contains('enable_mow_delete_on_delete_predicate property is not supported for unique merge-on-read table')) + assertTrue(e.getMessage().contains('enable_mow_light_delete property is only supported for unique merge-on-write table')) } try { @@ -67,10 +67,10 @@ suite('test_new_partial_update_delete') { "enable_unique_key_merge_on_write" = "false", "replication_num" = "1", "store_row_column" = "${use_row_store}"); """ - sql """alter table ${tableMorName2} set ("enable_mow_delete_on_delete_predicate"="true")""" + sql """alter table ${tableMorName2} set ("enable_mow_light_delete"="true")""" } catch (Exception e) { log.info(e.getMessage()) - assertTrue(e.getMessage().contains('enable_mow_delete_on_delete_predicate property is not supported for unique merge-on-read table')) + assertTrue(e.getMessage().contains('enable_mow_light_delete property is only supported for unique merge-on-write table')) } def tableName1 = "test_new_partial_update_delete1" @@ -89,7 +89,7 @@ suite('test_new_partial_update_delete') { "store_row_column" = "${use_row_store}"); """ def output1 = sql "show create table ${tableName1}" - assertTrue output1[0][1].contains("\"enable_mow_delete_on_delete_predicate\" = \"false\""); + assertTrue output1[0][1].contains("\"enable_mow_light_delete\" = \"false\""); sql "insert into ${tableName1} values(1,1,1,1,1)" // 1,1,1,1,1 qt_sql1 "select * from ${tableName1} order by k1;" @@ -108,11 +108,11 @@ suite('test_new_partial_update_delete') { - sql """alter table ${tableName1} set ("enable_mow_delete_on_delete_predicate"="true") """ + sql """alter table ${tableName1} set ("enable_mow_light_delete"="true") """ sql "set enable_unique_key_partial_update=false;" sql "set enable_insert_strict=true;" def output2 = sql "show create table ${tableName1}" - assertTrue output2[0][1].contains("\"enable_mow_delete_on_delete_predicate\" = \"true\""); + assertTrue output2[0][1].contains("\"enable_mow_light_delete\" = \"true\""); sql "insert into ${tableName1} values(2,2,2,2,2)" // 1,2,NULL,NULL,NULL // 2,2,2,2,2 @@ -156,12 +156,12 @@ suite('test_new_partial_update_delete') { PROPERTIES ( "disable_auto_compaction" = "true", "enable_unique_key_merge_on_write" = "false", - "enable_mow_delete_on_delete_predicate" = "true", + "enable_mow_light_delete" = "true", "replication_num" = "1", "store_row_column" = "${use_row_store}"); """ } catch (Exception e) { log.info(e.getMessage()) - assertTrue(e.getMessage().contains('enable_mow_delete_on_delete_predicate property is not supported for unique merge-on-read table')) + assertTrue(e.getMessage().contains('enable_mow_light_delete property is only supported for unique merge-on-write table')) } try { @@ -180,10 +180,10 @@ suite('test_new_partial_update_delete') { "enable_unique_key_merge_on_write" = "false", "replication_num" = "1", "store_row_column" = "${use_row_store}"); """ - sql """alter table ${tableMorName4} set ("enable_mow_delete_on_delete_predicate"="true")""" + sql """alter table ${tableMorName4} set ("enable_mow_light_delete"="true")""" } catch (Exception e) { log.info(e.getMessage()) - assertTrue(e.getMessage().contains('enable_mow_delete_on_delete_predicate property is not supported for unique merge-on-read table')) + assertTrue(e.getMessage().contains('enable_mow_light_delete property is only supported for unique merge-on-write table')) } sql "set enable_nereids_planner=false" def tableName2 = "test_new_partial_update_delete2" @@ -202,7 +202,7 @@ suite('test_new_partial_update_delete') { "store_row_column" = "${use_row_store}"); """ def output3 = sql "show create table ${tableName2}" - assertTrue output3[0][1].contains("\"enable_mow_delete_on_delete_predicate\" = \"false\""); + assertTrue output3[0][1].contains("\"enable_mow_light_delete\" = \"false\""); sql "insert into ${tableName2} values(1,1,1,1,1)" // 1,1,1,1,1 qt_sql21 "select * from ${tableName2} order by k1;" @@ -221,11 +221,11 @@ suite('test_new_partial_update_delete') { - sql """alter table ${tableName2} set ("enable_mow_delete_on_delete_predicate"="true") """ + sql """alter table ${tableName2} set ("enable_mow_light_delete"="true") """ sql "set enable_unique_key_partial_update=false;" sql "set enable_insert_strict=true;" def output4 = sql "show create table ${tableName2}" - assertTrue output4[0][1].contains("\"enable_mow_delete_on_delete_predicate\" = \"true\""); + assertTrue output4[0][1].contains("\"enable_mow_light_delete\" = \"true\""); sql "insert into ${tableName2} values(2,2,2,2,2)" // 1,2,NULL,NULL,NULL // 2,2,2,2,2 @@ -253,4 +253,47 @@ suite('test_new_partial_update_delete') { sql "drop table if exists ${tableName2};" } } + + connect(user = context.config.jdbcUser, password = context.config.jdbcPassword, url = context.config.jdbcUrl) { + sql "use ${db};" + try { + def tableAggName = "test_new_partial_update_agg_delete" + sql "DROP TABLE IF EXISTS ${tableAggName};" + sql """ CREATE TABLE IF NOT EXISTS ${tableAggName} ( + `k1` int NOT NULL, + `c1` int replace, + `c2` int replace, + `c3` int replace, + `c4` int replace + )AGGREGATE KEY(k1) + DISTRIBUTED BY HASH(k1) BUCKETS 1 + PROPERTIES ( + "disable_auto_compaction" = "true", + "enable_mow_light_delete" = "true", + "replication_num" = "1"); """ + } catch (Exception e) { + log.info(e.getMessage()) + assertTrue(e.getMessage().contains('enable_mow_light_delete property is only supported for unique merge-on-write table')) + } + + try { + def tableDupName = "test_new_partial_update_dup_delete" + sql "DROP TABLE IF EXISTS ${tableDupName};" + sql """ CREATE TABLE IF NOT EXISTS ${tableDupName} ( + `k1` int NOT NULL, + `c1` int, + `c2` int, + `c3` int, + `c4` int + )DUPLICATE KEY(k1) + DISTRIBUTED BY HASH(k1) BUCKETS 1 + PROPERTIES ( + "disable_auto_compaction" = "true", + "enable_mow_light_delete" = "true", + "replication_num" = "1"); """ + } catch (Exception e) { + log.info(e.getMessage()) + assertTrue(e.getMessage().contains('enable_mow_light_delete property is only supported for unique merge-on-write table')) + } + } }