From 63b09ef3a5f43b7218cb70af8d96de7fa17ca541 Mon Sep 17 00:00:00 2001 From: bobhan1 Date: Tue, 3 Sep 2024 16:44:50 +0800 Subject: [PATCH 1/5] opt --- .../olap/rowset/segment_v2/segment_writer.cpp | 3 +- .../segment_v2/vertical_segment_writer.cpp | 3 +- .../doris/analysis/DataDescription.java | 2 +- .../doris/planner/StreamLoadPlanner.java | 3 +- .../partial_update/partial_update_delete.csv | 3 +- .../test_partial_update_delete.out | 62 +++++--- .../test_partial_update_delete.groovy | 132 ++++++++++++++++-- 7 files changed, 173 insertions(+), 35 deletions(-) diff --git a/be/src/olap/rowset/segment_v2/segment_writer.cpp b/be/src/olap/rowset/segment_v2/segment_writer.cpp index 105433d2689e01..9f5811f67909bc 100644 --- a/be/src/olap/rowset/segment_v2/segment_writer.cpp +++ b/be/src/olap/rowset/segment_v2/segment_writer.cpp @@ -596,7 +596,8 @@ Status SegmentWriter::append_block_with_partial_content(const vectorized::Block* segment_pos); } else { - if (!_opts.rowset_ctx->partial_update_info->can_insert_new_rows_in_partial_update) { + if (!_opts.rowset_ctx->partial_update_info->can_insert_new_rows_in_partial_update && + !have_delete_sign) { std::string error_column; for (auto cid : _opts.rowset_ctx->partial_update_info->missing_cids) { const TabletColumn& col = _tablet_schema->column(cid); diff --git a/be/src/olap/rowset/segment_v2/vertical_segment_writer.cpp b/be/src/olap/rowset/segment_v2/vertical_segment_writer.cpp index 64f72bc0c4669d..7ec9236a4d8cd5 100644 --- a/be/src/olap/rowset/segment_v2/vertical_segment_writer.cpp +++ b/be/src/olap/rowset/segment_v2/vertical_segment_writer.cpp @@ -455,7 +455,8 @@ Status VerticalSegmentWriter::_append_block_with_partial_content(RowsInBlock& da DeleteBitmap::TEMP_VERSION_COMMON}, segment_pos); } else { - if (!_opts.rowset_ctx->partial_update_info->can_insert_new_rows_in_partial_update) { + if (!_opts.rowset_ctx->partial_update_info->can_insert_new_rows_in_partial_update && + !have_delete_sign) { std::string error_column; for (auto cid : _opts.rowset_ctx->partial_update_info->missing_cids) { const TabletColumn& col = _tablet_schema->column(cid); diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/DataDescription.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/DataDescription.java index 1ee3babc0d16f8..ae568237593e22 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/DataDescription.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/DataDescription.java @@ -940,7 +940,7 @@ private void analyzeSequenceCol(String fullDbName) throws AnalysisException { return; } // check olapTable schema and sequenceCol - if (olapTable.hasSequenceCol() && !hasSequenceCol()) { + if (olapTable.hasSequenceCol() && !hasSequenceCol() && getMergeType() != LoadTask.MergeType.DELETE) { throw new AnalysisException("Table " + olapTable.getName() + " has sequence column, need to specify the sequence column"); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/StreamLoadPlanner.java b/fe/fe-core/src/main/java/org/apache/doris/planner/StreamLoadPlanner.java index 8988d3220c8672..a3c864e9207cbe 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/StreamLoadPlanner.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/StreamLoadPlanner.java @@ -127,7 +127,8 @@ public TPipelineFragmentParams plan(TUniqueId loadId, int fragmentInstanceIdInde throw new AnalysisException("load by MERGE or DELETE need to upgrade table to support batch delete."); } - if (destTable.hasSequenceCol() && !taskInfo.hasSequenceCol() && destTable.getSequenceMapCol() == null) { + if (destTable.hasSequenceCol() && !taskInfo.hasSequenceCol() && destTable.getSequenceMapCol() == null + && taskInfo.getMergeType() != LoadTask.MergeType.DELETE) { throw new UserException("Table " + destTable.getName() + " has sequence column, need to specify the sequence column"); } diff --git a/regression-test/data/unique_with_mow_p0/partial_update/partial_update_delete.csv b/regression-test/data/unique_with_mow_p0/partial_update/partial_update_delete.csv index 5f5fbe759f10b9..e22b5b747a60e7 100644 --- a/regression-test/data/unique_with_mow_p0/partial_update/partial_update_delete.csv +++ b/regression-test/data/unique_with_mow_p0/partial_update/partial_update_delete.csv @@ -1,3 +1,4 @@ 1 2 -3 \ No newline at end of file +3 +7 \ No newline at end of file diff --git a/regression-test/data/unique_with_mow_p0/partial_update/test_partial_update_delete.out b/regression-test/data/unique_with_mow_p0/partial_update/test_partial_update_delete.out index e1623d42460181..df4b9208c0ee84 100644 --- a/regression-test/data/unique_with_mow_p0/partial_update/test_partial_update_delete.out +++ b/regression-test/data/unique_with_mow_p0/partial_update/test_partial_update_delete.out @@ -1,21 +1,21 @@ -- This file is automatically generated. You should know what you did if you want to edit this --- !sql -- +-- !sql1 -- 1 1 1 1 1 2 2 2 2 2 3 3 3 3 3 4 4 4 4 4 5 5 5 5 5 --- !sql -- +-- !sql1 -- 2 2 2 2 2 4 4 4 4 4 5 5 5 5 5 --- !sql -- +-- !sql1 -- 4 4 4 4 4 5 5 5 5 5 --- !with_delete_sign -- +-- !with_delete_sign1 -- 1 \N \N 0 \N 1 1 1 1 1 1 0 2 \N \N 0 \N 1 @@ -25,44 +25,55 @@ 4 4 4 4 4 0 5 5 5 5 5 0 --- !sql -- +-- !sql2 -- 1 1 1 1 1 2 2 2 2 2 3 3 3 3 3 4 4 4 4 4 5 5 5 5 5 +6 6 6 6 6 --- !sql -- +-- !sql2 -- 4 4 4 4 4 5 5 5 5 5 +6 6 6 6 6 --- !sql -- -1 \N \N \N \N 1 +-- !sql2 -- +5 5 5 5 5 +6 6 6 6 6 + +-- !sql2 -- +1 \N \N 0 \N 1 1 1 1 1 1 0 -2 \N \N \N \N 1 +2 \N \N 0 \N 1 2 2 2 2 2 0 -3 \N \N \N \N 1 +3 \N \N 0 \N 1 3 3 3 3 3 0 +4 \N \N 0 \N 1 4 4 4 4 4 0 5 5 5 5 5 0 +6 6 6 6 6 0 +7 \N \N 0 \N 1 +8 \N \N 0 \N 1 +9 \N \N 0 \N 1 --- !sql -- +-- !sql1 -- 1 1 1 1 1 2 2 2 2 2 3 3 3 3 3 4 4 4 4 4 5 5 5 5 5 --- !sql -- +-- !sql1 -- 2 2 2 2 2 4 4 4 4 4 5 5 5 5 5 --- !sql -- +-- !sql1 -- 4 4 4 4 4 5 5 5 5 5 --- !with_delete_sign -- +-- !with_delete_sign1 -- 1 \N \N 0 \N 1 1 1 1 1 1 0 2 \N \N 0 \N 1 @@ -72,24 +83,35 @@ 4 4 4 4 4 0 5 5 5 5 5 0 --- !sql -- +-- !sql2 -- 1 1 1 1 1 2 2 2 2 2 3 3 3 3 3 4 4 4 4 4 5 5 5 5 5 +6 6 6 6 6 --- !sql -- +-- !sql2 -- 4 4 4 4 4 5 5 5 5 5 +6 6 6 6 6 --- !sql -- -1 \N \N \N \N 1 +-- !sql2 -- +5 5 5 5 5 +6 6 6 6 6 + +-- !sql2 -- +1 \N \N 0 \N 1 1 1 1 1 1 0 -2 \N \N \N \N 1 +2 \N \N 0 \N 1 2 2 2 2 2 0 -3 \N \N \N \N 1 +3 \N \N 0 \N 1 3 3 3 3 3 0 +4 \N \N 0 \N 1 4 4 4 4 4 0 5 5 5 5 5 0 +6 6 6 6 6 0 +7 \N \N 0 \N 1 +8 \N \N 0 \N 1 +9 \N \N 0 \N 1 diff --git a/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_delete.groovy b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_delete.groovy index 0d83d94f91c918..392b1f1d6486ec 100644 --- a/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_delete.groovy +++ b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_delete.groovy @@ -42,6 +42,7 @@ suite('test_partial_update_delete') { "replication_num" = "1", "store_row_column" = "${use_row_store}"); """ + def tableName2 = "test_partial_update_delete2" sql "DROP TABLE IF EXISTS ${tableName2};" sql """ CREATE TABLE IF NOT EXISTS ${tableName2} ( @@ -55,21 +56,22 @@ suite('test_partial_update_delete') { "store_row_column" = "${use_row_store}"); """ sql "insert into ${tableName1} values(1,1,1,1,1),(2,2,2,2,2),(3,3,3,3,3),(4,4,4,4,4),(5,5,5,5,5);" - qt_sql "select * from ${tableName1} order by k1;" + qt_sql1 "select * from ${tableName1} order by k1;" sql "insert into ${tableName2} values(1),(3);" sql "delete from ${tableName1} A using ${tableName2} B where A.k1=B.k;" - qt_sql "select * from ${tableName1} order by k1;" + qt_sql1 "select * from ${tableName1} order by k1;" sql "delete from ${tableName1} where c2=2;" - qt_sql "select * from ${tableName1} order by k1;" + qt_sql1 "select * from ${tableName1} order by k1;" sql "set skip_delete_sign=true;" sql "set skip_storage_engine_merge=true;" sql "set skip_delete_bitmap=true;" - qt_with_delete_sign "select k1,c1,c2,c3,c4,__DORIS_DELETE_SIGN__ from ${tableName1} order by k1,c1,c2,c3,c4,__DORIS_DELETE_SIGN__;" + qt_with_delete_sign1 "select k1,c1,c2,c3,c4,__DORIS_DELETE_SIGN__ from ${tableName1} order by k1,c1,c2,c3,c4,__DORIS_DELETE_SIGN__;" sql "drop table if exists ${tableName1};" sql "drop table if exists ${tableName2};" + sql "set skip_delete_sign=false;" sql "set skip_storage_engine_merge=false;" sql "set skip_delete_bitmap=false;" @@ -79,7 +81,7 @@ suite('test_partial_update_delete') { `k1` int NOT NULL, `c1` int, `c2` int, - `c3` int, + `c3` int NOT NULL, `c4` int )UNIQUE KEY(k1) DISTRIBUTED BY HASH(k1) BUCKETS 1 @@ -89,27 +91,137 @@ suite('test_partial_update_delete') { "replication_num" = "1", "store_row_column" = "${use_row_store}"); """ - sql "insert into ${tableName3} values(1,1,1,1,1),(2,2,2,2,2),(3,3,3,3,3),(4,4,4,4,4),(5,5,5,5,5);" - qt_sql "select k1,c1,c2,c3,c4 from ${tableName3} order by k1,c1,c2,c3,c4;" + sql "insert into ${tableName3} values(1,1,1,1,1),(2,2,2,2,2),(3,3,3,3,3),(4,4,4,4,4),(5,5,5,5,5),(6,6,6,6,6);" + qt_sql2 "select k1,c1,c2,c3,c4 from ${tableName3} order by k1,c1,c2,c3,c4;" streamLoad { table "${tableName3}" + set 'column_separator', ',' + set 'format', 'csv' + set 'columns', 'k1' + set 'partial_columns', 'true' + set 'merge_type', 'DELETE' + file 'partial_update_delete.csv' + time 10000 + } + sql "sync" + qt_sql2 "select k1,c1,c2,c3,c4 from ${tableName3} order by k1,c1,c2,c3,c4;" + + sql "set enable_insert_strict=false;" + sql "set enable_unique_key_partial_update=true;" + sql "sync;" + sql "insert into ${tableName3}(k1, __DORIS_DELETE_SIGN__) values(8,1),(4,1),(9,1);" + qt_sql2 "select k1,c1,c2,c3,c4 from ${tableName3} order by k1,c1,c2,c3,c4;" + sql "set enable_insert_strict=true;" + sql "set enable_unique_key_partial_update=false;" + sql "sync;" + + sql "set skip_delete_sign=true;" + sql "set skip_storage_engine_merge=true;" + sql "set skip_delete_bitmap=true;" + qt_sql2 "select k1,c1,c2,c3,c4,__DORIS_DELETE_SIGN__ from ${tableName3} order by k1,c1,c2,c3,c4,__DORIS_DELETE_SIGN__;" + sql "drop table if exists ${tableName3};" + + sql "set skip_delete_sign=false;" + sql "set skip_storage_engine_merge=false;" + sql "set skip_delete_bitmap=false;" + def tableName4 = "test_partial_update_delete4" + sql "DROP TABLE IF EXISTS ${tableName4};" + sql """ CREATE TABLE IF NOT EXISTS ${tableName4} ( + `k1` int NOT NULL, + `c1` int, + `c2` int, + `c3` int NOT NULL, + `c4` int + )UNIQUE KEY(k1) + DISTRIBUTED BY HASH(k1) BUCKETS 1 + PROPERTIES ( + "enable_unique_key_merge_on_write" = "true", + "disable_auto_compaction" = "true", + "replication_num" = "1", + "store_row_column" = "${use_row_store}", + "function_column.sequence_col" = "c3"); """ + + sql "insert into ${tableName4} values(1,1,1,1,1),(2,2,2,2,2),(3,3,3,3,3),(4,4,4,4,4),(5,5,5,5,5),(6,6,6,6,6);" + qt_sql3 "select k1,c1,c2,c3,c4 from ${tableName4} order by k1,c1,c2,c3,c4;" + streamLoad { + table "${tableName4}" set 'column_separator', ',' set 'format', 'csv' set 'columns', 'k1' set 'partial_columns', 'true' set 'merge_type', 'DELETE' + file 'partial_update_delete.csv' + time 10000 + } + sql "sync" + qt_sql3 "select k1,c1,c2,c3,c4 from ${tableName4} order by k1,c1,c2,c3,c4;" + + sql "set enable_insert_strict=false;" + sql "set enable_unique_key_partial_update=true;" + sql "sync;" + sql "insert into ${tableName4}(k1, __DORIS_DELETE_SIGN__) values(8,1),(4,1),(9,1);" + qt_sql3 "select k1,c1,c2,c3,c4 from ${tableName4} order by k1,c1,c2,c3,c4;" + sql "set enable_insert_strict=true;" + sql "set enable_unique_key_partial_update=false;" + sql "sync;" + + sql "set skip_delete_sign=true;" + sql "set skip_storage_engine_merge=true;" + sql "set skip_delete_bitmap=true;" + qt_sql3 "select k1,c1,c2,c3,c4,__DORIS_DELETE_SIGN__,__DORIS_SEQUENCE_COL__ from ${tableName4} order by k1,c1,c2,c3,c4,__DORIS_DELETE_SIGN__;" + sql "drop table if exists ${tableName4};" + + + sql "set skip_delete_sign=false;" + sql "set skip_storage_engine_merge=false;" + sql "set skip_delete_bitmap=false;" + def tableName5 = "test_partial_update_delete5" + sql "DROP TABLE IF EXISTS ${tableName5};" + sql """ CREATE TABLE IF NOT EXISTS ${tableName5} ( + `k1` int NOT NULL, + `c1` int, + `c2` int, + `c3` int NOT NULL, + `c4` int + )UNIQUE KEY(k1) + DISTRIBUTED BY HASH(k1) BUCKETS 1 + PROPERTIES ( + "enable_unique_key_merge_on_write" = "true", + "disable_auto_compaction" = "true", + "replication_num" = "1", + "store_row_column" = "${use_row_store}", + "function_column.sequence_type" = "int"); """ + sql "insert into ${tableName5}(k1,c1,c2,c3,c4,__DORIS_SEQUENCE_COL__) values(1,1,1,1,1,1),(2,2,2,2,2,2),(3,3,3,3,3,3),(4,4,4,4,4,4),(5,5,5,5,5,5),(6,6,6,6,6,6);" + qt_sql4 "select k1,c1,c2,c3,c4 from ${tableName5} order by k1,c1,c2,c3,c4;" + streamLoad { + table "${tableName5}" + set 'column_separator', ',' + set 'format', 'csv' + set 'columns', 'k1' + set 'partial_columns', 'true' + set 'merge_type', 'DELETE' file 'partial_update_delete.csv' time 10000 } sql "sync" - qt_sql "select k1,c1,c2,c3,c4 from ${tableName3} order by k1,c1,c2,c3,c4;" + qt_sql4 "select k1,c1,c2,c3,c4 from ${tableName5} order by k1,c1,c2,c3,c4;" + + sql "set enable_insert_strict=false;" + sql "set enable_unique_key_partial_update=true;" + sql "sync;" + sql "insert into ${tableName5}(k1, __DORIS_DELETE_SIGN__) values(8,1),(4,1),(9,1);" + qt_sql4 "select k1,c1,c2,c3,c4 from ${tableName5} order by k1,c1,c2,c3,c4;" + sql "set enable_insert_strict=true;" + sql "set enable_unique_key_partial_update=false;" + sql "sync;" + sql "set skip_delete_sign=true;" sql "set skip_storage_engine_merge=true;" sql "set skip_delete_bitmap=true;" - qt_sql "select k1,c1,c2,c3,c4,__DORIS_DELETE_SIGN__ from ${tableName3} order by k1,c1,c2,c3,c4,__DORIS_DELETE_SIGN__;" - sql "drop table if exists ${tableName3};" + qt_sql4 "select k1,c1,c2,c3,c4,__DORIS_DELETE_SIGN__,__DORIS_SEQUENCE_COL__ from ${tableName5} order by k1,c1,c2,c3,c4,__DORIS_DELETE_SIGN__;" + sql "drop table if exists ${tableName5};" } } } From 48355b3a9f563f47466c80f45c37c97fa8cf5401 Mon Sep 17 00:00:00 2001 From: bobhan1 Date: Tue, 3 Sep 2024 19:18:33 +0800 Subject: [PATCH 2/5] fix --- .../doris/analysis/DataDescription.java | 11 +- .../doris/datasource/LoadScanProvider.java | 3 +- .../test_partial_update_delete.out | 128 ++++++++++++++++++ .../test_partial_update_delete.groovy | 1 - 4 files changed, 137 insertions(+), 6 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/DataDescription.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/DataDescription.java index ae568237593e22..a2b3e9050ce69a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/DataDescription.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/DataDescription.java @@ -939,14 +939,17 @@ private void analyzeSequenceCol(String fullDbName) throws AnalysisException { if (olapTable.hasSequenceCol() && olapTable.getSequenceMapCol() != null) { return; } + if (hasSequenceCol() && !olapTable.hasSequenceCol()) { + throw new AnalysisException("There is no sequence column in the table " + olapTable.getName()); + } + if (getMergeType() == LoadTask.MergeType.DELETE) { + return; + } // check olapTable schema and sequenceCol - if (olapTable.hasSequenceCol() && !hasSequenceCol() && getMergeType() != LoadTask.MergeType.DELETE) { + if (olapTable.hasSequenceCol() && !hasSequenceCol()) { throw new AnalysisException("Table " + olapTable.getName() + " has sequence column, need to specify the sequence column"); } - if (hasSequenceCol() && !olapTable.hasSequenceCol()) { - throw new AnalysisException("There is no sequence column in the table " + olapTable.getName()); - } // check source sequence column is in parsedColumnExprList or Table base schema boolean hasSourceSequenceCol = false; if (!parsedColumnExprList.isEmpty()) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/LoadScanProvider.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/LoadScanProvider.java index b4b123760e698c..3042597d05ad4b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/LoadScanProvider.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/LoadScanProvider.java @@ -189,7 +189,8 @@ private void initColumns(FileLoadScanNode.ParamCreateContext context, Analyzer a } // add columnExpr for sequence column TableIf targetTable = getTargetTable(); - if (targetTable instanceof OlapTable && ((OlapTable) targetTable).hasSequenceCol()) { + if (targetTable instanceof OlapTable && ((OlapTable) targetTable).hasSequenceCol() + && context.fileGroup.getMergeType() != LoadTask.MergeType.DELETE) { OlapTable olapTable = (OlapTable) targetTable; String sequenceCol = olapTable.getSequenceMapCol(); if (sequenceCol != null) { diff --git a/regression-test/data/unique_with_mow_p0/partial_update/test_partial_update_delete.out b/regression-test/data/unique_with_mow_p0/partial_update/test_partial_update_delete.out index df4b9208c0ee84..aa8299806c0a6c 100644 --- a/regression-test/data/unique_with_mow_p0/partial_update/test_partial_update_delete.out +++ b/regression-test/data/unique_with_mow_p0/partial_update/test_partial_update_delete.out @@ -57,6 +57,70 @@ 8 \N \N 0 \N 1 9 \N \N 0 \N 1 +-- !sql3 -- +1 1 1 1 1 +2 2 2 2 2 +3 3 3 3 3 +4 4 4 4 4 +5 5 5 5 5 +6 6 6 6 6 + +-- !sql3 -- +4 4 4 4 4 +5 5 5 5 5 +6 6 6 6 6 + +-- !sql3 -- +5 5 5 5 5 +6 6 6 6 6 + +-- !sql3 -- +1 1 1 1 1 0 1 +1 1 1 1 1 1 1 +2 2 2 2 2 0 2 +2 2 2 2 2 1 2 +3 3 3 3 3 0 3 +3 3 3 3 3 1 3 +4 4 4 4 4 0 4 +4 4 4 4 4 1 4 +5 5 5 5 5 0 5 +6 6 6 6 6 0 6 +7 \N \N 0 \N 1 \N +8 \N \N 0 \N 1 \N +9 \N \N 0 \N 1 \N + +-- !sql4 -- +1 1 1 1 1 +2 2 2 2 2 +3 3 3 3 3 +4 4 4 4 4 +5 5 5 5 5 +6 6 6 6 6 + +-- !sql4 -- +4 4 4 4 4 +5 5 5 5 5 +6 6 6 6 6 + +-- !sql4 -- +5 5 5 5 5 +6 6 6 6 6 + +-- !sql4 -- +1 1 1 1 1 0 1 +1 1 1 1 1 1 1 +2 2 2 2 2 0 2 +2 2 2 2 2 1 2 +3 3 3 3 3 0 3 +3 3 3 3 3 1 3 +4 4 4 4 4 0 4 +4 4 4 4 4 1 4 +5 5 5 5 5 0 5 +6 6 6 6 6 0 6 +7 \N \N 0 \N 1 \N +8 \N \N 0 \N 1 \N +9 \N \N 0 \N 1 \N + -- !sql1 -- 1 1 1 1 1 2 2 2 2 2 @@ -115,3 +179,67 @@ 8 \N \N 0 \N 1 9 \N \N 0 \N 1 +-- !sql3 -- +1 1 1 1 1 +2 2 2 2 2 +3 3 3 3 3 +4 4 4 4 4 +5 5 5 5 5 +6 6 6 6 6 + +-- !sql3 -- +4 4 4 4 4 +5 5 5 5 5 +6 6 6 6 6 + +-- !sql3 -- +5 5 5 5 5 +6 6 6 6 6 + +-- !sql3 -- +1 1 1 1 1 0 1 +1 1 1 1 1 1 1 +2 2 2 2 2 0 2 +2 2 2 2 2 1 2 +3 3 3 3 3 0 3 +3 3 3 3 3 1 3 +4 4 4 4 4 0 4 +4 4 4 4 4 1 4 +5 5 5 5 5 0 5 +6 6 6 6 6 0 6 +7 \N \N 0 \N 1 \N +8 \N \N 0 \N 1 \N +9 \N \N 0 \N 1 \N + +-- !sql4 -- +1 1 1 1 1 +2 2 2 2 2 +3 3 3 3 3 +4 4 4 4 4 +5 5 5 5 5 +6 6 6 6 6 + +-- !sql4 -- +4 4 4 4 4 +5 5 5 5 5 +6 6 6 6 6 + +-- !sql4 -- +5 5 5 5 5 +6 6 6 6 6 + +-- !sql4 -- +1 1 1 1 1 0 1 +1 1 1 1 1 1 1 +2 2 2 2 2 0 2 +2 2 2 2 2 1 2 +3 3 3 3 3 0 3 +3 3 3 3 3 1 3 +4 4 4 4 4 0 4 +4 4 4 4 4 1 4 +5 5 5 5 5 0 5 +6 6 6 6 6 0 6 +7 \N \N 0 \N 1 \N +8 \N \N 0 \N 1 \N +9 \N \N 0 \N 1 \N + diff --git a/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_delete.groovy b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_delete.groovy index 392b1f1d6486ec..4fa6f746a33703 100644 --- a/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_delete.groovy +++ b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_delete.groovy @@ -192,7 +192,6 @@ suite('test_partial_update_delete') { "replication_num" = "1", "store_row_column" = "${use_row_store}", "function_column.sequence_type" = "int"); """ - sql "insert into ${tableName5}(k1,c1,c2,c3,c4,__DORIS_SEQUENCE_COL__) values(1,1,1,1,1,1),(2,2,2,2,2,2),(3,3,3,3,3,3),(4,4,4,4,4,4),(5,5,5,5,5,5),(6,6,6,6,6,6);" qt_sql4 "select k1,c1,c2,c3,c4 from ${tableName5} order by k1,c1,c2,c3,c4;" streamLoad { From b05329ea0b131857e6e77913428e956c065cf0f7 Mon Sep 17 00:00:00 2001 From: bobhan1 Date: Tue, 3 Sep 2024 20:28:30 +0800 Subject: [PATCH 3/5] update --- .../doris/analysis/DataDescription.java | 9 +- .../doris/datasource/LoadScanProvider.java | 3 +- .../test_partial_update_delete.groovy | 408 +++++++++--------- 3 files changed, 212 insertions(+), 208 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/DataDescription.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/DataDescription.java index a2b3e9050ce69a..1ee3babc0d16f8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/DataDescription.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/DataDescription.java @@ -939,17 +939,14 @@ private void analyzeSequenceCol(String fullDbName) throws AnalysisException { if (olapTable.hasSequenceCol() && olapTable.getSequenceMapCol() != null) { return; } - if (hasSequenceCol() && !olapTable.hasSequenceCol()) { - throw new AnalysisException("There is no sequence column in the table " + olapTable.getName()); - } - if (getMergeType() == LoadTask.MergeType.DELETE) { - return; - } // check olapTable schema and sequenceCol if (olapTable.hasSequenceCol() && !hasSequenceCol()) { throw new AnalysisException("Table " + olapTable.getName() + " has sequence column, need to specify the sequence column"); } + if (hasSequenceCol() && !olapTable.hasSequenceCol()) { + throw new AnalysisException("There is no sequence column in the table " + olapTable.getName()); + } // check source sequence column is in parsedColumnExprList or Table base schema boolean hasSourceSequenceCol = false; if (!parsedColumnExprList.isEmpty()) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/LoadScanProvider.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/LoadScanProvider.java index 3042597d05ad4b..b4b123760e698c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/LoadScanProvider.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/LoadScanProvider.java @@ -189,8 +189,7 @@ private void initColumns(FileLoadScanNode.ParamCreateContext context, Analyzer a } // add columnExpr for sequence column TableIf targetTable = getTargetTable(); - if (targetTable instanceof OlapTable && ((OlapTable) targetTable).hasSequenceCol() - && context.fileGroup.getMergeType() != LoadTask.MergeType.DELETE) { + if (targetTable instanceof OlapTable && ((OlapTable) targetTable).hasSequenceCol()) { OlapTable olapTable = (OlapTable) targetTable; String sequenceCol = olapTable.getSequenceMapCol(); if (sequenceCol != null) { diff --git a/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_delete.groovy b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_delete.groovy index 4fa6f746a33703..738c4119e2dad8 100644 --- a/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_delete.groovy +++ b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_delete.groovy @@ -20,207 +20,215 @@ suite('test_partial_update_delete') { String db = context.config.getDbNameByFile(context.file) sql "select 1;" // to create database - for (def use_row_store : [false, true]) { - logger.info("current params: use_row_store: ${use_row_store}") - - connect(user = context.config.jdbcUser, password = context.config.jdbcPassword, url = context.config.jdbcUrl) { - sql "use ${db};" - - def tableName1 = "test_partial_update_delete1" - sql "DROP TABLE IF EXISTS ${tableName1};" - sql """ CREATE TABLE IF NOT EXISTS ${tableName1} ( - `k1` int NOT NULL, - `c1` int, - `c2` int, - `c3` int NOT NULL, - `c4` int - )UNIQUE KEY(k1) - DISTRIBUTED BY HASH(k1) BUCKETS 1 - PROPERTIES ( - "enable_unique_key_merge_on_write" = "true", - "disable_auto_compaction" = "true", - "replication_num" = "1", - "store_row_column" = "${use_row_store}"); """ - - - def tableName2 = "test_partial_update_delete2" - sql "DROP TABLE IF EXISTS ${tableName2};" - sql """ CREATE TABLE IF NOT EXISTS ${tableName2} ( - `k` BIGINT NULL - ) UNIQUE KEY(k) - DISTRIBUTED BY HASH(k) BUCKETS 1 - PROPERTIES ( - "enable_unique_key_merge_on_write" = "true", - "disable_auto_compaction" = "true", - "replication_num" = "1", - "store_row_column" = "${use_row_store}"); """ - - sql "insert into ${tableName1} values(1,1,1,1,1),(2,2,2,2,2),(3,3,3,3,3),(4,4,4,4,4),(5,5,5,5,5);" - qt_sql1 "select * from ${tableName1} order by k1;" - sql "insert into ${tableName2} values(1),(3);" - sql "delete from ${tableName1} A using ${tableName2} B where A.k1=B.k;" - qt_sql1 "select * from ${tableName1} order by k1;" - - sql "delete from ${tableName1} where c2=2;" - qt_sql1 "select * from ${tableName1} order by k1;" - - sql "set skip_delete_sign=true;" - sql "set skip_storage_engine_merge=true;" - sql "set skip_delete_bitmap=true;" - qt_with_delete_sign1 "select k1,c1,c2,c3,c4,__DORIS_DELETE_SIGN__ from ${tableName1} order by k1,c1,c2,c3,c4,__DORIS_DELETE_SIGN__;" - sql "drop table if exists ${tableName1};" - sql "drop table if exists ${tableName2};" - - - sql "set skip_delete_sign=false;" - sql "set skip_storage_engine_merge=false;" - sql "set skip_delete_bitmap=false;" - def tableName3 = "test_partial_update_delete3" - sql "DROP TABLE IF EXISTS ${tableName3};" - sql """ CREATE TABLE IF NOT EXISTS ${tableName3} ( - `k1` int NOT NULL, - `c1` int, - `c2` int, - `c3` int NOT NULL, - `c4` int - )UNIQUE KEY(k1) - DISTRIBUTED BY HASH(k1) BUCKETS 1 - PROPERTIES ( - "enable_unique_key_merge_on_write" = "true", - "disable_auto_compaction" = "true", - "replication_num" = "1", - "store_row_column" = "${use_row_store}"); """ - - sql "insert into ${tableName3} values(1,1,1,1,1),(2,2,2,2,2),(3,3,3,3,3),(4,4,4,4,4),(5,5,5,5,5),(6,6,6,6,6);" - qt_sql2 "select k1,c1,c2,c3,c4 from ${tableName3} order by k1,c1,c2,c3,c4;" - streamLoad { - table "${tableName3}" - set 'column_separator', ',' - set 'format', 'csv' - set 'columns', 'k1' - set 'partial_columns', 'true' - set 'merge_type', 'DELETE' - file 'partial_update_delete.csv' - time 10000 + for (def use_nereids : [true, false]) { + for (def use_row_store : [false, true]) { + logger.info("current params: use_nereids: ${use_nereids}, use_row_store: ${use_row_store}") + connect(user = context.config.jdbcUser, password = context.config.jdbcPassword, url = context.config.jdbcUrl) { + sql "use ${db};" + if (use_nereids) { + sql "set enable_nereids_planner=true" + sql "set enable_fallback_to_original_planner=false" + } else { + sql "set enable_nereids_planner=false" + } + sql "sync;" + + def tableName1 = "test_partial_update_delete1" + sql "DROP TABLE IF EXISTS ${tableName1};" + sql """ CREATE TABLE IF NOT EXISTS ${tableName1} ( + `k1` int NOT NULL, + `c1` int, + `c2` int, + `c3` int NOT NULL, + `c4` int + )UNIQUE KEY(k1) + DISTRIBUTED BY HASH(k1) BUCKETS 1 + PROPERTIES ( + "enable_unique_key_merge_on_write" = "true", + "disable_auto_compaction" = "true", + "replication_num" = "1", + "store_row_column" = "${use_row_store}"); """ + + + def tableName2 = "test_partial_update_delete2" + sql "DROP TABLE IF EXISTS ${tableName2};" + sql """ CREATE TABLE IF NOT EXISTS ${tableName2} ( + `k` BIGINT NULL + ) UNIQUE KEY(k) + DISTRIBUTED BY HASH(k) BUCKETS 1 + PROPERTIES ( + "enable_unique_key_merge_on_write" = "true", + "disable_auto_compaction" = "true", + "replication_num" = "1", + "store_row_column" = "${use_row_store}"); """ + + sql "insert into ${tableName1} values(1,1,1,1,1),(2,2,2,2,2),(3,3,3,3,3),(4,4,4,4,4),(5,5,5,5,5);" + qt_sql1 "select * from ${tableName1} order by k1;" + sql "insert into ${tableName2} values(1),(3);" + sql "delete from ${tableName1} A using ${tableName2} B where A.k1=B.k;" + qt_sql1 "select * from ${tableName1} order by k1;" + + sql "delete from ${tableName1} where c2=2;" + qt_sql1 "select * from ${tableName1} order by k1;" + + sql "set skip_delete_sign=true;" + sql "set skip_storage_engine_merge=true;" + sql "set skip_delete_bitmap=true;" + qt_with_delete_sign1 "select k1,c1,c2,c3,c4,__DORIS_DELETE_SIGN__ from ${tableName1} order by k1,c1,c2,c3,c4,__DORIS_DELETE_SIGN__;" + sql "drop table if exists ${tableName1};" + sql "drop table if exists ${tableName2};" + + + sql "set skip_delete_sign=false;" + sql "set skip_storage_engine_merge=false;" + sql "set skip_delete_bitmap=false;" + def tableName3 = "test_partial_update_delete3" + sql "DROP TABLE IF EXISTS ${tableName3};" + sql """ CREATE TABLE IF NOT EXISTS ${tableName3} ( + `k1` int NOT NULL, + `c1` int, + `c2` int, + `c3` int NOT NULL, + `c4` int + )UNIQUE KEY(k1) + DISTRIBUTED BY HASH(k1) BUCKETS 1 + PROPERTIES ( + "enable_unique_key_merge_on_write" = "true", + "disable_auto_compaction" = "true", + "replication_num" = "1", + "store_row_column" = "${use_row_store}"); """ + + sql "insert into ${tableName3} values(1,1,1,1,1),(2,2,2,2,2),(3,3,3,3,3),(4,4,4,4,4),(5,5,5,5,5),(6,6,6,6,6);" + qt_sql2 "select k1,c1,c2,c3,c4 from ${tableName3} order by k1,c1,c2,c3,c4;" + streamLoad { + table "${tableName3}" + set 'column_separator', ',' + set 'format', 'csv' + set 'columns', 'k1' + set 'partial_columns', 'true' + set 'merge_type', 'DELETE' + file 'partial_update_delete.csv' + time 10000 + } + sql "sync" + qt_sql2 "select k1,c1,c2,c3,c4 from ${tableName3} order by k1,c1,c2,c3,c4;" + + sql "set enable_insert_strict=false;" + sql "set enable_unique_key_partial_update=true;" + sql "sync;" + sql "insert into ${tableName3}(k1, __DORIS_DELETE_SIGN__) values(8,1),(4,1),(9,1);" + qt_sql2 "select k1,c1,c2,c3,c4 from ${tableName3} order by k1,c1,c2,c3,c4;" + sql "set enable_insert_strict=true;" + sql "set enable_unique_key_partial_update=false;" + sql "sync;" + + sql "set skip_delete_sign=true;" + sql "set skip_storage_engine_merge=true;" + sql "set skip_delete_bitmap=true;" + qt_sql2 "select k1,c1,c2,c3,c4,__DORIS_DELETE_SIGN__ from ${tableName3} order by k1,c1,c2,c3,c4,__DORIS_DELETE_SIGN__;" + sql "drop table if exists ${tableName3};" + + + // sql "set skip_delete_sign=false;" + // sql "set skip_storage_engine_merge=false;" + // sql "set skip_delete_bitmap=false;" + // def tableName4 = "test_partial_update_delete4" + // sql "DROP TABLE IF EXISTS ${tableName4};" + // sql """ CREATE TABLE IF NOT EXISTS ${tableName4} ( + // `k1` int NOT NULL, + // `c1` int, + // `c2` int, + // `c3` int NOT NULL, + // `c4` int + // )UNIQUE KEY(k1) + // DISTRIBUTED BY HASH(k1) BUCKETS 1 + // PROPERTIES ( + // "enable_unique_key_merge_on_write" = "true", + // "disable_auto_compaction" = "true", + // "replication_num" = "1", + // "store_row_column" = "${use_row_store}", + // "function_column.sequence_col" = "c3"); """ + + // sql "insert into ${tableName4} values(1,1,1,1,1),(2,2,2,2,2),(3,3,3,3,3),(4,4,4,4,4),(5,5,5,5,5),(6,6,6,6,6);" + // qt_sql3 "select k1,c1,c2,c3,c4 from ${tableName4} order by k1,c1,c2,c3,c4;" + // streamLoad { + // table "${tableName4}" + // set 'column_separator', ',' + // set 'format', 'csv' + // set 'columns', 'k1' + // set 'partial_columns', 'true' + // set 'merge_type', 'DELETE' + // file 'partial_update_delete.csv' + // time 10000 + // } + // sql "sync" + // qt_sql3 "select k1,c1,c2,c3,c4 from ${tableName4} order by k1,c1,c2,c3,c4;" + + // sql "set enable_insert_strict=false;" + // sql "set enable_unique_key_partial_update=true;" + // sql "sync;" + // sql "insert into ${tableName4}(k1, __DORIS_DELETE_SIGN__) values(8,1),(4,1),(9,1);" + // qt_sql3 "select k1,c1,c2,c3,c4 from ${tableName4} order by k1,c1,c2,c3,c4;" + // sql "set enable_insert_strict=true;" + // sql "set enable_unique_key_partial_update=false;" + // sql "sync;" + + // sql "set skip_delete_sign=true;" + // sql "set skip_storage_engine_merge=true;" + // sql "set skip_delete_bitmap=true;" + // qt_sql3 "select k1,c1,c2,c3,c4,__DORIS_DELETE_SIGN__,__DORIS_SEQUENCE_COL__ from ${tableName4} order by k1,c1,c2,c3,c4,__DORIS_DELETE_SIGN__;" + // sql "drop table if exists ${tableName4};" + + + // sql "set skip_delete_sign=false;" + // sql "set skip_storage_engine_merge=false;" + // sql "set skip_delete_bitmap=false;" + // def tableName5 = "test_partial_update_delete5" + // sql "DROP TABLE IF EXISTS ${tableName5};" + // sql """ CREATE TABLE IF NOT EXISTS ${tableName5} ( + // `k1` int NOT NULL, + // `c1` int, + // `c2` int, + // `c3` int NOT NULL, + // `c4` int + // )UNIQUE KEY(k1) + // DISTRIBUTED BY HASH(k1) BUCKETS 1 + // PROPERTIES ( + // "enable_unique_key_merge_on_write" = "true", + // "disable_auto_compaction" = "true", + // "replication_num" = "1", + // "store_row_column" = "${use_row_store}", + // "function_column.sequence_type" = "int"); """ + // sql "insert into ${tableName5}(k1,c1,c2,c3,c4,__DORIS_SEQUENCE_COL__) values(1,1,1,1,1,1),(2,2,2,2,2,2),(3,3,3,3,3,3),(4,4,4,4,4,4),(5,5,5,5,5,5),(6,6,6,6,6,6);" + // qt_sql4 "select k1,c1,c2,c3,c4 from ${tableName5} order by k1,c1,c2,c3,c4;" + // streamLoad { + // table "${tableName5}" + // set 'column_separator', ',' + // set 'format', 'csv' + // set 'columns', 'k1' + // set 'partial_columns', 'true' + // set 'merge_type', 'DELETE' + // file 'partial_update_delete.csv' + // time 10000 + // } + // sql "sync" + // qt_sql4 "select k1,c1,c2,c3,c4 from ${tableName5} order by k1,c1,c2,c3,c4;" + + // sql "set enable_insert_strict=false;" + // sql "set enable_unique_key_partial_update=true;" + // sql "sync;" + // sql "insert into ${tableName5}(k1, __DORIS_DELETE_SIGN__) values(8,1),(4,1),(9,1);" + // qt_sql4 "select k1,c1,c2,c3,c4 from ${tableName5} order by k1,c1,c2,c3,c4;" + // sql "set enable_insert_strict=true;" + // sql "set enable_unique_key_partial_update=false;" + // sql "sync;" + + // sql "set skip_delete_sign=true;" + // sql "set skip_storage_engine_merge=true;" + // sql "set skip_delete_bitmap=true;" + // qt_sql4 "select k1,c1,c2,c3,c4,__DORIS_DELETE_SIGN__,__DORIS_SEQUENCE_COL__ from ${tableName5} order by k1,c1,c2,c3,c4,__DORIS_DELETE_SIGN__;" + // sql "drop table if exists ${tableName5};" } - sql "sync" - qt_sql2 "select k1,c1,c2,c3,c4 from ${tableName3} order by k1,c1,c2,c3,c4;" - - sql "set enable_insert_strict=false;" - sql "set enable_unique_key_partial_update=true;" - sql "sync;" - sql "insert into ${tableName3}(k1, __DORIS_DELETE_SIGN__) values(8,1),(4,1),(9,1);" - qt_sql2 "select k1,c1,c2,c3,c4 from ${tableName3} order by k1,c1,c2,c3,c4;" - sql "set enable_insert_strict=true;" - sql "set enable_unique_key_partial_update=false;" - sql "sync;" - - sql "set skip_delete_sign=true;" - sql "set skip_storage_engine_merge=true;" - sql "set skip_delete_bitmap=true;" - qt_sql2 "select k1,c1,c2,c3,c4,__DORIS_DELETE_SIGN__ from ${tableName3} order by k1,c1,c2,c3,c4,__DORIS_DELETE_SIGN__;" - sql "drop table if exists ${tableName3};" - - - sql "set skip_delete_sign=false;" - sql "set skip_storage_engine_merge=false;" - sql "set skip_delete_bitmap=false;" - def tableName4 = "test_partial_update_delete4" - sql "DROP TABLE IF EXISTS ${tableName4};" - sql """ CREATE TABLE IF NOT EXISTS ${tableName4} ( - `k1` int NOT NULL, - `c1` int, - `c2` int, - `c3` int NOT NULL, - `c4` int - )UNIQUE KEY(k1) - DISTRIBUTED BY HASH(k1) BUCKETS 1 - PROPERTIES ( - "enable_unique_key_merge_on_write" = "true", - "disable_auto_compaction" = "true", - "replication_num" = "1", - "store_row_column" = "${use_row_store}", - "function_column.sequence_col" = "c3"); """ - - sql "insert into ${tableName4} values(1,1,1,1,1),(2,2,2,2,2),(3,3,3,3,3),(4,4,4,4,4),(5,5,5,5,5),(6,6,6,6,6);" - qt_sql3 "select k1,c1,c2,c3,c4 from ${tableName4} order by k1,c1,c2,c3,c4;" - streamLoad { - table "${tableName4}" - set 'column_separator', ',' - set 'format', 'csv' - set 'columns', 'k1' - set 'partial_columns', 'true' - set 'merge_type', 'DELETE' - file 'partial_update_delete.csv' - time 10000 - } - sql "sync" - qt_sql3 "select k1,c1,c2,c3,c4 from ${tableName4} order by k1,c1,c2,c3,c4;" - - sql "set enable_insert_strict=false;" - sql "set enable_unique_key_partial_update=true;" - sql "sync;" - sql "insert into ${tableName4}(k1, __DORIS_DELETE_SIGN__) values(8,1),(4,1),(9,1);" - qt_sql3 "select k1,c1,c2,c3,c4 from ${tableName4} order by k1,c1,c2,c3,c4;" - sql "set enable_insert_strict=true;" - sql "set enable_unique_key_partial_update=false;" - sql "sync;" - - sql "set skip_delete_sign=true;" - sql "set skip_storage_engine_merge=true;" - sql "set skip_delete_bitmap=true;" - qt_sql3 "select k1,c1,c2,c3,c4,__DORIS_DELETE_SIGN__,__DORIS_SEQUENCE_COL__ from ${tableName4} order by k1,c1,c2,c3,c4,__DORIS_DELETE_SIGN__;" - sql "drop table if exists ${tableName4};" - - - sql "set skip_delete_sign=false;" - sql "set skip_storage_engine_merge=false;" - sql "set skip_delete_bitmap=false;" - def tableName5 = "test_partial_update_delete5" - sql "DROP TABLE IF EXISTS ${tableName5};" - sql """ CREATE TABLE IF NOT EXISTS ${tableName5} ( - `k1` int NOT NULL, - `c1` int, - `c2` int, - `c3` int NOT NULL, - `c4` int - )UNIQUE KEY(k1) - DISTRIBUTED BY HASH(k1) BUCKETS 1 - PROPERTIES ( - "enable_unique_key_merge_on_write" = "true", - "disable_auto_compaction" = "true", - "replication_num" = "1", - "store_row_column" = "${use_row_store}", - "function_column.sequence_type" = "int"); """ - sql "insert into ${tableName5}(k1,c1,c2,c3,c4,__DORIS_SEQUENCE_COL__) values(1,1,1,1,1,1),(2,2,2,2,2,2),(3,3,3,3,3,3),(4,4,4,4,4,4),(5,5,5,5,5,5),(6,6,6,6,6,6);" - qt_sql4 "select k1,c1,c2,c3,c4 from ${tableName5} order by k1,c1,c2,c3,c4;" - streamLoad { - table "${tableName5}" - set 'column_separator', ',' - set 'format', 'csv' - set 'columns', 'k1' - set 'partial_columns', 'true' - set 'merge_type', 'DELETE' - file 'partial_update_delete.csv' - time 10000 - } - sql "sync" - qt_sql4 "select k1,c1,c2,c3,c4 from ${tableName5} order by k1,c1,c2,c3,c4;" - - sql "set enable_insert_strict=false;" - sql "set enable_unique_key_partial_update=true;" - sql "sync;" - sql "insert into ${tableName5}(k1, __DORIS_DELETE_SIGN__) values(8,1),(4,1),(9,1);" - qt_sql4 "select k1,c1,c2,c3,c4 from ${tableName5} order by k1,c1,c2,c3,c4;" - sql "set enable_insert_strict=true;" - sql "set enable_unique_key_partial_update=false;" - sql "sync;" - - sql "set skip_delete_sign=true;" - sql "set skip_storage_engine_merge=true;" - sql "set skip_delete_bitmap=true;" - qt_sql4 "select k1,c1,c2,c3,c4,__DORIS_DELETE_SIGN__,__DORIS_SEQUENCE_COL__ from ${tableName5} order by k1,c1,c2,c3,c4,__DORIS_DELETE_SIGN__;" - sql "drop table if exists ${tableName5};" } } } From 4ac41229f1ae1e102dfcdb19cf426a7af5a8fb9d Mon Sep 17 00:00:00 2001 From: bobhan1 Date: Tue, 3 Sep 2024 21:33:16 +0800 Subject: [PATCH 4/5] fix --- .../test_partial_update_delete.out | 160 ++++++++++++++- .../test_partial_update_delete.groovy | 193 +++++++++--------- 2 files changed, 248 insertions(+), 105 deletions(-) diff --git a/regression-test/data/unique_with_mow_p0/partial_update/test_partial_update_delete.out b/regression-test/data/unique_with_mow_p0/partial_update/test_partial_update_delete.out index aa8299806c0a6c..7a639b8d6e29db 100644 --- a/regression-test/data/unique_with_mow_p0/partial_update/test_partial_update_delete.out +++ b/regression-test/data/unique_with_mow_p0/partial_update/test_partial_update_delete.out @@ -97,16 +97,82 @@ 5 5 5 5 5 6 6 6 6 6 --- !sql4 -- +-- !sql1 -- +1 1 1 1 1 +2 2 2 2 2 +3 3 3 3 3 +4 4 4 4 4 +5 5 5 5 5 + +-- !sql1 -- +2 2 2 2 2 +4 4 4 4 4 +5 5 5 5 5 + +-- !sql1 -- +4 4 4 4 4 +5 5 5 5 5 + +-- !with_delete_sign1 -- +1 \N \N 0 \N 1 +1 1 1 1 1 0 +2 \N \N 0 \N 1 +2 2 2 2 2 0 +3 \N \N 0 \N 1 +3 3 3 3 3 0 +4 4 4 4 4 0 +5 5 5 5 5 0 + +-- !sql2 -- +1 1 1 1 1 +2 2 2 2 2 +3 3 3 3 3 4 4 4 4 4 5 5 5 5 5 6 6 6 6 6 --- !sql4 -- +-- !sql2 -- +4 4 4 4 4 5 5 5 5 5 6 6 6 6 6 --- !sql4 -- +-- !sql2 -- +5 5 5 5 5 +6 6 6 6 6 + +-- !sql2 -- +1 \N \N 0 \N 1 +1 1 1 1 1 0 +2 \N \N 0 \N 1 +2 2 2 2 2 0 +3 \N \N 0 \N 1 +3 3 3 3 3 0 +4 \N \N 0 \N 1 +4 4 4 4 4 0 +5 5 5 5 5 0 +6 6 6 6 6 0 +7 \N \N 0 \N 1 +8 \N \N 0 \N 1 +9 \N \N 0 \N 1 + +-- !sql3 -- +1 1 1 1 1 +2 2 2 2 2 +3 3 3 3 3 +4 4 4 4 4 +5 5 5 5 5 +6 6 6 6 6 + +-- !sql3 -- +4 4 4 4 4 +5 5 5 5 5 +6 6 6 6 6 + +-- !sql3 -- +5 5 5 5 5 +6 6 6 6 6 + +-- !sql3 -- 1 1 1 1 1 0 1 1 1 1 1 1 1 1 2 2 2 2 2 0 2 @@ -121,6 +187,14 @@ 8 \N \N 0 \N 1 \N 9 \N \N 0 \N 1 \N +-- !sql4 -- +1 1 1 1 1 +2 2 2 2 2 +3 3 3 3 3 +4 4 4 4 4 +5 5 5 5 5 +6 6 6 6 6 + -- !sql1 -- 1 1 1 1 1 2 2 2 2 2 @@ -219,16 +293,82 @@ 5 5 5 5 5 6 6 6 6 6 --- !sql4 -- +-- !sql1 -- +1 1 1 1 1 +2 2 2 2 2 +3 3 3 3 3 +4 4 4 4 4 +5 5 5 5 5 + +-- !sql1 -- +2 2 2 2 2 +4 4 4 4 4 +5 5 5 5 5 + +-- !sql1 -- +4 4 4 4 4 +5 5 5 5 5 + +-- !with_delete_sign1 -- +1 \N \N 0 \N 1 +1 1 1 1 1 0 +2 \N \N 0 \N 1 +2 2 2 2 2 0 +3 \N \N 0 \N 1 +3 3 3 3 3 0 +4 4 4 4 4 0 +5 5 5 5 5 0 + +-- !sql2 -- +1 1 1 1 1 +2 2 2 2 2 +3 3 3 3 3 4 4 4 4 4 5 5 5 5 5 6 6 6 6 6 --- !sql4 -- +-- !sql2 -- +4 4 4 4 4 5 5 5 5 5 6 6 6 6 6 --- !sql4 -- +-- !sql2 -- +5 5 5 5 5 +6 6 6 6 6 + +-- !sql2 -- +1 \N \N 0 \N 1 +1 1 1 1 1 0 +2 \N \N 0 \N 1 +2 2 2 2 2 0 +3 \N \N 0 \N 1 +3 3 3 3 3 0 +4 \N \N 0 \N 1 +4 4 4 4 4 0 +5 5 5 5 5 0 +6 6 6 6 6 0 +7 \N \N 0 \N 1 +8 \N \N 0 \N 1 +9 \N \N 0 \N 1 + +-- !sql3 -- +1 1 1 1 1 +2 2 2 2 2 +3 3 3 3 3 +4 4 4 4 4 +5 5 5 5 5 +6 6 6 6 6 + +-- !sql3 -- +4 4 4 4 4 +5 5 5 5 5 +6 6 6 6 6 + +-- !sql3 -- +5 5 5 5 5 +6 6 6 6 6 + +-- !sql3 -- 1 1 1 1 1 0 1 1 1 1 1 1 1 1 2 2 2 2 2 0 2 @@ -243,3 +383,11 @@ 8 \N \N 0 \N 1 \N 9 \N \N 0 \N 1 \N +-- !sql4 -- +1 1 1 1 1 +2 2 2 2 2 +3 3 3 3 3 +4 4 4 4 4 +5 5 5 5 5 +6 6 6 6 6 + diff --git a/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_delete.groovy b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_delete.groovy index 738c4119e2dad8..fc59038122a1d2 100644 --- a/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_delete.groovy +++ b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_delete.groovy @@ -129,105 +129,100 @@ suite('test_partial_update_delete') { sql "drop table if exists ${tableName3};" - // sql "set skip_delete_sign=false;" - // sql "set skip_storage_engine_merge=false;" - // sql "set skip_delete_bitmap=false;" - // def tableName4 = "test_partial_update_delete4" - // sql "DROP TABLE IF EXISTS ${tableName4};" - // sql """ CREATE TABLE IF NOT EXISTS ${tableName4} ( - // `k1` int NOT NULL, - // `c1` int, - // `c2` int, - // `c3` int NOT NULL, - // `c4` int - // )UNIQUE KEY(k1) - // DISTRIBUTED BY HASH(k1) BUCKETS 1 - // PROPERTIES ( - // "enable_unique_key_merge_on_write" = "true", - // "disable_auto_compaction" = "true", - // "replication_num" = "1", - // "store_row_column" = "${use_row_store}", - // "function_column.sequence_col" = "c3"); """ - - // sql "insert into ${tableName4} values(1,1,1,1,1),(2,2,2,2,2),(3,3,3,3,3),(4,4,4,4,4),(5,5,5,5,5),(6,6,6,6,6);" - // qt_sql3 "select k1,c1,c2,c3,c4 from ${tableName4} order by k1,c1,c2,c3,c4;" - // streamLoad { - // table "${tableName4}" - // set 'column_separator', ',' - // set 'format', 'csv' - // set 'columns', 'k1' - // set 'partial_columns', 'true' - // set 'merge_type', 'DELETE' - // file 'partial_update_delete.csv' - // time 10000 - // } - // sql "sync" - // qt_sql3 "select k1,c1,c2,c3,c4 from ${tableName4} order by k1,c1,c2,c3,c4;" - - // sql "set enable_insert_strict=false;" - // sql "set enable_unique_key_partial_update=true;" - // sql "sync;" - // sql "insert into ${tableName4}(k1, __DORIS_DELETE_SIGN__) values(8,1),(4,1),(9,1);" - // qt_sql3 "select k1,c1,c2,c3,c4 from ${tableName4} order by k1,c1,c2,c3,c4;" - // sql "set enable_insert_strict=true;" - // sql "set enable_unique_key_partial_update=false;" - // sql "sync;" - - // sql "set skip_delete_sign=true;" - // sql "set skip_storage_engine_merge=true;" - // sql "set skip_delete_bitmap=true;" - // qt_sql3 "select k1,c1,c2,c3,c4,__DORIS_DELETE_SIGN__,__DORIS_SEQUENCE_COL__ from ${tableName4} order by k1,c1,c2,c3,c4,__DORIS_DELETE_SIGN__;" - // sql "drop table if exists ${tableName4};" - - - // sql "set skip_delete_sign=false;" - // sql "set skip_storage_engine_merge=false;" - // sql "set skip_delete_bitmap=false;" - // def tableName5 = "test_partial_update_delete5" - // sql "DROP TABLE IF EXISTS ${tableName5};" - // sql """ CREATE TABLE IF NOT EXISTS ${tableName5} ( - // `k1` int NOT NULL, - // `c1` int, - // `c2` int, - // `c3` int NOT NULL, - // `c4` int - // )UNIQUE KEY(k1) - // DISTRIBUTED BY HASH(k1) BUCKETS 1 - // PROPERTIES ( - // "enable_unique_key_merge_on_write" = "true", - // "disable_auto_compaction" = "true", - // "replication_num" = "1", - // "store_row_column" = "${use_row_store}", - // "function_column.sequence_type" = "int"); """ - // sql "insert into ${tableName5}(k1,c1,c2,c3,c4,__DORIS_SEQUENCE_COL__) values(1,1,1,1,1,1),(2,2,2,2,2,2),(3,3,3,3,3,3),(4,4,4,4,4,4),(5,5,5,5,5,5),(6,6,6,6,6,6);" - // qt_sql4 "select k1,c1,c2,c3,c4 from ${tableName5} order by k1,c1,c2,c3,c4;" - // streamLoad { - // table "${tableName5}" - // set 'column_separator', ',' - // set 'format', 'csv' - // set 'columns', 'k1' - // set 'partial_columns', 'true' - // set 'merge_type', 'DELETE' - // file 'partial_update_delete.csv' - // time 10000 - // } - // sql "sync" - // qt_sql4 "select k1,c1,c2,c3,c4 from ${tableName5} order by k1,c1,c2,c3,c4;" - - // sql "set enable_insert_strict=false;" - // sql "set enable_unique_key_partial_update=true;" - // sql "sync;" - // sql "insert into ${tableName5}(k1, __DORIS_DELETE_SIGN__) values(8,1),(4,1),(9,1);" - // qt_sql4 "select k1,c1,c2,c3,c4 from ${tableName5} order by k1,c1,c2,c3,c4;" - // sql "set enable_insert_strict=true;" - // sql "set enable_unique_key_partial_update=false;" - // sql "sync;" - - // sql "set skip_delete_sign=true;" - // sql "set skip_storage_engine_merge=true;" - // sql "set skip_delete_bitmap=true;" - // qt_sql4 "select k1,c1,c2,c3,c4,__DORIS_DELETE_SIGN__,__DORIS_SEQUENCE_COL__ from ${tableName5} order by k1,c1,c2,c3,c4,__DORIS_DELETE_SIGN__;" - // sql "drop table if exists ${tableName5};" + sql "set skip_delete_sign=false;" + sql "set skip_storage_engine_merge=false;" + sql "set skip_delete_bitmap=false;" + def tableName4 = "test_partial_update_delete4" + sql "DROP TABLE IF EXISTS ${tableName4};" + sql """ CREATE TABLE IF NOT EXISTS ${tableName4} ( + `k1` int NOT NULL, + `c1` int, + `c2` int, + `c3` int NOT NULL, + `c4` int + )UNIQUE KEY(k1) + DISTRIBUTED BY HASH(k1) BUCKETS 1 + PROPERTIES ( + "enable_unique_key_merge_on_write" = "true", + "disable_auto_compaction" = "true", + "replication_num" = "1", + "store_row_column" = "${use_row_store}", + "function_column.sequence_col" = "c3"); """ + + sql "insert into ${tableName4} values(1,1,1,1,1),(2,2,2,2,2),(3,3,3,3,3),(4,4,4,4,4),(5,5,5,5,5),(6,6,6,6,6);" + qt_sql3 "select k1,c1,c2,c3,c4 from ${tableName4} order by k1,c1,c2,c3,c4;" + // if the table has sequence map col, can not set sequence map col when merge_type=delete + streamLoad { + table "${tableName4}" + set 'column_separator', ',' + set 'format', 'csv' + set 'columns', 'k1' + set 'partial_columns', 'true' + set 'merge_type', 'DELETE' + file 'partial_update_delete.csv' + time 10000 + } + sql "sync" + qt_sql3 "select k1,c1,c2,c3,c4 from ${tableName4} order by k1,c1,c2,c3,c4;" + + sql "set enable_insert_strict=false;" + sql "set enable_unique_key_partial_update=true;" + sql "sync;" + sql "insert into ${tableName4}(k1, __DORIS_DELETE_SIGN__) values(8,1),(4,1),(9,1);" + qt_sql3 "select k1,c1,c2,c3,c4 from ${tableName4} order by k1,c1,c2,c3,c4;" + sql "set enable_insert_strict=true;" + sql "set enable_unique_key_partial_update=false;" + sql "sync;" + + sql "set skip_delete_sign=true;" + sql "set skip_storage_engine_merge=true;" + sql "set skip_delete_bitmap=true;" + qt_sql3 "select k1,c1,c2,c3,c4,__DORIS_DELETE_SIGN__,__DORIS_SEQUENCE_COL__ from ${tableName4} order by k1,c1,c2,c3,c4,__DORIS_DELETE_SIGN__;" + sql "drop table if exists ${tableName4};" + + + sql "set skip_delete_sign=false;" + sql "set skip_storage_engine_merge=false;" + sql "set skip_delete_bitmap=false;" + def tableName5 = "test_partial_update_delete5" + sql "DROP TABLE IF EXISTS ${tableName5};" + sql """ CREATE TABLE IF NOT EXISTS ${tableName5} ( + `k1` int NOT NULL, + `c1` int, + `c2` int, + `c3` int NOT NULL, + `c4` int + )UNIQUE KEY(k1) + DISTRIBUTED BY HASH(k1) BUCKETS 1 + PROPERTIES ( + "enable_unique_key_merge_on_write" = "true", + "disable_auto_compaction" = "true", + "replication_num" = "1", + "store_row_column" = "${use_row_store}", + "function_column.sequence_type" = "int"); """ + sql "insert into ${tableName5}(k1,c1,c2,c3,c4,__DORIS_SEQUENCE_COL__) values(1,1,1,1,1,1),(2,2,2,2,2,2),(3,3,3,3,3,3),(4,4,4,4,4,4),(5,5,5,5,5,5),(6,6,6,6,6,6);" + qt_sql4 "select k1,c1,c2,c3,c4 from ${tableName5} order by k1,c1,c2,c3,c4;" + // if the table has sequence type col, users must set sequence col even if merge_type=delete + streamLoad { + table "${tableName5}" + set 'column_separator', ',' + set 'format', 'csv' + set 'columns', 'k1' + set 'partial_columns', 'true' + set 'merge_type', 'DELETE' + file 'partial_update_delete.csv' + time 10000 + check { result, exception, startTime, endTime -> + if (exception != null) { + throw exception + } + log.info("Stream load result: ${result}".toString()) + def json = parseJson(result) + assertEquals("fail", json.Status.toLowerCase()) + assertTrue(json.Message.contains('need to specify the sequence column')) + } + } + sql "drop table if exists ${tableName5};" } } } From b0d7d0dbe0f9e14eaa0f336afcd1ec974897b01d Mon Sep 17 00:00:00 2001 From: bobhan1 Date: Tue, 3 Sep 2024 21:35:46 +0800 Subject: [PATCH 5/5] fix --- .../main/java/org/apache/doris/planner/StreamLoadPlanner.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/StreamLoadPlanner.java b/fe/fe-core/src/main/java/org/apache/doris/planner/StreamLoadPlanner.java index a3c864e9207cbe..8988d3220c8672 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/StreamLoadPlanner.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/StreamLoadPlanner.java @@ -127,8 +127,7 @@ public TPipelineFragmentParams plan(TUniqueId loadId, int fragmentInstanceIdInde throw new AnalysisException("load by MERGE or DELETE need to upgrade table to support batch delete."); } - if (destTable.hasSequenceCol() && !taskInfo.hasSequenceCol() && destTable.getSequenceMapCol() == null - && taskInfo.getMergeType() != LoadTask.MergeType.DELETE) { + if (destTable.hasSequenceCol() && !taskInfo.hasSequenceCol() && destTable.getSequenceMapCol() == null) { throw new UserException("Table " + destTable.getName() + " has sequence column, need to specify the sequence column"); }