diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/StreamLoadPlanner.java b/fe/fe-core/src/main/java/org/apache/doris/planner/StreamLoadPlanner.java index 8988d3220c8672..c302c4e905bb60 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/StreamLoadPlanner.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/StreamLoadPlanner.java @@ -195,7 +195,8 @@ public TPipelineFragmentParams plan(TUniqueId loadId, int fragmentInstanceIdInde + " by generated columns, missing: " + col.getName()); } } - if (taskInfo.getMergeType() == LoadTask.MergeType.DELETE) { + if (taskInfo.getMergeType() == LoadTask.MergeType.DELETE + || taskInfo.getMergeType() == LoadTask.MergeType.MERGE) { partialUpdateInputColumns.add(Column.DELETE_SIGN); } } diff --git a/regression-test/data/unique_with_mow_p0/partial_update/merge1.csv b/regression-test/data/unique_with_mow_p0/partial_update/merge1.csv new file mode 100644 index 00000000000000..24364822bccb0f --- /dev/null +++ b/regression-test/data/unique_with_mow_p0/partial_update/merge1.csv @@ -0,0 +1,6 @@ +1,10,0 +2,20,1 +3,30,0 +4,40,1 +10,999,0 +11,888,1 \ No newline at end of file diff --git a/regression-test/data/unique_with_mow_p0/partial_update/merge2.csv b/regression-test/data/unique_with_mow_p0/partial_update/merge2.csv new file mode 100644 index 00000000000000..8292fab318056e --- /dev/null +++ b/regression-test/data/unique_with_mow_p0/partial_update/merge2.csv @@ -0,0 +1,2 @@ +1,11 +5,50 \ No newline at end of file diff --git a/regression-test/data/unique_with_mow_p0/partial_update/merge3.csv b/regression-test/data/unique_with_mow_p0/partial_update/merge3.csv new file mode 100644 index 00000000000000..2aad468b2970ff --- /dev/null +++ b/regression-test/data/unique_with_mow_p0/partial_update/merge3.csv @@ -0,0 +1,6 @@ +1,10,2,0 +2,20,2,1 +3,30,2,0 +4,40,2,1 +10,999,2,0 +11,888,2,1 \ No newline at end of file diff --git a/regression-test/data/unique_with_mow_p0/partial_update/merge4.csv b/regression-test/data/unique_with_mow_p0/partial_update/merge4.csv new file mode 100644 index 00000000000000..cec571fcb9df49 --- /dev/null +++ b/regression-test/data/unique_with_mow_p0/partial_update/merge4.csv @@ -0,0 +1,2 @@ +1,11,4 +5,50,4 \ No newline at end of file diff --git a/regression-test/data/unique_with_mow_p0/partial_update/merge5.csv b/regression-test/data/unique_with_mow_p0/partial_update/merge5.csv new file mode 100644 index 00000000000000..4170f9d8b26370 --- /dev/null +++ b/regression-test/data/unique_with_mow_p0/partial_update/merge5.csv @@ -0,0 +1,2 @@ +6,1 +11,0 \ No newline at end of file diff --git a/regression-test/data/unique_with_mow_p0/partial_update/merge6.csv b/regression-test/data/unique_with_mow_p0/partial_update/merge6.csv new file mode 100644 index 00000000000000..a8a7c5a2414a6b --- /dev/null +++ b/regression-test/data/unique_with_mow_p0/partial_update/merge6.csv @@ -0,0 +1,2 @@ +7 +11 \ No newline at end of file diff --git a/regression-test/data/unique_with_mow_p0/partial_update/merge7.csv b/regression-test/data/unique_with_mow_p0/partial_update/merge7.csv new file mode 100644 index 00000000000000..8e29be1395cd50 --- /dev/null +++ b/regression-test/data/unique_with_mow_p0/partial_update/merge7.csv @@ -0,0 +1,2 @@ +6,3,1 +11,3,0 \ No newline at end of file diff --git a/regression-test/data/unique_with_mow_p0/partial_update/merge8.csv b/regression-test/data/unique_with_mow_p0/partial_update/merge8.csv new file mode 100644 index 00000000000000..20bbd5c6142536 --- /dev/null +++ b/regression-test/data/unique_with_mow_p0/partial_update/merge8.csv @@ -0,0 +1,2 @@ +7,5 +11,5 \ No newline at end of file diff --git a/regression-test/data/unique_with_mow_p0/partial_update/test_partial_update_merge_type.out b/regression-test/data/unique_with_mow_p0/partial_update/test_partial_update_merge_type.out new file mode 100644 index 00000000000000..04e4b07f2dce3f --- /dev/null +++ b/regression-test/data/unique_with_mow_p0/partial_update/test_partial_update_merge_type.out @@ -0,0 +1,341 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !sql -- +0 0 0 0 +1 1 1 1 +2 2 2 2 +3 3 3 3 +4 4 4 4 +5 5 5 5 +6 6 6 6 +7 7 7 7 +8 8 8 8 + +-- !sql_1_1 -- +0 0 0 0 +1 1 10 1 +3 3 30 3 +5 5 5 5 +6 6 6 6 +7 7 7 7 +8 8 8 8 +10 \N 999 \N + +-- !sql_1_2 -- +0 0 0 0 +1 1 10 1 +3 3 30 3 +5 5 5 5 +7 7 7 7 +8 8 8 8 +10 \N 999 \N +11 \N 888 \N + +-- !sql_2_1 -- +0 0 0 0 +3 3 30 3 +7 7 7 7 +8 8 8 8 +10 \N 999 \N +11 \N 888 \N + +-- !sql_2_2 -- +0 0 0 0 +3 3 30 3 +8 8 8 8 +10 \N 999 \N + +-- !sql -- +0 0 0 0 +1 1 1 1 +2 2 2 2 +3 3 3 3 +4 4 4 4 +5 5 5 5 +6 6 6 6 +7 7 7 7 +8 8 8 8 + +-- !sql_3_1 -- +0 0 0 0 +1 1 10 1 +3 3 30 3 +5 5 5 5 +6 6 6 6 +7 7 7 7 +8 8 8 8 +10 \N 999 \N + +-- !inspect -- +0 0 0 0 1 0 +1 1 1 1 1 0 +1 1 10 1 2 0 +2 2 2 2 1 0 +2 2 20 2 2 1 +3 3 3 3 1 0 +3 3 30 3 2 0 +4 4 4 4 1 0 +4 4 40 4 2 1 +5 5 5 5 1 0 +6 6 6 6 1 0 +7 7 7 7 1 0 +8 8 8 8 1 0 +10 \N 999 \N 2 0 +11 \N 888 \N 2 1 + +-- !sql_3_2 -- +0 0 0 0 +1 1 10 1 +3 3 30 3 +5 5 5 5 +7 7 7 7 +8 8 8 8 +10 \N 999 \N +11 \N 888 \N + +-- !inspect -- +0 0 0 0 1 0 +1 1 1 1 1 0 +1 1 10 1 2 0 +2 2 2 2 1 0 +2 2 20 2 2 1 +3 3 3 3 1 0 +3 3 30 3 2 0 +4 4 4 4 1 0 +4 4 40 4 2 1 +5 5 5 5 1 0 +6 6 6 6 1 0 +6 6 6 6 3 1 +7 7 7 7 1 0 +8 8 8 8 1 0 +10 \N 999 \N 2 0 +11 \N 888 \N 2 1 +11 \N 888 \N 3 0 + +-- !sql_4_1 -- +0 0 0 0 +3 3 30 3 +7 7 7 7 +8 8 8 8 +10 \N 999 \N +11 \N 888 \N + +-- !inspect -- +0 0 0 0 1 0 +1 1 1 1 1 0 +1 1 10 1 2 0 +1 1 11 1 4 1 +2 2 2 2 1 0 +2 2 20 2 2 1 +3 3 3 3 1 0 +3 3 30 3 2 0 +4 4 4 4 1 0 +4 4 40 4 2 1 +5 5 5 5 1 0 +5 5 50 5 4 1 +6 6 6 6 1 0 +6 6 6 6 3 1 +7 7 7 7 1 0 +8 8 8 8 1 0 +10 \N 999 \N 2 0 +11 \N 888 \N 2 1 +11 \N 888 \N 3 0 + +-- !sql_4_2 -- +0 0 0 0 +3 3 30 3 +8 8 8 8 +10 \N 999 \N + +-- !inspect -- +0 0 0 0 1 0 +1 1 1 1 1 0 +1 1 10 1 2 0 +1 1 11 1 4 1 +2 2 2 2 1 0 +2 2 20 2 2 1 +3 3 3 3 1 0 +3 3 30 3 2 0 +4 4 4 4 1 0 +4 4 40 4 2 1 +5 5 5 5 1 0 +5 5 50 5 4 1 +6 6 6 6 1 0 +6 6 6 6 3 1 +7 7 7 7 1 0 +7 7 7 7 5 1 +8 8 8 8 1 0 +10 \N 999 \N 2 0 +11 \N 888 \N 2 1 +11 \N 888 \N 3 0 +11 \N 888 \N 5 1 + +-- !sql -- +0 0 0 0 +1 1 1 1 +2 2 2 2 +3 3 3 3 +4 4 4 4 +5 5 5 5 +6 6 6 6 +7 7 7 7 +8 8 8 8 + +-- !sql_1_1 -- +0 0 0 0 +1 1 10 1 +3 3 30 3 +5 5 5 5 +6 6 6 6 +7 7 7 7 +8 8 8 8 +10 \N 999 \N + +-- !sql_1_2 -- +0 0 0 0 +1 1 10 1 +3 3 30 3 +5 5 5 5 +7 7 7 7 +8 8 8 8 +10 \N 999 \N +11 \N 888 \N + +-- !sql_2_1 -- +0 0 0 0 +3 3 30 3 +7 7 7 7 +8 8 8 8 +10 \N 999 \N +11 \N 888 \N + +-- !sql_2_2 -- +0 0 0 0 +3 3 30 3 +8 8 8 8 +10 \N 999 \N + +-- !sql -- +0 0 0 0 +1 1 1 1 +2 2 2 2 +3 3 3 3 +4 4 4 4 +5 5 5 5 +6 6 6 6 +7 7 7 7 +8 8 8 8 + +-- !sql_3_1 -- +0 0 0 0 +1 1 10 1 +3 3 30 3 +5 5 5 5 +6 6 6 6 +7 7 7 7 +8 8 8 8 +10 \N 999 \N + +-- !inspect -- +0 0 0 0 1 0 +1 1 1 1 1 0 +1 1 10 1 2 0 +2 2 2 2 1 0 +2 2 20 2 2 1 +3 3 3 3 1 0 +3 3 30 3 2 0 +4 4 4 4 1 0 +4 4 40 4 2 1 +5 5 5 5 1 0 +6 6 6 6 1 0 +7 7 7 7 1 0 +8 8 8 8 1 0 +10 \N 999 \N 2 0 +11 \N 888 \N 2 1 + +-- !sql_3_2 -- +0 0 0 0 +1 1 10 1 +3 3 30 3 +5 5 5 5 +7 7 7 7 +8 8 8 8 +10 \N 999 \N +11 \N 888 \N + +-- !inspect -- +0 0 0 0 1 0 +1 1 1 1 1 0 +1 1 10 1 2 0 +2 2 2 2 1 0 +2 2 20 2 2 1 +3 3 3 3 1 0 +3 3 30 3 2 0 +4 4 4 4 1 0 +4 4 40 4 2 1 +5 5 5 5 1 0 +6 6 6 6 1 0 +6 6 6 6 3 1 +7 7 7 7 1 0 +8 8 8 8 1 0 +10 \N 999 \N 2 0 +11 \N 888 \N 2 1 +11 \N 888 \N 3 0 + +-- !sql_4_1 -- +0 0 0 0 +3 3 30 3 +7 7 7 7 +8 8 8 8 +10 \N 999 \N +11 \N 888 \N + +-- !inspect -- +0 0 0 0 1 0 +1 1 1 1 1 0 +1 1 10 1 2 0 +1 1 11 1 4 1 +2 2 2 2 1 0 +2 2 20 2 2 1 +3 3 3 3 1 0 +3 3 30 3 2 0 +4 4 4 4 1 0 +4 4 40 4 2 1 +5 5 5 5 1 0 +5 5 50 5 4 1 +6 6 6 6 1 0 +6 6 6 6 3 1 +7 7 7 7 1 0 +8 8 8 8 1 0 +10 \N 999 \N 2 0 +11 \N 888 \N 2 1 +11 \N 888 \N 3 0 + +-- !sql_4_2 -- +0 0 0 0 +3 3 30 3 +8 8 8 8 +10 \N 999 \N + +-- !inspect -- +0 0 0 0 1 0 +1 1 1 1 1 0 +1 1 10 1 2 0 +1 1 11 1 4 1 +2 2 2 2 1 0 +2 2 20 2 2 1 +3 3 3 3 1 0 +3 3 30 3 2 0 +4 4 4 4 1 0 +4 4 40 4 2 1 +5 5 5 5 1 0 +5 5 50 5 4 1 +6 6 6 6 1 0 +6 6 6 6 3 1 +7 7 7 7 1 0 +7 7 7 7 5 1 +8 8 8 8 1 0 +10 \N 999 \N 2 0 +11 \N 888 \N 2 1 +11 \N 888 \N 3 0 +11 \N 888 \N 5 1 + diff --git a/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_merge_type.groovy b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_merge_type.groovy new file mode 100644 index 00000000000000..ece523838b0076 --- /dev/null +++ b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_merge_type.groovy @@ -0,0 +1,185 @@ + +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_partial_update_merge_type", "p0") { + + String db = context.config.getDbNameByFile(context.file) + sql "select 1;" // to create database + + def inspect_rows = { sqlStr -> + sql "set skip_delete_sign=true;" + sql "set skip_delete_bitmap=true;" + sql "sync" + qt_inspect sqlStr + sql "set skip_delete_sign=false;" + sql "set skip_delete_bitmap=false;" + sql "sync" + } + + for (def use_row_store : [false, true]) { + logger.info("current params: use_row_store: ${use_row_store}") + + connect(user = context.config.jdbcUser, password = context.config.jdbcPassword, url = context.config.jdbcUrl) { + sql "use ${db};" + + def tableName = "test_partial_update_merge_type" + sql """ DROP TABLE IF EXISTS ${tableName} force""" + sql """ CREATE TABLE ${tableName} ( + `k` BIGINT NOT NULL, + `c1` int, + `c2` int, + `c3` int) + UNIQUE KEY(`k`) DISTRIBUTED BY HASH(`k`) BUCKETS 1 + PROPERTIES( + "replication_num" = "1", + "enable_unique_key_merge_on_write" = "true", + "store_row_column" = "${use_row_store}"); """ + sql """insert into ${tableName} select number,number,number,number from numbers("number"="9");""" + sql "sync" + qt_sql """select * from ${tableName} order by k;""" + // 1.1 merge_type=MERGE, no sequence col + streamLoad { + table "${tableName}" + set 'column_separator', ',' + set 'format', 'csv' + set 'columns', 'k,c2,del' + set 'partial_columns', 'true' + set 'merge_type', 'MERGE' + set 'delete', 'del=1' + file 'merge1.csv' + time 10000 + } + qt_sql_1_1 """select * from ${tableName} order by k;""" + // 1.2 merge_type=MERGE, no sequence col, no value col + streamLoad { + table "${tableName}" + set 'column_separator', ',' + set 'format', 'csv' + set 'columns', 'k,del' + set 'partial_columns', 'true' + set 'merge_type', 'MERGE' + set 'delete', 'del=1' + file 'merge5.csv' + time 10000 + } + qt_sql_1_2 """select * from ${tableName} order by k;""" + // 2.1 merge_type=DELETE, no sequence col + streamLoad { + table "${tableName}" + set 'column_separator', ',' + set 'format', 'csv' + set 'columns', 'k,c1' + set 'partial_columns', 'true' + set 'merge_type', 'DELETE' + file 'merge2.csv' + time 10000 + } + qt_sql_2_1 """select * from ${tableName} order by k;""" + // 2.2 merge_type=DELETE, no sequence col, no value col + streamLoad { + table "${tableName}" + set 'column_separator', ',' + set 'format', 'csv' + set 'columns', 'k' + set 'partial_columns', 'true' + set 'merge_type', 'DELETE' + file 'merge6.csv' + time 10000 + } + qt_sql_2_2 """select * from ${tableName} order by k;""" + + + tableName = "test_partial_update_merge_type2" + sql """ DROP TABLE IF EXISTS ${tableName} force""" + sql """ CREATE TABLE ${tableName} ( + `k` BIGINT NOT NULL, + `c1` int, + `c2` int, + `c3` int) + UNIQUE KEY(`k`) DISTRIBUTED BY HASH(`k`) BUCKETS 1 + PROPERTIES( + "replication_num" = "1", + "enable_unique_key_merge_on_write" = "true", + "function_column.sequence_type" = "BIGINT", + "store_row_column" = "${use_row_store}"); """ + sql """insert into ${tableName}(k,c1,c2,c3,__DORIS_SEQUENCE_COL__) select number,number,number,number,1 from numbers("number"="9");""" + qt_sql """select * from ${tableName} order by k;""" + // 3.1 merge_type=MERGE, has sequence type col + streamLoad { + table "${tableName}" + set 'column_separator', ',' + set 'format', 'csv' + set 'columns', 'k,c2,seq,del' + set 'partial_columns', 'true' + set 'function_column.sequence_col', 'seq' + set 'merge_type', 'MERGE' + set 'delete', 'del=1' + file 'merge3.csv' + time 10000 + } + qt_sql_3_1 """select * from ${tableName} order by k;""" + inspect_rows """select k,c1,c2,c3,__DORIS_SEQUENCE_COL__,__DORIS_DELETE_SIGN__ from ${tableName} order by k,__DORIS_SEQUENCE_COL__;""" + // 3.2 merge_type=MERGE, has sequence type col, no value col + streamLoad { + table "${tableName}" + set 'column_separator', ',' + set 'format', 'csv' + set 'columns', 'k,seq,del' + set 'partial_columns', 'true' + set 'function_column.sequence_col', 'seq' + set 'merge_type', 'MERGE' + set 'delete', 'del=1' + file 'merge7.csv' + time 10000 + } + qt_sql_3_2 """select * from ${tableName} order by k;""" + inspect_rows """select k,c1,c2,c3,__DORIS_SEQUENCE_COL__,__DORIS_DELETE_SIGN__ from ${tableName} order by k,__DORIS_SEQUENCE_COL__;""" + + // 4.1 merge_type=DELETE, has sequence type col + streamLoad { + table "${tableName}" + set 'column_separator', ',' + set 'format', 'csv' + set 'columns', 'k,c2,seq' + set 'partial_columns', 'true' + set 'function_column.sequence_col', 'seq' + set 'merge_type', 'DELETE' + file 'merge4.csv' + time 10000 + } + qt_sql_4_1 """select * from ${tableName} order by k;""" + inspect_rows """select k,c1,c2,c3,__DORIS_SEQUENCE_COL__,__DORIS_DELETE_SIGN__ from ${tableName} order by k,__DORIS_SEQUENCE_COL__;""" + // 4.2 merge_type=DELETE, has sequence type col, no value col + streamLoad { + table "${tableName}" + set 'column_separator', ',' + set 'format', 'csv' + set 'columns', 'k,seq' + set 'partial_columns', 'true' + set 'function_column.sequence_col', 'seq' + set 'merge_type', 'DELETE' + file 'merge8.csv' + time 10000 + } + qt_sql_4_2 """select * from ${tableName} order by k;""" + inspect_rows """select k,c1,c2,c3,__DORIS_SEQUENCE_COL__,__DORIS_DELETE_SIGN__ from ${tableName} order by k,__DORIS_SEQUENCE_COL__;""" + + sql """ DROP TABLE IF EXISTS ${tableName}; """ + } + } +}