From d50c2dc3131ebb39979615b9e87461a651ba1cb6 Mon Sep 17 00:00:00 2001 From: bobhan1 Date: Mon, 17 Jul 2023 17:11:25 +0800 Subject: [PATCH 1/9] tmp --- .../src/main/java/org/apache/doris/load/DeleteHandler.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/DeleteHandler.java b/fe/fe-core/src/main/java/org/apache/doris/load/DeleteHandler.java index 578b3cd4d0af50..59461ac476b3a3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/DeleteHandler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/DeleteHandler.java @@ -688,8 +688,7 @@ private void checkDeleteV2(OlapTable table, List partitions, // Due to rounding errors, most floating-point numbers end up being slightly imprecise, // it also means that numbers expected to be equal often differ slightly, so we do not allow compare with // floating-point numbers, floating-point number not allowed in where clause - if (!column.isKey() && table.getKeysType() != KeysType.DUP_KEYS - || column.getDataType().isFloatingPointType()) { + if (column.isSequenceColumn() || column.getDataType().isFloatingPointType()) { // ErrorReport.reportDdlException(ErrorCode.ERR_NOT_KEY_COLUMN, columnName); throw new DdlException("Column[" + columnName + "] is not key column or storage model " + "is not duplicate or column type is float or double."); From 578abe0bd04f301f611c3dc9861e75920b0852ca Mon Sep 17 00:00:00 2001 From: bobhan1 Date: Tue, 18 Jul 2023 18:22:00 +0800 Subject: [PATCH 2/9] delte on value --- be/src/olap/delete_handler.cpp | 7 +- .../org/apache/doris/load/DeleteHandler.java | 16 +++- .../data/delete_p0/test_delete_on_value.out | 43 +++++++++ .../delete_p0/test_delete_on_value.groovy | 87 +++++++++++++++++++ 4 files changed, 145 insertions(+), 8 deletions(-) create mode 100644 regression-test/data/delete_p0/test_delete_on_value.out create mode 100644 regression-test/suites/delete_p0/test_delete_on_value.groovy diff --git a/be/src/olap/delete_handler.cpp b/be/src/olap/delete_handler.cpp index b7bbfe2908a9b7..1df44e000408de 100644 --- a/be/src/olap/delete_handler.cpp +++ b/be/src/olap/delete_handler.cpp @@ -177,12 +177,9 @@ Status DeleteHandler::check_condition_valid(const TabletSchema& schema, const TC // the condition column type should not be float or double. const TabletColumn& column = schema.column(field_index); - if ((!column.is_key() && schema.keys_type() != KeysType::DUP_KEYS) || - column.type() == FieldType::OLAP_FIELD_TYPE_DOUBLE || + if (column.type() == FieldType::OLAP_FIELD_TYPE_DOUBLE || column.type() == FieldType::OLAP_FIELD_TYPE_FLOAT) { - return Status::Error( - "field is not key column, or storage model is not duplicate, or data type is float " - "or double."); + return Status::Error("data type is float or double."); } // Check operator and operands size are matched. diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/DeleteHandler.java b/fe/fe-core/src/main/java/org/apache/doris/load/DeleteHandler.java index 59461ac476b3a3..f4c205887a09df 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/DeleteHandler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/DeleteHandler.java @@ -688,10 +688,20 @@ private void checkDeleteV2(OlapTable table, List partitions, // Due to rounding errors, most floating-point numbers end up being slightly imprecise, // it also means that numbers expected to be equal often differ slightly, so we do not allow compare with // floating-point numbers, floating-point number not allowed in where clause - if (column.isSequenceColumn() || column.getDataType().isFloatingPointType()) { + if (column.getDataType().isFloatingPointType()) { // ErrorReport.reportDdlException(ErrorCode.ERR_NOT_KEY_COLUMN, columnName); - throw new DdlException("Column[" + columnName + "] is not key column or storage model " - + "is not duplicate or column type is float or double."); + throw new DdlException("Column[" + columnName + "] type is float or double."); + } + LOG.warn("table {}, keysType: {}, column: {}, isSequenceColumn: {}, isKey: {}", table.getName(), + table.getKeysType(), columnName, column.isSequenceColumn(), column.isKey()); + if (!column.isKey()) { + if (table.getKeysType() == KeysType.AGG_KEYS) { + throw new DdlException("delete predicate on value column only supports Unique table" + + "Table[" + table.getName() + "] is Aggregate table."); + } else if (table.getKeysType() == KeysType.UNIQUE_KEYS && column.isSequenceColumn()) { + throw new DdlException("delete predicate on value column only supports Unique table and the column" + + "must not be the sequence column. Column[" + columnName + "] is sequence column."); + } } if (condition instanceof BinaryPredicate) { diff --git a/regression-test/data/delete_p0/test_delete_on_value.out b/regression-test/data/delete_p0/test_delete_on_value.out new file mode 100644 index 00000000000000..7b38711fdcc544 --- /dev/null +++ b/regression-test/data/delete_p0/test_delete_on_value.out @@ -0,0 +1,43 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !sql -- +1 1 1 +2 2 2 +3 3 3 +4 4 4 +5 5 5 +6 6 6 +7 7 7 +8 8 8 +9 9 9 + +-- !sql -- +1 1 1 +2 2 2 +3 3 3 +5 5 5 +6 6 6 +7 7 7 +8 8 8 +9 9 9 + +-- !sql -- +1 1 1 +2 2 2 +8 8 8 +9 9 9 + +-- !sql -- +1 1 1 +2 2 2 +4 4 4 +5 5 5 +8 8 8 +9 9 9 + +-- !sql -- +1 1 1 +2 2 2 +4 4 4 +8 8 8 +9 9 9 + diff --git a/regression-test/suites/delete_p0/test_delete_on_value.groovy b/regression-test/suites/delete_p0/test_delete_on_value.groovy new file mode 100644 index 00000000000000..20114d51e621d3 --- /dev/null +++ b/regression-test/suites/delete_p0/test_delete_on_value.groovy @@ -0,0 +1,87 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_delete_on_value") { + + def tableName = "test_delete_on_value" + sql """ DROP TABLE IF EXISTS ${tableName} """ + sql """ CREATE TABLE ${tableName} ( + `x` BIGINT NOT NULL, + `y` BIGINT NULL, + `z` BIGINT NULL) + ENGINE=OLAP + UNIQUE KEY(`x`) + COMMENT 'OLAP' + DISTRIBUTED BY HASH(`x`) BUCKETS 4 + PROPERTIES ( + "replication_num" = "1", + "enable_unique_key_merge_on_write" = "true" + );""" + sql """ insert into ${tableName} values(1,1,1),(2,2,2),(3,3,3),(4,4,4),(5,5,5),(6,6,6),(7,7,7),(8,8,8),(9,9,9); """ + qt_sql "select * from ${tableName} order by x;" + sql "delete from ${tableName} where y=4;" + qt_sql "select * from ${tableName} order by x;" + sql "delete from ${tableName} where z>=3 and z<=7;" + qt_sql "select * from ${tableName} order by x;" + sql "insert into ${tableName} values(4,4,4),(5,5,5);" + qt_sql "select * from ${tableName} order by x;" + sql "delete from ${tableName} where y=5;" + qt_sql "select * from ${tableName} order by x;" + + + def tableName2 = "test_delete_on_value2" + sql """ DROP TABLE IF EXISTS ${tableName2} """ + sql """ CREATE TABLE ${tableName2} ( + `x` BIGINT NOT NULL, + `y` BIGINT REPLACE_IF_NOT_NULL NULL, + `z` BIGINT REPLACE_IF_NOT_NULL NULL) + ENGINE=OLAP + AGGREGATE KEY(`x`) + COMMENT 'OLAP' + DISTRIBUTED BY HASH(`x`) BUCKETS 4 + PROPERTIES ( + "replication_num" = "1" + );""" + sql """ insert into ${tableName2} values(1,1,1); """ + test { + sql "delete from ${tableName} where y=4;" + exception "delete predicate on value column only supports Unique table" + } + + + def tableName3 = "test_delete_on_value3" + sql """ DROP TABLE IF EXISTS ${tableName3} """ + sql """ CREATE TABLE ${tableName3} ( + `x` BIGINT NOT NULL, + `y` BIGINT NULL, + `z` BIGINT NULL) + ENGINE=OLAP + UNIQUE KEY(`x`) + COMMENT 'OLAP' + DISTRIBUTED BY HASH(`x`) BUCKETS 4 + PROPERTIES ( + "replication_num" = "1", + "enable_unique_key_merge_on_write" = "true", + "function_column.sequence_col" = 'y' + );""" + sql """ insert into ${tableName3} values(1,1,1),(2,2,2),(3,3,3),(4,4,4),(5,5,5); """ + sql "delete from ${tableName3} where y=4;" + test { + qt_sql "select * from ${tableName3} order by x;" + exception "delete predicate on value column only supports Unique table and the column must not be the sequence column" + } +} From 9814854de777e1747f3da5a2eadf3ea72fb32ce0 Mon Sep 17 00:00:00 2001 From: bobhan1 Date: Wed, 19 Jul 2023 07:21:45 +0800 Subject: [PATCH 3/9] tmp --- .../org/apache/doris/load/DeleteHandler.java | 3 +- .../data/delete_p0/test_delete_on_value.out | 31 ------------------- .../delete_p0/test_delete_on_value.groovy | 3 +- 3 files changed, 3 insertions(+), 34 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/DeleteHandler.java b/fe/fe-core/src/main/java/org/apache/doris/load/DeleteHandler.java index f4c205887a09df..25cb3cc27848ed 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/DeleteHandler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/DeleteHandler.java @@ -698,7 +698,8 @@ private void checkDeleteV2(OlapTable table, List partitions, if (table.getKeysType() == KeysType.AGG_KEYS) { throw new DdlException("delete predicate on value column only supports Unique table" + "Table[" + table.getName() + "] is Aggregate table."); - } else if (table.getKeysType() == KeysType.UNIQUE_KEYS && column.isSequenceColumn()) { + } else if (table.getKeysType() == KeysType.UNIQUE_KEYS + && table.getSequenceMapCol().equals(columnName)) { throw new DdlException("delete predicate on value column only supports Unique table and the column" + "must not be the sequence column. Column[" + columnName + "] is sequence column."); } diff --git a/regression-test/data/delete_p0/test_delete_on_value.out b/regression-test/data/delete_p0/test_delete_on_value.out index 7b38711fdcc544..6b4f554128fbe0 100644 --- a/regression-test/data/delete_p0/test_delete_on_value.out +++ b/regression-test/data/delete_p0/test_delete_on_value.out @@ -10,34 +10,3 @@ 8 8 8 9 9 9 --- !sql -- -1 1 1 -2 2 2 -3 3 3 -5 5 5 -6 6 6 -7 7 7 -8 8 8 -9 9 9 - --- !sql -- -1 1 1 -2 2 2 -8 8 8 -9 9 9 - --- !sql -- -1 1 1 -2 2 2 -4 4 4 -5 5 5 -8 8 8 -9 9 9 - --- !sql -- -1 1 1 -2 2 2 -4 4 4 -8 8 8 -9 9 9 - diff --git a/regression-test/suites/delete_p0/test_delete_on_value.groovy b/regression-test/suites/delete_p0/test_delete_on_value.groovy index 20114d51e621d3..0b7ca2fc9236f9 100644 --- a/regression-test/suites/delete_p0/test_delete_on_value.groovy +++ b/regression-test/suites/delete_p0/test_delete_on_value.groovy @@ -79,9 +79,8 @@ suite("test_delete_on_value") { "function_column.sequence_col" = 'y' );""" sql """ insert into ${tableName3} values(1,1,1),(2,2,2),(3,3,3),(4,4,4),(5,5,5); """ - sql "delete from ${tableName3} where y=4;" test { - qt_sql "select * from ${tableName3} order by x;" + sql "delete from ${tableName3} where y=4;" exception "delete predicate on value column only supports Unique table and the column must not be the sequence column" } } From a0cdd4b459e45eb62845bcd635b88c4b930350ff Mon Sep 17 00:00:00 2001 From: bobhan1 Date: Wed, 19 Jul 2023 15:51:52 +0800 Subject: [PATCH 4/9] update --- .../org/apache/doris/load/DeleteHandler.java | 8 ++----- .../delete_p0/test_delete_on_value.groovy | 22 ------------------- 2 files changed, 2 insertions(+), 28 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/DeleteHandler.java b/fe/fe-core/src/main/java/org/apache/doris/load/DeleteHandler.java index 25cb3cc27848ed..49b52ffd289aa2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/DeleteHandler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/DeleteHandler.java @@ -696,12 +696,8 @@ private void checkDeleteV2(OlapTable table, List partitions, table.getKeysType(), columnName, column.isSequenceColumn(), column.isKey()); if (!column.isKey()) { if (table.getKeysType() == KeysType.AGG_KEYS) { - throw new DdlException("delete predicate on value column only supports Unique table" - + "Table[" + table.getName() + "] is Aggregate table."); - } else if (table.getKeysType() == KeysType.UNIQUE_KEYS - && table.getSequenceMapCol().equals(columnName)) { - throw new DdlException("delete predicate on value column only supports Unique table and the column" - + "must not be the sequence column. Column[" + columnName + "] is sequence column."); + throw new DdlException("delete predicate on value column only supports Unique table and" + + "Duplicate table" + "Table[" + table.getName() + "] is Aggregate table."); } } diff --git a/regression-test/suites/delete_p0/test_delete_on_value.groovy b/regression-test/suites/delete_p0/test_delete_on_value.groovy index 0b7ca2fc9236f9..6117313ea0bb42 100644 --- a/regression-test/suites/delete_p0/test_delete_on_value.groovy +++ b/regression-test/suites/delete_p0/test_delete_on_value.groovy @@ -61,26 +61,4 @@ suite("test_delete_on_value") { sql "delete from ${tableName} where y=4;" exception "delete predicate on value column only supports Unique table" } - - - def tableName3 = "test_delete_on_value3" - sql """ DROP TABLE IF EXISTS ${tableName3} """ - sql """ CREATE TABLE ${tableName3} ( - `x` BIGINT NOT NULL, - `y` BIGINT NULL, - `z` BIGINT NULL) - ENGINE=OLAP - UNIQUE KEY(`x`) - COMMENT 'OLAP' - DISTRIBUTED BY HASH(`x`) BUCKETS 4 - PROPERTIES ( - "replication_num" = "1", - "enable_unique_key_merge_on_write" = "true", - "function_column.sequence_col" = 'y' - );""" - sql """ insert into ${tableName3} values(1,1,1),(2,2,2),(3,3,3),(4,4,4),(5,5,5); """ - test { - sql "delete from ${tableName3} where y=4;" - exception "delete predicate on value column only supports Unique table and the column must not be the sequence column" - } } From 6d22b6ff788718c674bbf976810e264d90b1474c Mon Sep 17 00:00:00 2001 From: bobhan1 Date: Wed, 9 Aug 2023 16:23:30 +0800 Subject: [PATCH 5/9] upadte --- .../org/apache/doris/analysis/DeleteStmt.java | 12 ++-- .../org/apache/doris/load/DeleteHandler.java | 3 - .../data/delete_p0/test_delete_on_value.out | 55 +++++++++++++++++++ .../delete_p0/test_delete_on_value.groovy | 41 ++++++-------- 4 files changed, 79 insertions(+), 32 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/DeleteStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/DeleteStmt.java index ac28d5850e0c42..f65bac6326dec9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/DeleteStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/DeleteStmt.java @@ -329,10 +329,14 @@ private void checkDeleteConditions() throws AnalysisException { // Due to rounding errors, most floating-point numbers end up being slightly imprecise, // it also means that numbers expected to be equal often differ slightly, so we do not allow compare with // floating-point numbers, floating-point number not allowed in where clause - if (!column.isKey() && table.getKeysType() != KeysType.DUP_KEYS - || column.getDataType().isFloatingPointType()) { - throw new AnalysisException("Column[" + columnName + "] is not key column or storage model " - + "is not duplicate or column type is float or double."); + if (column.getDataType().isFloatingPointType()) { + throw new AnalysisException("Column[" + columnName + "] type is float or double."); + } + if (!column.isKey()) { + if (table.getKeysType() == KeysType.AGG_KEYS) { + throw new AnalysisException("delete predicate on value column only supports Unique table and" + + "Duplicate table" + "Table[" + table.getName() + "] is Aggregate table."); + } } if (condition instanceof BinaryPredicate) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/DeleteHandler.java b/fe/fe-core/src/main/java/org/apache/doris/load/DeleteHandler.java index 49b52ffd289aa2..289c5f2289327a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/DeleteHandler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/DeleteHandler.java @@ -689,11 +689,8 @@ private void checkDeleteV2(OlapTable table, List partitions, // it also means that numbers expected to be equal often differ slightly, so we do not allow compare with // floating-point numbers, floating-point number not allowed in where clause if (column.getDataType().isFloatingPointType()) { - // ErrorReport.reportDdlException(ErrorCode.ERR_NOT_KEY_COLUMN, columnName); throw new DdlException("Column[" + columnName + "] type is float or double."); } - LOG.warn("table {}, keysType: {}, column: {}, isSequenceColumn: {}, isKey: {}", table.getName(), - table.getKeysType(), columnName, column.isSequenceColumn(), column.isKey()); if (!column.isKey()) { if (table.getKeysType() == KeysType.AGG_KEYS) { throw new DdlException("delete predicate on value column only supports Unique table and" diff --git a/regression-test/data/delete_p0/test_delete_on_value.out b/regression-test/data/delete_p0/test_delete_on_value.out index 6b4f554128fbe0..17745f342f021a 100644 --- a/regression-test/data/delete_p0/test_delete_on_value.out +++ b/regression-test/data/delete_p0/test_delete_on_value.out @@ -10,3 +10,58 @@ 8 8 8 9 9 9 +-- !sql -- +1 1 1 +2 2 2 +3 3 3 +5 5 5 +6 6 6 +7 7 7 +8 8 8 +9 9 9 + +-- !sql -- +1 1 1 +2 2 2 +8 8 8 +9 9 9 + +-- !sql -- +1 1 1 0 +2 2 2 0 +3 3 3 0 +4 4 4 0 +5 5 5 0 +6 6 6 0 +7 7 7 0 +8 8 8 0 +9 9 9 0 + +-- !sql -- +1 1 1 +2 2 2 +4 4 4 +5 5 5 +8 8 8 +9 9 9 + +-- !sql -- +1 1 1 +2 2 2 +4 4 4 +8 8 8 +9 9 9 + +-- !sql -- +1 1 1 0 +2 2 2 0 +3 3 3 0 +4 4 4 0 +4 4 4 0 +5 5 5 0 +5 5 5 0 +6 6 6 0 +7 7 7 0 +8 8 8 0 +9 9 9 0 + diff --git a/regression-test/suites/delete_p0/test_delete_on_value.groovy b/regression-test/suites/delete_p0/test_delete_on_value.groovy index 6117313ea0bb42..e77dfe1f7690df 100644 --- a/regression-test/suites/delete_p0/test_delete_on_value.groovy +++ b/regression-test/suites/delete_p0/test_delete_on_value.groovy @@ -16,6 +16,9 @@ // under the License. suite("test_delete_on_value") { + sql 'set enable_nereids_planner=false' + sql "set experimental_enable_nereids_planner=false;" + sql 'set enable_nereids_dml=false' def tableName = "test_delete_on_value" sql """ DROP TABLE IF EXISTS ${tableName} """ @@ -32,33 +35,21 @@ suite("test_delete_on_value") { "enable_unique_key_merge_on_write" = "true" );""" sql """ insert into ${tableName} values(1,1,1),(2,2,2),(3,3,3),(4,4,4),(5,5,5),(6,6,6),(7,7,7),(8,8,8),(9,9,9); """ - qt_sql "select * from ${tableName} order by x;" + sql "set skip_delete_sign=true;" + qt_sql "select * from ${tableName} order by x,y,z;" sql "delete from ${tableName} where y=4;" - qt_sql "select * from ${tableName} order by x;" + qt_sql "select * from ${tableName} order by x,y,z;" sql "delete from ${tableName} where z>=3 and z<=7;" - qt_sql "select * from ${tableName} order by x;" + qt_sql "select * from ${tableName} order by x,y,z;" + sql "set skip_delete_predicate=true;" + qt_sql "select x,y,z,__DORIS_DELETE_SIGN__ from ${tableName} order by x,y,z,__DORIS_DELETE_SIGN__;" + sql "set skip_delete_predicate=false;" sql "insert into ${tableName} values(4,4,4),(5,5,5);" - qt_sql "select * from ${tableName} order by x;" + qt_sql "select * from ${tableName} order by x,y,z;" sql "delete from ${tableName} where y=5;" - qt_sql "select * from ${tableName} order by x;" - - - def tableName2 = "test_delete_on_value2" - sql """ DROP TABLE IF EXISTS ${tableName2} """ - sql """ CREATE TABLE ${tableName2} ( - `x` BIGINT NOT NULL, - `y` BIGINT REPLACE_IF_NOT_NULL NULL, - `z` BIGINT REPLACE_IF_NOT_NULL NULL) - ENGINE=OLAP - AGGREGATE KEY(`x`) - COMMENT 'OLAP' - DISTRIBUTED BY HASH(`x`) BUCKETS 4 - PROPERTIES ( - "replication_num" = "1" - );""" - sql """ insert into ${tableName2} values(1,1,1); """ - test { - sql "delete from ${tableName} where y=4;" - exception "delete predicate on value column only supports Unique table" - } + qt_sql "select * from ${tableName} order by x,y,z;" + sql "set skip_storage_engine_merge=true;" + sql "set skip_delete_bitmap=true;" + sql "set skip_delete_predicate=true;" + qt_sql "select x,y,z,__DORIS_DELETE_SIGN__ from ${tableName} order by x,y,z,__DORIS_DELETE_SIGN__;" } From bbc0480f1fe16ef8fb9a6d661582bb0ad39277c8 Mon Sep 17 00:00:00 2001 From: bobhan1 Date: Mon, 14 Aug 2023 14:19:48 +0800 Subject: [PATCH 6/9] fix --- .../org/apache/doris/analysis/DeleteStmt.java | 2 +- .../org/apache/doris/load/DeleteHandler.java | 2 +- .../delete_p0/test_delete_on_value.groovy | 20 +++++++++++++++++++ 3 files changed, 22 insertions(+), 2 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/DeleteStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/DeleteStmt.java index f65bac6326dec9..f36bf8db02c1e5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/DeleteStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/DeleteStmt.java @@ -335,7 +335,7 @@ private void checkDeleteConditions() throws AnalysisException { if (!column.isKey()) { if (table.getKeysType() == KeysType.AGG_KEYS) { throw new AnalysisException("delete predicate on value column only supports Unique table and" - + "Duplicate table" + "Table[" + table.getName() + "] is Aggregate table."); + + " Duplicate table, but " + "Table[" + table.getName() + "] is an Aggregate table."); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/DeleteHandler.java b/fe/fe-core/src/main/java/org/apache/doris/load/DeleteHandler.java index 289c5f2289327a..c7c5587b7444bc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/DeleteHandler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/DeleteHandler.java @@ -694,7 +694,7 @@ private void checkDeleteV2(OlapTable table, List partitions, if (!column.isKey()) { if (table.getKeysType() == KeysType.AGG_KEYS) { throw new DdlException("delete predicate on value column only supports Unique table and" - + "Duplicate table" + "Table[" + table.getName() + "] is Aggregate table."); + + " Duplicate table, but " + "Table[" + table.getName() + "] is an Aggregate table."); } } diff --git a/regression-test/suites/delete_p0/test_delete_on_value.groovy b/regression-test/suites/delete_p0/test_delete_on_value.groovy index e77dfe1f7690df..03f04624c5f6bb 100644 --- a/regression-test/suites/delete_p0/test_delete_on_value.groovy +++ b/regression-test/suites/delete_p0/test_delete_on_value.groovy @@ -52,4 +52,24 @@ suite("test_delete_on_value") { sql "set skip_delete_bitmap=true;" sql "set skip_delete_predicate=true;" qt_sql "select x,y,z,__DORIS_DELETE_SIGN__ from ${tableName} order by x,y,z,__DORIS_DELETE_SIGN__;" + + + def tableName2 = "test_delete_on_value2" + sql """ DROP TABLE IF EXISTS ${tableName2} """ + sql """ CREATE TABLE ${tableName2} ( + `x` BIGINT NOT NULL, + `y` BIGINT REPLACE_IF_NOT_NULL NULL, + `z` BIGINT REPLACE_IF_NOT_NULL NULL) + ENGINE=OLAP + AGGREGATE KEY(`x`) + COMMENT 'OLAP' + DISTRIBUTED BY HASH(`x`) BUCKETS 4 + PROPERTIES ( + "replication_num" = "1" + );""" + sql """ insert into ${tableName2} values(1,1,1); """ + test { + sql "delete from ${tableName2} where y=4;" + exception "delete predicate on value column only supports Unique table and Duplicate table, but Table[test_delete_on_value2] is an Aggregate table." + } } From ce7e60d134093cfbb0e8d27859ae5c809e9eb07d Mon Sep 17 00:00:00 2001 From: bobhan1 Date: Mon, 14 Aug 2023 16:10:44 +0800 Subject: [PATCH 7/9] fix ut --- be/test/olap/delete_handler_test.cpp | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/be/test/olap/delete_handler_test.cpp b/be/test/olap/delete_handler_test.cpp index 28308b10487e8f..c2aa5f1efebca5 100644 --- a/be/test/olap/delete_handler_test.cpp +++ b/be/test/olap/delete_handler_test.cpp @@ -406,7 +406,6 @@ TEST_F(TestDeleteConditionHandler, StoreCondInvalidParameters) { DeletePredicatePB del_pred; Status failed_res = DeleteHandler::generate_delete_predicate(*tablet->tablet_schema(), conditions, &del_pred); - ; EXPECT_EQ(Status::Error(""), failed_res); } @@ -423,7 +422,6 @@ TEST_F(TestDeleteConditionHandler, StoreCondNonexistentColumn) { DeletePredicatePB del_pred; Status failed_res = DeleteHandler::generate_delete_predicate(*tablet->tablet_schema(), conditions, &del_pred); - ; EXPECT_EQ(Status::Error(""), failed_res); // 'v'是value列 @@ -434,10 +432,9 @@ TEST_F(TestDeleteConditionHandler, StoreCondNonexistentColumn) { condition.condition_values.push_back("5"); conditions.push_back(condition); - failed_res = DeleteHandler::generate_delete_predicate(*tablet->tablet_schema(), conditions, - &del_pred); - ; - EXPECT_EQ(Status::Error(""), failed_res); + Status success_res = DeleteHandler::generate_delete_predicate(*tablet->tablet_schema(), + conditions, &del_pred); + EXPECT_EQ(Status::OK(), success_res); // value column in duplicate model can be deleted; conditions.clear(); @@ -447,9 +444,8 @@ TEST_F(TestDeleteConditionHandler, StoreCondNonexistentColumn) { condition.condition_values.push_back("5"); conditions.push_back(condition); - Status success_res = DeleteHandler::generate_delete_predicate(*dup_tablet->tablet_schema(), - conditions, &del_pred); - ; + success_res = DeleteHandler::generate_delete_predicate(*dup_tablet->tablet_schema(), conditions, + &del_pred); EXPECT_EQ(Status::OK(), success_res); } From 3d9ca5b41560fc96743b3108788c0f58ece681c4 Mon Sep 17 00:00:00 2001 From: bobhan1 Date: Tue, 15 Aug 2023 19:42:32 +0800 Subject: [PATCH 8/9] update --- .../org/apache/doris/analysis/DeleteStmt.java | 9 +- .../org/apache/doris/load/DeleteHandler.java | 9 +- .../data/delete_p0/test_delete_on_value.out | 39 +++++++-- .../delete_p0/test_delete_on_value.groovy | 82 ++++++++++++++++++- 4 files changed, 128 insertions(+), 11 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/DeleteStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/DeleteStmt.java index f36bf8db02c1e5..351fd4f82cb356 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/DeleteStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/DeleteStmt.java @@ -334,8 +334,13 @@ private void checkDeleteConditions() throws AnalysisException { } if (!column.isKey()) { if (table.getKeysType() == KeysType.AGG_KEYS) { - throw new AnalysisException("delete predicate on value column only supports Unique table and" - + " Duplicate table, but " + "Table[" + table.getName() + "] is an Aggregate table."); + throw new AnalysisException("delete predicate on value column only supports Unique table with" + + " merge-on-write enabled and Duplicate table, but " + "Table[" + table.getName() + + "] is an Aggregate table."); + } else if (table.getKeysType() == KeysType.UNIQUE_KEYS && !table.getEnableUniqueKeyMergeOnWrite()) { + throw new AnalysisException("delete predicate on value column only supports Unique table with" + + " merge-on-write enabled and Duplicate table, but " + "Table[" + table.getName() + + "] is an Aggregate table."); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/DeleteHandler.java b/fe/fe-core/src/main/java/org/apache/doris/load/DeleteHandler.java index c7c5587b7444bc..494fbd5db336e4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/DeleteHandler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/DeleteHandler.java @@ -693,8 +693,13 @@ private void checkDeleteV2(OlapTable table, List partitions, } if (!column.isKey()) { if (table.getKeysType() == KeysType.AGG_KEYS) { - throw new DdlException("delete predicate on value column only supports Unique table and" - + " Duplicate table, but " + "Table[" + table.getName() + "] is an Aggregate table."); + throw new DdlException("delete predicate on value column only supports Unique table with" + + " merge-on-write enabled and Duplicate table, but " + "Table[" + table.getName() + + "] is an Aggregate table."); + } else if (table.getKeysType() == KeysType.UNIQUE_KEYS && !table.getEnableUniqueKeyMergeOnWrite()) { + throw new DdlException("delete predicate on value column only supports Unique table with" + + " merge-on-write enabled and Duplicate table, but " + "Table[" + table.getName() + + "] is an Aggregate table."); } } diff --git a/regression-test/data/delete_p0/test_delete_on_value.out b/regression-test/data/delete_p0/test_delete_on_value.out index 17745f342f021a..987962bb10e12b 100644 --- a/regression-test/data/delete_p0/test_delete_on_value.out +++ b/regression-test/data/delete_p0/test_delete_on_value.out @@ -29,11 +29,6 @@ -- !sql -- 1 1 1 0 2 2 2 0 -3 3 3 0 -4 4 4 0 -5 5 5 0 -6 6 6 0 -7 7 7 0 8 8 8 0 9 9 9 0 @@ -55,13 +50,47 @@ -- !sql -- 1 1 1 0 2 2 2 0 +3 \N \N 1 3 3 3 0 +4 \N \N 1 4 4 4 0 4 4 4 0 +5 \N \N 1 +5 \N \N 1 5 5 5 0 5 5 5 0 +6 \N \N 1 6 6 6 0 +7 \N \N 1 7 7 7 0 8 8 8 0 9 9 9 0 +-- !sql -- +1 1 10 + +-- !sql -- +1 1 5 0 3 5 +1 1 10 0 2 10 + +-- !sql -- + +-- !sql -- +1 \N \N 1 4 \N +1 1 5 0 3 5 +1 1 10 0 2 10 + +-- !sql -- +1 1 10 + +-- !sql -- +1 1 5 0 3 5 +1 1 10 0 2 10 + +-- !sql -- +1 1 5 + +-- !sql -- +1 1 5 0 3 5 +1 1 10 0 2 10 + diff --git a/regression-test/suites/delete_p0/test_delete_on_value.groovy b/regression-test/suites/delete_p0/test_delete_on_value.groovy index 03f04624c5f6bb..353e3d64f76c58 100644 --- a/regression-test/suites/delete_p0/test_delete_on_value.groovy +++ b/regression-test/suites/delete_p0/test_delete_on_value.groovy @@ -20,6 +20,9 @@ suite("test_delete_on_value") { sql "set experimental_enable_nereids_planner=false;" sql 'set enable_nereids_dml=false' + sql "set skip_storage_engine_merge=false;" + sql "set skip_delete_bitmap=false;" + sql "set skip_delete_predicate=false;" def tableName = "test_delete_on_value" sql """ DROP TABLE IF EXISTS ${tableName} """ sql """ CREATE TABLE ${tableName} ( @@ -35,7 +38,6 @@ suite("test_delete_on_value") { "enable_unique_key_merge_on_write" = "true" );""" sql """ insert into ${tableName} values(1,1,1),(2,2,2),(3,3,3),(4,4,4),(5,5,5),(6,6,6),(7,7,7),(8,8,8),(9,9,9); """ - sql "set skip_delete_sign=true;" qt_sql "select * from ${tableName} order by x,y,z;" sql "delete from ${tableName} where y=4;" qt_sql "select * from ${tableName} order by x,y,z;" @@ -52,6 +54,7 @@ suite("test_delete_on_value") { sql "set skip_delete_bitmap=true;" sql "set skip_delete_predicate=true;" qt_sql "select x,y,z,__DORIS_DELETE_SIGN__ from ${tableName} order by x,y,z,__DORIS_DELETE_SIGN__;" + sql "DROP TABLE IF EXISTS ${tableName};" def tableName2 = "test_delete_on_value2" @@ -70,6 +73,81 @@ suite("test_delete_on_value") { sql """ insert into ${tableName2} values(1,1,1); """ test { sql "delete from ${tableName2} where y=4;" - exception "delete predicate on value column only supports Unique table and Duplicate table, but Table[test_delete_on_value2] is an Aggregate table." + exception "delete predicate on value column only supports Unique table with merge-on-write enabled and Duplicate table, but Table[test_delete_on_value2] is an Aggregate table." } + + sql "set skip_storage_engine_merge=false;" + sql "set skip_delete_bitmap=false;" + sql "set skip_delete_predicate=false;" + def tableName3 = "test_delete_on_value_with_seq_col" + sql """ DROP TABLE IF EXISTS ${tableName3} """ + sql """ CREATE TABLE ${tableName3} ( + `x` BIGINT NOT NULL, + `y` BIGINT NULL, + `z` BIGINT NULL) + ENGINE=OLAP + UNIQUE KEY(`x`) + COMMENT 'OLAP' + DISTRIBUTED BY HASH(`x`) BUCKETS 4 + PROPERTIES ( + "replication_num" = "1", + "enable_unique_key_merge_on_write" = "true", + "function_column.sequence_col" = "z" + );""" + sql "insert into ${tableName3} values(1,1,10);" + sql "insert into ${tableName3} values(1,1,5);" + qt_sql "select * from ${tableName3} order by x,y,z;" + sql "set skip_storage_engine_merge=true;" + sql "set skip_delete_bitmap=true;" + sql "set skip_delete_predicate=true;" + qt_sql "select * from ${tableName3} order by x,y,z;" + sql "set skip_storage_engine_merge=false;" + sql "set skip_delete_bitmap=false;" + sql "set skip_delete_predicate=false;" + sql "delete from ${tableName3} where z>=10;" + qt_sql "select * from ${tableName3} order by x,y,z;" + sql "set skip_storage_engine_merge=true;" + sql "set skip_delete_bitmap=true;" + sql "set skip_delete_predicate=true;" + qt_sql "select * from ${tableName3} order by x,y,z;" + sql "DROP TABLE IF EXISTS ${tableName3}" + + sql "set skip_storage_engine_merge=false;" + sql "set skip_delete_bitmap=false;" + sql "set skip_delete_predicate=false;" + def tableName4 = "test_delete_on_value_with_seq_col_mor" + sql """ DROP TABLE IF EXISTS ${tableName4} """ + sql """ CREATE TABLE ${tableName4} ( + `x` BIGINT NOT NULL, + `y` BIGINT NULL, + `z` BIGINT NULL) + ENGINE=OLAP + UNIQUE KEY(`x`) + COMMENT 'OLAP' + DISTRIBUTED BY HASH(`x`) BUCKETS 4 + PROPERTIES ( + "replication_num" = "1", + "function_column.sequence_col" = "z" + );""" + sql "insert into ${tableName4} values(1,1,10);" + sql "insert into ${tableName4} values(1,1,5);" + qt_sql "select * from ${tableName4} order by x,y,z;" + sql "set skip_storage_engine_merge=true;" + sql "set skip_delete_bitmap=true;" + sql "set skip_delete_predicate=true;" + qt_sql "select * from ${tableName4} order by x,y,z;" + sql "set skip_storage_engine_merge=false;" + sql "set skip_delete_bitmap=false;" + sql "set skip_delete_predicate=false;" + sql "delete from ${tableName4} where z>=10;" + qt_sql "select * from ${tableName4} order by x,y,z;" + sql "set skip_storage_engine_merge=true;" + sql "set skip_delete_bitmap=true;" + sql "set skip_delete_predicate=true;" + qt_sql "select * from ${tableName4} order by x,y,z;" + sql "DROP TABLE IF EXISTS ${tableName4};" + + sql "set skip_storage_engine_merge=false;" + sql "set skip_delete_bitmap=false;" + sql "set skip_delete_predicate=false;" } From d51a4fd04f0366f91c135d8f6f52b279089dff28 Mon Sep 17 00:00:00 2001 From: bobhan1 Date: Tue, 15 Aug 2023 20:21:49 +0800 Subject: [PATCH 9/9] fix --- .../data/delete_p0/test_delete_on_value.out | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/regression-test/data/delete_p0/test_delete_on_value.out b/regression-test/data/delete_p0/test_delete_on_value.out index 987962bb10e12b..2bc8846297d3dd 100644 --- a/regression-test/data/delete_p0/test_delete_on_value.out +++ b/regression-test/data/delete_p0/test_delete_on_value.out @@ -29,6 +29,11 @@ -- !sql -- 1 1 1 0 2 2 2 0 +3 3 3 0 +4 4 4 0 +5 5 5 0 +6 6 6 0 +7 7 7 0 8 8 8 0 9 9 9 0 @@ -50,18 +55,12 @@ -- !sql -- 1 1 1 0 2 2 2 0 -3 \N \N 1 3 3 3 0 -4 \N \N 1 4 4 4 0 4 4 4 0 -5 \N \N 1 -5 \N \N 1 5 5 5 0 5 5 5 0 -6 \N \N 1 6 6 6 0 -7 \N \N 1 7 7 7 0 8 8 8 0 9 9 9 0 @@ -76,7 +75,6 @@ -- !sql -- -- !sql -- -1 \N \N 1 4 \N 1 1 5 0 3 5 1 1 10 0 2 10 @@ -88,9 +86,9 @@ 1 1 10 0 2 10 -- !sql -- -1 1 5 -- !sql -- +1 \N \N 1 4 10 1 1 5 0 3 5 1 1 10 0 2 10