From 2e0a3fd728a833004a13262ed5f240e976069d78 Mon Sep 17 00:00:00 2001 From: Kang Date: Sat, 17 Feb 2024 21:21:45 +0800 Subject: [PATCH 1/5] allow index for non key columns of mor unique table --- be/src/olap/rowset/beta_rowset_reader.cpp | 23 +++++++++++++++---- .../rowset/segment_v2/segment_iterator.cpp | 13 ++++++++--- .../org/apache/doris/analysis/IndexDef.java | 9 +++----- .../doris/common/util/PropertyAnalyzer.java | 4 ++-- .../doris/common/PropertyAnalyzerTest.java | 2 +- 5 files changed, 35 insertions(+), 16 deletions(-) diff --git a/be/src/olap/rowset/beta_rowset_reader.cpp b/be/src/olap/rowset/beta_rowset_reader.cpp index 36f4e9262d35eb..cbceb8e33b1cee 100644 --- a/be/src/olap/rowset/beta_rowset_reader.cpp +++ b/be/src/olap/rowset/beta_rowset_reader.cpp @@ -161,10 +161,25 @@ Status BetaRowsetReader::get_segment_iterators(RowsetReaderContext* read_context } if (_read_context->predicates_except_leafnode_of_andnode != nullptr) { - _read_options.column_predicates_except_leafnode_of_andnode.insert( - _read_options.column_predicates_except_leafnode_of_andnode.end(), - _read_context->predicates_except_leafnode_of_andnode->begin(), - _read_context->predicates_except_leafnode_of_andnode->end()); + bool should_push_down = true; + bool should_push_down_value_predicates = _should_push_down_value_predicates(); + for (auto pred : *_read_context->predicates_except_leafnode_of_andnode) { + if (_rowset->keys_type() == UNIQUE_KEYS && + !should_push_down_value_predicates && + !_read_context->tablet_schema->column(pred->column_id()).is_key()) { + VLOG_DEBUG << "do not push down except_leafnode_of_andnode value pred " + << pred->debug_string(); + should_push_down = false; + break; + } + } + + if (should_push_down) { + _read_options.column_predicates_except_leafnode_of_andnode.insert( + _read_options.column_predicates_except_leafnode_of_andnode.end(), + _read_context->predicates_except_leafnode_of_andnode->begin(), + _read_context->predicates_except_leafnode_of_andnode->end()); + } } // Take a delete-bitmap for each segment, the bitmap contains all deletes diff --git a/be/src/olap/rowset/segment_v2/segment_iterator.cpp b/be/src/olap/rowset/segment_v2/segment_iterator.cpp index 10293d2aeb72c9..24c452f383a11d 100644 --- a/be/src/olap/rowset/segment_v2/segment_iterator.cpp +++ b/be/src/olap/rowset/segment_v2/segment_iterator.cpp @@ -459,12 +459,16 @@ Status SegmentIterator::_get_row_ranges_by_column_conditions() { it != _remaining_conjunct_roots.end();) { _pred_except_leafnode_of_andnode_evaluate_result.clear(); auto res = _execute_predicates_except_leafnode_of_andnode(*it); + VLOG_DEBUG << "_execute_predicates_except_leafnode_of_andnode expr: " + << (*it)->debug_string() << " res: " << res; if (res.ok() && _pred_except_leafnode_of_andnode_evaluate_result.size() == 1) { _row_bitmap &= _pred_except_leafnode_of_andnode_evaluate_result[0]; // Delete expr after it obtains the final result. { std::erase_if(_common_expr_ctxs_push_down, [&it](const auto& iter) { return iter->root() == *it; }); + VLOG_DEBUG << "_remaining_conjunct_roots erase expr: " + << (*it)->debug_string(); it = _remaining_conjunct_roots.erase(it); } } else { @@ -720,21 +724,24 @@ Status SegmentIterator::_execute_compound_fn(const std::string& function_name) { auto size = _pred_except_leafnode_of_andnode_evaluate_result.size(); if (function_name == "and") { if (size < 2) { - return Status::InternalError("execute and logic compute error."); + return Status::InvalidArgument("_execute_compound_fn {} arg num {} < 2", function_name, + size); } _pred_except_leafnode_of_andnode_evaluate_result.at(size - 2) &= _pred_except_leafnode_of_andnode_evaluate_result.at(size - 1); _pred_except_leafnode_of_andnode_evaluate_result.pop_back(); } else if (function_name == "or") { if (size < 2) { - return Status::InternalError("execute or logic compute error."); + return Status::InvalidArgument("_execute_compound_fn {} arg num {} < 2", function_name, + size); } _pred_except_leafnode_of_andnode_evaluate_result.at(size - 2) |= _pred_except_leafnode_of_andnode_evaluate_result.at(size - 1); _pred_except_leafnode_of_andnode_evaluate_result.pop_back(); } else if (function_name == "not") { if (size < 1) { - return Status::InternalError("execute not logic compute error."); + return Status::InvalidArgument("_execute_compound_fn {} arg num {} < 1", function_name, + size); } roaring::Roaring tmp = _row_bitmap; tmp -= _pred_except_leafnode_of_andnode_evaluate_result.at(size - 1); diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/IndexDef.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/IndexDef.java index 18d0f22a38c5f7..ee620572f6f22f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/IndexDef.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/IndexDef.java @@ -224,12 +224,9 @@ public void checkColumn(Column column, KeysType keysType, boolean enableUniqueKe throw new AnalysisException(colType + " is not supported in " + indexType.toString() + " index. " + "invalid index: " + indexName); } - if (!column.isKey() - && ((keysType == KeysType.UNIQUE_KEYS && !enableUniqueKeyMergeOnWrite) - || keysType == KeysType.AGG_KEYS)) { - throw new AnalysisException(indexType.toString() - + " index only used in columns of DUP_KEYS/UNIQUE_KEYS MOW table or key columns of all table." - + " invalid index: " + indexName); + if (!column.isKey() && keysType == KeysType.AGG_KEYS) { + throw new AnalysisException("index should only be used in columns of DUP_KEYS/UNIQUE_KEYS table" + + " or key columns of AGG_KEYS table. invalid index: " + indexName); } if (indexType == IndexType.INVERTED) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java index 475d0ae2161be8..98d24c4c681886 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java @@ -466,8 +466,8 @@ public static Set analyzeBloomFilterColumns(Map properti found = true; break; } else { - throw new AnalysisException("Bloom filter index only used in columns of" - + " UNIQUE_KEYS/DUP_KEYS table or key columns of AGG_KEYS table." + throw new AnalysisException("Bloom filter index should only be used in columns" + + " of UNIQUE_KEYS/DUP_KEYS table or key columns of AGG_KEYS table." + " invalid column: " + bfColumn); } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/common/PropertyAnalyzerTest.java b/fe/fe-core/src/test/java/org/apache/doris/common/PropertyAnalyzerTest.java index 64164608bccbd7..eb3500d13d96fa 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/common/PropertyAnalyzerTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/common/PropertyAnalyzerTest.java @@ -122,7 +122,7 @@ public void testBfColumnsError() { try { PropertyAnalyzer.analyzeBloomFilterColumns(properties, columns, KeysType.AGG_KEYS); } catch (AnalysisException e) { - Assert.assertTrue(e.getMessage().contains("Bloom filter index only used in")); + Assert.assertTrue(e.getMessage().contains("Bloom filter index should only be used")); } // reduplicated column From dc8e074104ec87abac214c6ded0720faf007aa38 Mon Sep 17 00:00:00 2001 From: Kang Date: Sat, 17 Feb 2024 21:23:11 +0800 Subject: [PATCH 2/5] add mor index testcase --- .../test_inverted_index_mor.out | 30 +++++++++ .../test_inverted_index_mor.groovy | 63 +++++++++++++++++++ 2 files changed, 93 insertions(+) create mode 100644 regression-test/data/inverted_index_p0/test_inverted_index_mor.out create mode 100644 regression-test/suites/inverted_index_p0/test_inverted_index_mor.groovy diff --git a/regression-test/data/inverted_index_p0/test_inverted_index_mor.out b/regression-test/data/inverted_index_p0/test_inverted_index_mor.out new file mode 100644 index 00000000000000..22cc423acdc803 --- /dev/null +++ b/regression-test/data/inverted_index_p0/test_inverted_index_mor.out @@ -0,0 +1,30 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !11 -- +1 2 12 1.2 1 2 +3 4 34 3.4 3 4 +11 12 1112 11.12 11 22 +13 14 1314 13.14 13 14 + +-- !12 -- +1 2 12 1.2 1 2 +3 4 34 3.4 3 4 +11 12 1112 11.12 11 22 +13 14 1314 13.14 13 14 + +-- !21 -- +1 2 112 11.2 112 +3 4 34 3.4 3 4 +11 12 1112 11.12 11 22 +13 14 1314 13.14 13 14 + +-- !22 -- +3 4 34 3.4 3 4 +11 12 1112 11.12 11 22 +13 14 1314 13.14 13 14 + +-- !23 -- +1 2 112 11.2 112 +3 4 34 3.4 3 4 +11 12 1112 11.12 11 22 +13 14 1314 13.14 13 14 + diff --git a/regression-test/suites/inverted_index_p0/test_inverted_index_mor.groovy b/regression-test/suites/inverted_index_p0/test_inverted_index_mor.groovy new file mode 100644 index 00000000000000..5e4562e08b101f --- /dev/null +++ b/regression-test/suites/inverted_index_p0/test_inverted_index_mor.groovy @@ -0,0 +1,63 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + +suite("test_inverted_index_mor", "p0"){ + def timeout = 60000 + def delta_time = 1000 + def alter_res = "null" + def useTime = 0 + + def indexTblName = "test_inverted_index_mor" + + sql "DROP TABLE IF EXISTS ${indexTblName}" + + sql """ + CREATE TABLE IF NOT EXISTS ${indexTblName}( + `k1` INT, + `k2` INT, + `c_int` INT, + `c_float` FLOAT, + `c_string` STRING, + INDEX idx_k2(`k2`) USING INVERTED COMMENT '', + INDEX idx_c_int(`c_int`) USING INVERTED COMMENT '', + INDEX idx_c_string(`c_string`) USING INVERTED PROPERTIES("parser"="english") COMMENT '' + ) ENGINE=OLAP + UNIQUE KEY(`k1`, `k2`) + COMMENT 'OLAP' + DISTRIBUTED BY HASH(`k1`, `k2`) BUCKETS 1 + PROPERTIES( + "replication_allocation" = "tag.location.default: 1", + "enable_unique_key_merge_on_write" = "false", + "disable_auto_compaction" = "true" + ); + """ + + sql """ INSERT INTO $indexTblName VALUES (1, 2, 12, 1.2, '1 2'), (3, 4, 34, 3.4, '3 4'); """ + sql """ INSERT INTO $indexTblName VALUES (11, 12, 1112, 11.12, '11 22'), (13, 14, 1314, 13.14, '13 14'); """ + + // original data + qt_11 """ SELECT * FROM $indexTblName ORDER BY k1,k2 """ + + // + qt_12 """ SELECT * FROM $indexTblName WHERE k2 > 2 OR c_int = 12 ORDER BY k1,k2 """ + + sql """ INSERT INTO $indexTblName VALUES (1, 2, 112, 11.2, '112'); """ + qt_21 """ SELECT * FROM $indexTblName ORDER BY k1,k2 """ + qt_22 """ SELECT * FROM $indexTblName WHERE k2 > 2 OR c_int = 12 ORDER BY k1,k2 """ + qt_23 """ SELECT * FROM $indexTblName WHERE k2 > 2 OR c_int = 112 ORDER BY k1,k2 """ +} \ No newline at end of file From ab06f4184c3c3a9bd182682b81437bf964d3733f Mon Sep 17 00:00:00 2001 From: Kang Date: Sat, 17 Feb 2024 21:23:52 +0800 Subject: [PATCH 3/5] add back mor bitmap index testcase --- .../datatype_p0/scalar_types/load.groovy | 54 +++++++++++++++++++ .../sql/unique2_bitmap_index_q01_bool.sql | 1 + .../sql/unique2_bitmap_index_q02_tinyint.sql | 1 + .../sql/unique2_bitmap_index_q03_smallint.sql | 1 + .../sql/unique2_bitmap_index_q04_int.sql | 1 + .../sql/unique2_bitmap_index_q05_bigint.sql | 1 + .../sql/unique2_bitmap_index_q06_largeint.sql | 1 + .../sql/unique2_bitmap_index_q07_float.sql | 1 + .../sql/unique2_bitmap_index_q08_double.sql | 1 + .../sql/unique2_bitmap_index_q09_decimal.sql | 1 + .../unique2_bitmap_index_q10_decimalv3.sql | 1 + .../sql/unique2_bitmap_index_q11_date.sql | 1 + .../sql/unique2_bitmap_index_q12_datetime.sql | 1 + .../sql/unique2_bitmap_index_q13_datev2.sql | 1 + .../unique2_bitmap_index_q14_datetimev2.sql | 1 + .../sql/unique2_bitmap_index_q15_char.sql | 1 + .../sql/unique2_bitmap_index_q16_varchar.sql | 1 + .../sql/unique2_bitmap_index_q17_string.sql | 1 + 18 files changed, 71 insertions(+) create mode 100644 regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q01_bool.sql create mode 100644 regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q02_tinyint.sql create mode 100644 regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q03_smallint.sql create mode 100644 regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q04_int.sql create mode 100644 regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q05_bigint.sql create mode 100644 regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q06_largeint.sql create mode 100644 regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q07_float.sql create mode 100644 regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q08_double.sql create mode 100644 regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q09_decimal.sql create mode 100644 regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q10_decimalv3.sql create mode 100644 regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q11_date.sql create mode 100644 regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q12_datetime.sql create mode 100644 regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q13_datev2.sql create mode 100644 regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q14_datetimev2.sql create mode 100644 regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q15_char.sql create mode 100644 regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q16_varchar.sql create mode 100644 regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q17_string.sql diff --git a/regression-test/suites/datatype_p0/scalar_types/load.groovy b/regression-test/suites/datatype_p0/scalar_types/load.groovy index 0c5c81576a3252..7367a1468fcdbd 100644 --- a/regression-test/suites/datatype_p0/scalar_types/load.groovy +++ b/regression-test/suites/datatype_p0/scalar_types/load.groovy @@ -305,6 +305,60 @@ suite("test_scalar_types_load", "p0") { `c_float`, `c_double`, `c_decimal`, `c_date`, `c_datetime`, `c_datev2`, `c_char`, `c_varchar`, `c_string` FROM tbl_scalar_types_dup""" + // define unique key table2 enable mow + testTable = "tbl_scalar_types_unique2_bitmapindex" + sql "DROP TABLE IF EXISTS ${testTable}" + sql """ + CREATE TABLE IF NOT EXISTS ${testTable} ( + `c_datetimev2` datetimev2(0) NULL, + `c_bigint` bigint(20) NULL, + `c_decimalv3` decimalv3(20, 3) NULL, + `c_bool` boolean NULL, + `c_tinyint` tinyint(4) NULL, + `c_smallint` smallint(6) NULL, + `c_int` int(11) NULL, + `c_largeint` largeint(40) NULL, + `c_float` float NULL, + `c_double` double NULL, + `c_decimal` decimal(20, 3) NULL, + `c_date` date NULL, + `c_datetime` datetime NULL, + `c_datev2` datev2 NULL, + `c_char` char(15) NULL, + `c_varchar` varchar(100) NULL, + `c_string` text NULL, + INDEX idx_c_bool (c_bool) USING BITMAP, + INDEX idx_c_tinyint (c_tinyint) USING BITMAP, + INDEX idx_c_smallint (c_smallint) USING BITMAP, + INDEX idx_c_int (c_int) USING BITMAP, + INDEX idx_c_bigint (c_bigint) USING BITMAP, + INDEX idx_c_largeint (c_largeint) USING BITMAP, + INDEX idx_c_decimal (c_decimal) USING BITMAP, + INDEX idx_c_decimalv3 (c_decimalv3) USING BITMAP, + INDEX idx_c_date (c_date) USING BITMAP, + INDEX idx_c_datetime (c_datetime) USING BITMAP, + INDEX idx_c_datev2 (c_datev2) USING BITMAP, + INDEX idx_c_datetimev2 (c_datetimev2) USING BITMAP, + INDEX idx_c_char (c_char) USING BITMAP, + INDEX idx_c_varchar (c_varchar) USING BITMAP, + INDEX idx_c_string (c_string) USING BITMAP + ) ENGINE=OLAP + UNIQUE KEY(`c_datetimev2`, `c_bigint`, `c_decimalv3`) + COMMENT 'OLAP' + DISTRIBUTED BY HASH(`c_bigint`) BUCKETS 10 + PROPERTIES("replication_num" = "1", "enable_unique_key_merge_on_write" = "false"); + """ + + // insert data into unique key table2 2 times + sql """INSERT INTO ${testTable} SELECT `c_datetimev2`, `c_bigint`, `c_decimalv3`, + `c_bool`, `c_tinyint`, `c_smallint`, `c_int`, `c_largeint`, + `c_float`, `c_double`, `c_decimal`, `c_date`, `c_datetime`, `c_datev2`, + `c_char`, `c_varchar`, `c_string` FROM tbl_scalar_types_dup""" + sql """INSERT INTO ${testTable} SELECT `c_datetimev2`, `c_bigint`, `c_decimalv3`, + `c_bool`, `c_tinyint`, `c_smallint`, `c_int`, `c_largeint`, + `c_float`, `c_double`, `c_decimal`, `c_date`, `c_datetime`, `c_datev2`, + `c_char`, `c_varchar`, `c_string` FROM tbl_scalar_types_dup""" + // define dup key table with index testTable = "tbl_scalar_types_dup_inverted_index" diff --git a/regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q01_bool.sql b/regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q01_bool.sql new file mode 100644 index 00000000000000..3e38418168833e --- /dev/null +++ b/regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q01_bool.sql @@ -0,0 +1 @@ +SELECT count() FROM tbl_scalar_types_unique2_bitmapindex WHERE c_bool = 1; diff --git a/regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q02_tinyint.sql b/regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q02_tinyint.sql new file mode 100644 index 00000000000000..ed75ebca45779e --- /dev/null +++ b/regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q02_tinyint.sql @@ -0,0 +1 @@ +SELECT count() FROM tbl_scalar_types_unique2_bitmapindex WHERE c_tinyint = 14; diff --git a/regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q03_smallint.sql b/regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q03_smallint.sql new file mode 100644 index 00000000000000..faa2159f0dda2c --- /dev/null +++ b/regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q03_smallint.sql @@ -0,0 +1 @@ +SELECT * FROM tbl_scalar_types_unique2_bitmapindex WHERE c_smallint = 12321 ORDER BY c_bigint, c_largeint; diff --git a/regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q04_int.sql b/regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q04_int.sql new file mode 100644 index 00000000000000..db82b79873aa29 --- /dev/null +++ b/regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q04_int.sql @@ -0,0 +1 @@ +SELECT * FROM tbl_scalar_types_unique2_bitmapindex WHERE c_int = -16441 ORDER BY c_bigint, c_largeint; diff --git a/regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q05_bigint.sql b/regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q05_bigint.sql new file mode 100644 index 00000000000000..9ef4811f62aefd --- /dev/null +++ b/regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q05_bigint.sql @@ -0,0 +1 @@ +SELECT * FROM tbl_scalar_types_unique2_bitmapindex WHERE c_bigint = 859063837 ORDER BY c_bigint, c_largeint; diff --git a/regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q06_largeint.sql b/regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q06_largeint.sql new file mode 100644 index 00000000000000..2a6b48064c24a7 --- /dev/null +++ b/regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q06_largeint.sql @@ -0,0 +1 @@ +SELECT * FROM tbl_scalar_types_unique2_bitmapindex WHERE c_largeint = 1524137702 ORDER BY c_bigint, c_largeint; diff --git a/regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q07_float.sql b/regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q07_float.sql new file mode 100644 index 00000000000000..1ae3d82f151acb --- /dev/null +++ b/regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q07_float.sql @@ -0,0 +1 @@ +SELECT * FROM tbl_scalar_types_unique2_bitmapindex WHERE c_float = -13023.051 ORDER BY c_bigint, c_largeint; diff --git a/regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q08_double.sql b/regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q08_double.sql new file mode 100644 index 00000000000000..8c21bc08b79394 --- /dev/null +++ b/regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q08_double.sql @@ -0,0 +1 @@ +SELECT * FROM tbl_scalar_types_unique2_bitmapindex WHERE c_double = -679666086.055166 ORDER BY c_bigint, c_largeint; diff --git a/regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q09_decimal.sql b/regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q09_decimal.sql new file mode 100644 index 00000000000000..9fd1e832bed7f3 --- /dev/null +++ b/regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q09_decimal.sql @@ -0,0 +1 @@ +SELECT * FROM tbl_scalar_types_unique2_bitmapindex WHERE c_decimal = 5355625357307892.981 ORDER BY c_bigint, c_largeint; diff --git a/regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q10_decimalv3.sql b/regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q10_decimalv3.sql new file mode 100644 index 00000000000000..ee676941451607 --- /dev/null +++ b/regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q10_decimalv3.sql @@ -0,0 +1 @@ +SELECT * FROM tbl_scalar_types_unique2_bitmapindex WHERE c_decimalv3 = 89599956401481176.347 ORDER BY c_bigint, c_largeint; diff --git a/regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q11_date.sql b/regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q11_date.sql new file mode 100644 index 00000000000000..7f5d383765fa38 --- /dev/null +++ b/regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q11_date.sql @@ -0,0 +1 @@ +SELECT * FROM tbl_scalar_types_unique2_bitmapindex WHERE c_date = '2022-10-09' ORDER BY c_bigint, c_largeint; diff --git a/regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q12_datetime.sql b/regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q12_datetime.sql new file mode 100644 index 00000000000000..a1d7728c621b15 --- /dev/null +++ b/regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q12_datetime.sql @@ -0,0 +1 @@ +SELECT * FROM tbl_scalar_types_unique2_bitmapindex WHERE c_datetime = '2022-07-31 12:07:00' ORDER BY c_bigint, c_largeint; diff --git a/regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q13_datev2.sql b/regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q13_datev2.sql new file mode 100644 index 00000000000000..eee7b1110841b9 --- /dev/null +++ b/regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q13_datev2.sql @@ -0,0 +1 @@ +SELECT * FROM tbl_scalar_types_unique2_bitmapindex WHERE c_datev2 = '2022-03-06' ORDER BY c_bigint, c_largeint; diff --git a/regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q14_datetimev2.sql b/regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q14_datetimev2.sql new file mode 100644 index 00000000000000..83b86dadbf2b1c --- /dev/null +++ b/regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q14_datetimev2.sql @@ -0,0 +1 @@ +SELECT * FROM tbl_scalar_types_unique2_bitmapindex WHERE c_datetimev2 = '2022-11-18 06:05:56' ORDER BY c_bigint, c_largeint; diff --git a/regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q15_char.sql b/regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q15_char.sql new file mode 100644 index 00000000000000..67486dde69fd33 --- /dev/null +++ b/regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q15_char.sql @@ -0,0 +1 @@ +SELECT * FROM tbl_scalar_types_unique2_bitmapindex WHERE c_char = '202.171.74.217' ORDER BY c_bigint, c_largeint; diff --git a/regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q16_varchar.sql b/regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q16_varchar.sql new file mode 100644 index 00000000000000..677bc5614a819a --- /dev/null +++ b/regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q16_varchar.sql @@ -0,0 +1 @@ +SELECT * FROM tbl_scalar_types_unique2_bitmapindex WHERE c_varchar = 'modi_a@Feedmix.biz' ORDER BY c_bigint, c_largeint; diff --git a/regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q17_string.sql b/regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q17_string.sql new file mode 100644 index 00000000000000..b02fb1d72e87dd --- /dev/null +++ b/regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q17_string.sql @@ -0,0 +1 @@ +SELECT * FROM tbl_scalar_types_unique2_bitmapindex WHERE c_string = 'Reinke Junction 63' ORDER BY c_bigint, c_largeint; From 660dbdb0d8c53e45c12fe4193b06078b07af5340 Mon Sep 17 00:00:00 2001 From: Kang Date: Sat, 17 Feb 2024 21:24:37 +0800 Subject: [PATCH 4/5] change index error message --- regression-test/suites/index_p0/test_bitmap_index.groovy | 2 +- .../suites/inverted_index_p0/test_bitmap_index.groovy | 2 +- .../suites/inverted_index_p0/test_inverted_index.groovy | 2 +- .../test_segcompaction_unique_keys_index.groovy | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/regression-test/suites/index_p0/test_bitmap_index.groovy b/regression-test/suites/index_p0/test_bitmap_index.groovy index a5cbe31504f68a..65f75e609a18dd 100644 --- a/regression-test/suites/index_p0/test_bitmap_index.groovy +++ b/regression-test/suites/index_p0/test_bitmap_index.groovy @@ -175,7 +175,7 @@ suite("test_bitmap_index") { } test{ sql "ALTER TABLE ${tbName2} ADD INDEX index16 (v1) USING BITMAP;" - exception "errCode = 2, detailMessage = INVERTED index only used in columns of DUP_KEYS/UNIQUE_KEYS MOW table or key columns of all table. invalid index: index16" + exception "errCode = 2, detailMessage = index should only be used in columns of DUP_KEYS/UNIQUE_KEYS table or key columns of AGG_KEYS table. invalid index: index16" } sql "insert into ${tbName2} values(1,1,1,1,'1','1','2022-05-31','2022-05-31 10:00:00',1,1.0,1,'2022-05-31','2022-05-31 10:00:00.111111','2022-05-31 10:00:00.111111','2022-05-31 10:00:00.111111',1);" diff --git a/regression-test/suites/inverted_index_p0/test_bitmap_index.groovy b/regression-test/suites/inverted_index_p0/test_bitmap_index.groovy index c6bbc2089249f1..2aff10609086b3 100644 --- a/regression-test/suites/inverted_index_p0/test_bitmap_index.groovy +++ b/regression-test/suites/inverted_index_p0/test_bitmap_index.groovy @@ -155,7 +155,7 @@ suite("test_bitmap_index", "inverted_index") { } test{ sql "ALTER TABLE ${tbName2} ADD INDEX index12 (v1) USING BITMAP;" - exception "errCode = 2, detailMessage = INVERTED index only used in columns of DUP_KEYS/UNIQUE_KEYS MOW table or key columns of all table. invalid index: index12" + exception "errCode = 2, detailMessage = index should only be used in columns of DUP_KEYS/UNIQUE_KEYS table or key columns of AGG_KEYS table. invalid index: index12" } sql "insert into ${tbName2} values(1,1,1,1,'1','1','2022-05-31','2022-05-31 10:00:00',1,1.0,1,1);" diff --git a/regression-test/suites/inverted_index_p0/test_inverted_index.groovy b/regression-test/suites/inverted_index_p0/test_inverted_index.groovy index 60d85999763933..033366b2eba3a7 100644 --- a/regression-test/suites/inverted_index_p0/test_inverted_index.groovy +++ b/regression-test/suites/inverted_index_p0/test_inverted_index.groovy @@ -175,7 +175,7 @@ suite("test_inverted_index", "inverted_index") { } test{ sql "ALTER TABLE ${tbName2} ADD INDEX index16 (v1) USING INVERTED;" - exception "errCode = 2, detailMessage = INVERTED index only used in columns of DUP_KEYS/UNIQUE_KEYS MOW table or key columns of all table. invalid index: index16" + exception "errCode = 2, detailMessage = index should only be used in columns of DUP_KEYS/UNIQUE_KEYS table or key columns of AGG_KEYS table. invalid index: index16" } sql "insert into ${tbName2} values(1,1,1,1,'1','1','2022-05-31','2022-05-31 10:00:00',1,1.0,1,'2022-05-31','2022-05-31 10:00:00.111111','2022-05-31 10:00:00.111111','2022-05-31 10:00:00.111111',1);" diff --git a/regression-test/suites/segcompaction_p2/test_segcompaction_unique_keys_index.groovy b/regression-test/suites/segcompaction_p2/test_segcompaction_unique_keys_index.groovy index 4e716f48951b82..3a9503ba723e94 100644 --- a/regression-test/suites/segcompaction_p2/test_segcompaction_unique_keys_index.groovy +++ b/regression-test/suites/segcompaction_p2/test_segcompaction_unique_keys_index.groovy @@ -68,7 +68,7 @@ suite("test_segcompaction_unique_keys_index") { UNIQUE KEY(`col_0`) DISTRIBUTED BY HASH(`col_0`) BUCKETS 1 PROPERTIES ( "replication_num" = "1" ); """ - exception "INVERTED index only used in columns of DUP_KEYS/UNIQUE_KEYS MOW table or key columns of all table. invalid indexName: index_col_1" + exception "index should only be used in columns of DUP_KEYS/UNIQUE_KEYS table or key columns of AGG_KEYS table. invalid indexName: index_col_1" } } finally { try_sql("DROP TABLE IF EXISTS ${tableName}") From b6d73b97ae3350727cbb9d6f87a08af72defdb72 Mon Sep 17 00:00:00 2001 From: Kang Date: Sat, 17 Feb 2024 21:30:08 +0800 Subject: [PATCH 5/5] clang format --- be/src/olap/rowset/beta_rowset_reader.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/be/src/olap/rowset/beta_rowset_reader.cpp b/be/src/olap/rowset/beta_rowset_reader.cpp index cbceb8e33b1cee..0f8ee1715621db 100644 --- a/be/src/olap/rowset/beta_rowset_reader.cpp +++ b/be/src/olap/rowset/beta_rowset_reader.cpp @@ -164,8 +164,7 @@ Status BetaRowsetReader::get_segment_iterators(RowsetReaderContext* read_context bool should_push_down = true; bool should_push_down_value_predicates = _should_push_down_value_predicates(); for (auto pred : *_read_context->predicates_except_leafnode_of_andnode) { - if (_rowset->keys_type() == UNIQUE_KEYS && - !should_push_down_value_predicates && + if (_rowset->keys_type() == UNIQUE_KEYS && !should_push_down_value_predicates && !_read_context->tablet_schema->column(pred->column_id()).is_key()) { VLOG_DEBUG << "do not push down except_leafnode_of_andnode value pred " << pred->debug_string();