diff --git a/be/src/olap/compaction.cpp b/be/src/olap/compaction.cpp index 99969cc5b72a06..c94e6c0b4d5983 100644 --- a/be/src/olap/compaction.cpp +++ b/be/src/olap/compaction.cpp @@ -600,7 +600,8 @@ Status Compaction::construct_output_rowset_writer(RowsetWriterContext& ctx, bool ctx.newest_write_timestamp = _newest_write_timestamp; ctx.write_type = DataWriteType::TYPE_COMPACTION; if (config::inverted_index_compaction_enable && - ((_tablet->keys_type() == KeysType::UNIQUE_KEYS || + (((_tablet->keys_type() == KeysType::UNIQUE_KEYS && + _tablet->enable_unique_key_merge_on_write()) || _tablet->keys_type() == KeysType::DUP_KEYS))) { for (const auto& index : _cur_tablet_schema->indexes()) { if (index.index_type() == IndexType::INVERTED) { diff --git a/be/src/olap/rowset/beta_rowset_reader.cpp b/be/src/olap/rowset/beta_rowset_reader.cpp index 568fa89d848b81..d57873619b1c1b 100644 --- a/be/src/olap/rowset/beta_rowset_reader.cpp +++ b/be/src/olap/rowset/beta_rowset_reader.cpp @@ -163,10 +163,24 @@ Status BetaRowsetReader::get_segment_iterators(RowsetReaderContext* read_context } if (_read_context->predicates_except_leafnode_of_andnode != nullptr) { - _read_options.column_predicates_except_leafnode_of_andnode.insert( - _read_options.column_predicates_except_leafnode_of_andnode.end(), - _read_context->predicates_except_leafnode_of_andnode->begin(), - _read_context->predicates_except_leafnode_of_andnode->end()); + bool should_push_down = true; + bool should_push_down_value_predicates = _should_push_down_value_predicates(); + for (auto pred : *_read_context->predicates_except_leafnode_of_andnode) { + if (_rowset->keys_type() == UNIQUE_KEYS && !should_push_down_value_predicates && + !_read_context->tablet_schema->column(pred->column_id()).is_key()) { + VLOG_DEBUG << "do not push down except_leafnode_of_andnode value pred " + << pred->debug_string(); + should_push_down = false; + break; + } + } + + if (should_push_down) { + _read_options.column_predicates_except_leafnode_of_andnode.insert( + _read_options.column_predicates_except_leafnode_of_andnode.end(), + _read_context->predicates_except_leafnode_of_andnode->begin(), + _read_context->predicates_except_leafnode_of_andnode->end()); + } } // Take a delete-bitmap for each segment, the bitmap contains all deletes diff --git a/be/src/olap/rowset/segment_v2/segment_iterator.cpp b/be/src/olap/rowset/segment_v2/segment_iterator.cpp index 8cc25d39898172..f1cb1f2c61c39f 100644 --- a/be/src/olap/rowset/segment_v2/segment_iterator.cpp +++ b/be/src/olap/rowset/segment_v2/segment_iterator.cpp @@ -497,12 +497,16 @@ Status SegmentIterator::_get_row_ranges_by_column_conditions() { it != _remaining_conjunct_roots.end();) { _pred_except_leafnode_of_andnode_evaluate_result.clear(); auto res = _execute_predicates_except_leafnode_of_andnode(*it); + VLOG_DEBUG << "_execute_predicates_except_leafnode_of_andnode expr: " + << (*it)->debug_string() << " res: " << res; if (res.ok() && _pred_except_leafnode_of_andnode_evaluate_result.size() == 1) { _row_bitmap &= _pred_except_leafnode_of_andnode_evaluate_result[0]; // Delete expr after it obtains the final result. { std::erase_if(_common_expr_ctxs_push_down, [&it](const auto& iter) { return iter->root() == *it; }); + VLOG_DEBUG << "_remaining_conjunct_roots erase expr: " + << (*it)->debug_string(); it = _remaining_conjunct_roots.erase(it); } } else { @@ -771,21 +775,24 @@ Status SegmentIterator::_execute_compound_fn(const std::string& function_name) { auto size = _pred_except_leafnode_of_andnode_evaluate_result.size(); if (function_name == "and") { if (size < 2) { - return Status::Uninitialized("execute and logic compute error."); + return Status::InvalidArgument("_execute_compound_fn {} arg num {} < 2", function_name, + size); } _pred_except_leafnode_of_andnode_evaluate_result.at(size - 2) &= _pred_except_leafnode_of_andnode_evaluate_result.at(size - 1); _pred_except_leafnode_of_andnode_evaluate_result.pop_back(); } else if (function_name == "or") { if (size < 2) { - return Status::Uninitialized("execute or logic compute error."); + return Status::InvalidArgument("_execute_compound_fn {} arg num {} < 2", function_name, + size); } _pred_except_leafnode_of_andnode_evaluate_result.at(size - 2) |= _pred_except_leafnode_of_andnode_evaluate_result.at(size - 1); _pred_except_leafnode_of_andnode_evaluate_result.pop_back(); } else if (function_name == "not") { if (size < 1) { - return Status::Uninitialized("execute not logic compute error."); + return Status::InvalidArgument("_execute_compound_fn {} arg num {} < 1", function_name, + size); } roaring::Roaring tmp = _row_bitmap; tmp -= _pred_except_leafnode_of_andnode_evaluate_result.at(size - 1); diff --git a/fe/fe-core/src/main/cup/sql_parser.cup b/fe/fe-core/src/main/cup/sql_parser.cup index 5ad7fa07756eee..fb37eee9c76ca9 100644 --- a/fe/fe-core/src/main/cup/sql_parser.cup +++ b/fe/fe-core/src/main/cup/sql_parser.cup @@ -3812,7 +3812,7 @@ opt_index_type ::= :} | KW_USING KW_BITMAP {: - RESULT = IndexDef.IndexType.INVERTED; + RESULT = IndexDef.IndexType.BITMAP; :} | KW_USING KW_NGRAM_BF {: diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/IndexDef.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/IndexDef.java index 6612a27dfaa529..b23bb3f22770fe 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/IndexDef.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/IndexDef.java @@ -224,12 +224,16 @@ public void checkColumn(Column column, KeysType keysType, boolean enableUniqueKe throw new AnalysisException(colType + " is not supported in " + indexType.toString() + " index. " + "invalid index: " + indexName); } - if (!column.isKey() - && ((keysType == KeysType.UNIQUE_KEYS && !enableUniqueKeyMergeOnWrite) - || keysType == KeysType.AGG_KEYS)) { - throw new AnalysisException(indexType.toString() - + " index only used in columns of DUP_KEYS/UNIQUE_KEYS MOW table or key columns of all table." - + " invalid index: " + indexName); + if (!column.isKey()) { + if (keysType == KeysType.AGG_KEYS) { + throw new AnalysisException("index should only be used in columns of DUP_KEYS/UNIQUE_KEYS table" + + " or key columns of AGG_KEYS table. invalid index: " + indexName); + } else if (keysType == KeysType.UNIQUE_KEYS && !enableUniqueKeyMergeOnWrite + && indexType == IndexType.INVERTED && properties != null + && properties.containsKey(InvertedIndexUtil.INVERTED_INDEX_PARSER_KEY)) { + throw new AnalysisException("INVERTED index with parser can NOT be used in value columns of" + + " UNIQUE_KEYS table with merge_on_write disable. invalid index: " + indexName); + } } if (indexType == IndexType.INVERTED) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java index fd3b8243766706..42d62046948ffe 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java @@ -525,8 +525,8 @@ public static Set analyzeBloomFilterColumns(Map properti found = true; break; } else { - throw new AnalysisException("Bloom filter index only used in columns of" - + " UNIQUE_KEYS/DUP_KEYS table or key columns of AGG_KEYS table." + throw new AnalysisException("Bloom filter index should only be used in columns" + + " of UNIQUE_KEYS/DUP_KEYS table or key columns of AGG_KEYS table." + " invalid column: " + bfColumn); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java index 550785999f4167..4fd62427823d25 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java @@ -2537,10 +2537,6 @@ public IndexDefinition visitIndexDef(IndexDefContext ctx) { Map properties = visitPropertyItemList(ctx.properties); String indexType = ctx.indexType != null ? ctx.indexType.getText().toUpperCase() : null; String comment = ctx.comment != null ? ctx.comment.getText() : ""; - // change BITMAP index to INVERTED index - if (indexType.equalsIgnoreCase("BITMAP")) { - indexType = "INVERTED"; - } return new IndexDefinition(indexName, indexCols, indexType, properties, comment); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/IndexDefinition.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/IndexDefinition.java index 6f298e73e35e0b..ba56cfa4a8bba9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/IndexDefinition.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/IndexDefinition.java @@ -110,12 +110,16 @@ public void checkColumn(ColumnDefinition column, KeysType keysType, throw new AnalysisException(colType + " is not supported in " + indexType.toString() + " index. " + "invalid index: " + name); } - if (!column.isKey() - && ((keysType == KeysType.UNIQUE_KEYS && !enableUniqueKeyMergeOnWrite) - || keysType == KeysType.AGG_KEYS)) { - throw new AnalysisException(indexType.toString() - + " index only used in columns of DUP_KEYS/UNIQUE_KEYS MOW table or key columns of all table." - + " invalid index: " + name); + if (!column.isKey()) { + if (keysType == KeysType.AGG_KEYS) { + throw new AnalysisException("index should only be used in columns of DUP_KEYS/UNIQUE_KEYS table" + + " or key columns of AGG_KEYS table. invalid index: " + name); + } else if (keysType == KeysType.UNIQUE_KEYS && !enableUniqueKeyMergeOnWrite + && indexType == IndexType.INVERTED && properties != null + && properties.containsKey(InvertedIndexUtil.INVERTED_INDEX_PARSER_KEY)) { + throw new AnalysisException("INVERTED index with parser can NOT be used in value columns of" + + " UNIQUE_KEYS table with merge_on_write disable. invalid index: " + name); + } } if (indexType == IndexType.INVERTED) { diff --git a/fe/fe-core/src/test/java/org/apache/doris/common/PropertyAnalyzerTest.java b/fe/fe-core/src/test/java/org/apache/doris/common/PropertyAnalyzerTest.java index 64164608bccbd7..eb3500d13d96fa 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/common/PropertyAnalyzerTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/common/PropertyAnalyzerTest.java @@ -122,7 +122,7 @@ public void testBfColumnsError() { try { PropertyAnalyzer.analyzeBloomFilterColumns(properties, columns, KeysType.AGG_KEYS); } catch (AnalysisException e) { - Assert.assertTrue(e.getMessage().contains("Bloom filter index only used in")); + Assert.assertTrue(e.getMessage().contains("Bloom filter index should only be used")); } // reduplicated column diff --git a/regression-test/data/index_p0/test_bitmap_index.out b/regression-test/data/index_p0/test_bitmap_index.out index b26540484409d5..1ee61ff37e9aef 100644 --- a/regression-test/data/index_p0/test_bitmap_index.out +++ b/regression-test/data/index_p0/test_bitmap_index.out @@ -65,9 +65,9 @@ v1 INT Yes false \N NONE create_time DATETIME No true \N vid VARCHAR(64) No true \N report_time DATETIME Yes true \N -block_version INT Yes false \N NONE -vehicle_mode INT Yes false \N NONE -usage_mode INT Yes false \N NONE +block_version INT Yes false \N REPLACE +vehicle_mode INT Yes false \N REPLACE +usage_mode INT Yes false \N REPLACE -- !sql -- 2 diff --git a/regression-test/data/inverted_index_p0/test_inverted_index_mor.out b/regression-test/data/inverted_index_p0/test_inverted_index_mor.out new file mode 100644 index 00000000000000..22cc423acdc803 --- /dev/null +++ b/regression-test/data/inverted_index_p0/test_inverted_index_mor.out @@ -0,0 +1,30 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !11 -- +1 2 12 1.2 1 2 +3 4 34 3.4 3 4 +11 12 1112 11.12 11 22 +13 14 1314 13.14 13 14 + +-- !12 -- +1 2 12 1.2 1 2 +3 4 34 3.4 3 4 +11 12 1112 11.12 11 22 +13 14 1314 13.14 13 14 + +-- !21 -- +1 2 112 11.2 112 +3 4 34 3.4 3 4 +11 12 1112 11.12 11 22 +13 14 1314 13.14 13 14 + +-- !22 -- +3 4 34 3.4 3 4 +11 12 1112 11.12 11 22 +13 14 1314 13.14 13 14 + +-- !23 -- +1 2 112 11.2 112 +3 4 34 3.4 3 4 +11 12 1112 11.12 11 22 +13 14 1314 13.14 13 14 + diff --git a/regression-test/data/segcompaction_p2/test_segcompaction_unique_keys_index.out b/regression-test/data/segcompaction_p2/test_segcompaction_unique_keys_index.out new file mode 100644 index 00000000000000..80f1df28b54620 --- /dev/null +++ b/regression-test/data/segcompaction_p2/test_segcompaction_unique_keys_index.out @@ -0,0 +1,9 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !select_default -- + +-- !select_default -- +0 + +-- !select_default -- +0 + diff --git a/regression-test/suites/datatype_p0/scalar_types/load.groovy b/regression-test/suites/datatype_p0/scalar_types/load.groovy index 43f6c9bffffc95..91cd4eb1f1f1d6 100644 --- a/regression-test/suites/datatype_p0/scalar_types/load.groovy +++ b/regression-test/suites/datatype_p0/scalar_types/load.groovy @@ -302,6 +302,60 @@ suite("test_scalar_types_load", "p0") { `c_float`, `c_double`, `c_decimal`, `c_date`, `c_datetime`, `c_datev2`, `c_char`, `c_varchar`, `c_string` FROM tbl_scalar_types_dup""" + // define unique key table2 enable mow + testTable = "tbl_scalar_types_unique2_bitmapindex" + sql "DROP TABLE IF EXISTS ${testTable}" + sql """ + CREATE TABLE IF NOT EXISTS ${testTable} ( + `c_datetimev2` datetimev2(0) NULL, + `c_bigint` bigint(20) NULL, + `c_decimalv3` decimalv3(20, 3) NULL, + `c_bool` boolean NULL, + `c_tinyint` tinyint(4) NULL, + `c_smallint` smallint(6) NULL, + `c_int` int(11) NULL, + `c_largeint` largeint(40) NULL, + `c_float` float NULL, + `c_double` double NULL, + `c_decimal` decimal(20, 3) NULL, + `c_date` date NULL, + `c_datetime` datetime NULL, + `c_datev2` datev2 NULL, + `c_char` char(15) NULL, + `c_varchar` varchar(100) NULL, + `c_string` text NULL, + INDEX idx_c_bool (c_bool) USING BITMAP, + INDEX idx_c_tinyint (c_tinyint) USING BITMAP, + INDEX idx_c_smallint (c_smallint) USING BITMAP, + INDEX idx_c_int (c_int) USING BITMAP, + INDEX idx_c_bigint (c_bigint) USING BITMAP, + INDEX idx_c_largeint (c_largeint) USING BITMAP, + INDEX idx_c_decimal (c_decimal) USING BITMAP, + INDEX idx_c_decimalv3 (c_decimalv3) USING BITMAP, + INDEX idx_c_date (c_date) USING BITMAP, + INDEX idx_c_datetime (c_datetime) USING BITMAP, + INDEX idx_c_datev2 (c_datev2) USING BITMAP, + INDEX idx_c_datetimev2 (c_datetimev2) USING BITMAP, + INDEX idx_c_char (c_char) USING BITMAP, + INDEX idx_c_varchar (c_varchar) USING BITMAP, + INDEX idx_c_string (c_string) USING BITMAP + ) ENGINE=OLAP + UNIQUE KEY(`c_datetimev2`, `c_bigint`, `c_decimalv3`) + COMMENT 'OLAP' + DISTRIBUTED BY HASH(`c_bigint`) BUCKETS 10 + PROPERTIES("replication_num" = "1", "enable_unique_key_merge_on_write" = "false"); + """ + + // insert data into unique key table2 2 times + sql """INSERT INTO ${testTable} SELECT `c_datetimev2`, `c_bigint`, `c_decimalv3`, + `c_bool`, `c_tinyint`, `c_smallint`, `c_int`, `c_largeint`, + `c_float`, `c_double`, `c_decimal`, `c_date`, `c_datetime`, `c_datev2`, + `c_char`, `c_varchar`, `c_string` FROM tbl_scalar_types_dup""" + sql """INSERT INTO ${testTable} SELECT `c_datetimev2`, `c_bigint`, `c_decimalv3`, + `c_bool`, `c_tinyint`, `c_smallint`, `c_int`, `c_largeint`, + `c_float`, `c_double`, `c_decimal`, `c_date`, `c_datetime`, `c_datev2`, + `c_char`, `c_varchar`, `c_string` FROM tbl_scalar_types_dup""" + // define dup key table with index testTable = "tbl_scalar_types_dup_inverted_index" diff --git a/regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q01_bool.sql b/regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q01_bool.sql new file mode 100644 index 00000000000000..3e38418168833e --- /dev/null +++ b/regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q01_bool.sql @@ -0,0 +1 @@ +SELECT count() FROM tbl_scalar_types_unique2_bitmapindex WHERE c_bool = 1; diff --git a/regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q02_tinyint.sql b/regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q02_tinyint.sql new file mode 100644 index 00000000000000..ed75ebca45779e --- /dev/null +++ b/regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q02_tinyint.sql @@ -0,0 +1 @@ +SELECT count() FROM tbl_scalar_types_unique2_bitmapindex WHERE c_tinyint = 14; diff --git a/regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q03_smallint.sql b/regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q03_smallint.sql new file mode 100644 index 00000000000000..faa2159f0dda2c --- /dev/null +++ b/regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q03_smallint.sql @@ -0,0 +1 @@ +SELECT * FROM tbl_scalar_types_unique2_bitmapindex WHERE c_smallint = 12321 ORDER BY c_bigint, c_largeint; diff --git a/regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q04_int.sql b/regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q04_int.sql new file mode 100644 index 00000000000000..db82b79873aa29 --- /dev/null +++ b/regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q04_int.sql @@ -0,0 +1 @@ +SELECT * FROM tbl_scalar_types_unique2_bitmapindex WHERE c_int = -16441 ORDER BY c_bigint, c_largeint; diff --git a/regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q05_bigint.sql b/regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q05_bigint.sql new file mode 100644 index 00000000000000..9ef4811f62aefd --- /dev/null +++ b/regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q05_bigint.sql @@ -0,0 +1 @@ +SELECT * FROM tbl_scalar_types_unique2_bitmapindex WHERE c_bigint = 859063837 ORDER BY c_bigint, c_largeint; diff --git a/regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q06_largeint.sql b/regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q06_largeint.sql new file mode 100644 index 00000000000000..2a6b48064c24a7 --- /dev/null +++ b/regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q06_largeint.sql @@ -0,0 +1 @@ +SELECT * FROM tbl_scalar_types_unique2_bitmapindex WHERE c_largeint = 1524137702 ORDER BY c_bigint, c_largeint; diff --git a/regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q07_float.sql b/regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q07_float.sql new file mode 100644 index 00000000000000..1ae3d82f151acb --- /dev/null +++ b/regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q07_float.sql @@ -0,0 +1 @@ +SELECT * FROM tbl_scalar_types_unique2_bitmapindex WHERE c_float = -13023.051 ORDER BY c_bigint, c_largeint; diff --git a/regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q08_double.sql b/regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q08_double.sql new file mode 100644 index 00000000000000..8c21bc08b79394 --- /dev/null +++ b/regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q08_double.sql @@ -0,0 +1 @@ +SELECT * FROM tbl_scalar_types_unique2_bitmapindex WHERE c_double = -679666086.055166 ORDER BY c_bigint, c_largeint; diff --git a/regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q09_decimal.sql b/regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q09_decimal.sql new file mode 100644 index 00000000000000..9fd1e832bed7f3 --- /dev/null +++ b/regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q09_decimal.sql @@ -0,0 +1 @@ +SELECT * FROM tbl_scalar_types_unique2_bitmapindex WHERE c_decimal = 5355625357307892.981 ORDER BY c_bigint, c_largeint; diff --git a/regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q10_decimalv3.sql b/regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q10_decimalv3.sql new file mode 100644 index 00000000000000..ee676941451607 --- /dev/null +++ b/regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q10_decimalv3.sql @@ -0,0 +1 @@ +SELECT * FROM tbl_scalar_types_unique2_bitmapindex WHERE c_decimalv3 = 89599956401481176.347 ORDER BY c_bigint, c_largeint; diff --git a/regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q11_date.sql b/regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q11_date.sql new file mode 100644 index 00000000000000..7f5d383765fa38 --- /dev/null +++ b/regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q11_date.sql @@ -0,0 +1 @@ +SELECT * FROM tbl_scalar_types_unique2_bitmapindex WHERE c_date = '2022-10-09' ORDER BY c_bigint, c_largeint; diff --git a/regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q12_datetime.sql b/regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q12_datetime.sql new file mode 100644 index 00000000000000..a1d7728c621b15 --- /dev/null +++ b/regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q12_datetime.sql @@ -0,0 +1 @@ +SELECT * FROM tbl_scalar_types_unique2_bitmapindex WHERE c_datetime = '2022-07-31 12:07:00' ORDER BY c_bigint, c_largeint; diff --git a/regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q13_datev2.sql b/regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q13_datev2.sql new file mode 100644 index 00000000000000..eee7b1110841b9 --- /dev/null +++ b/regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q13_datev2.sql @@ -0,0 +1 @@ +SELECT * FROM tbl_scalar_types_unique2_bitmapindex WHERE c_datev2 = '2022-03-06' ORDER BY c_bigint, c_largeint; diff --git a/regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q14_datetimev2.sql b/regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q14_datetimev2.sql new file mode 100644 index 00000000000000..83b86dadbf2b1c --- /dev/null +++ b/regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q14_datetimev2.sql @@ -0,0 +1 @@ +SELECT * FROM tbl_scalar_types_unique2_bitmapindex WHERE c_datetimev2 = '2022-11-18 06:05:56' ORDER BY c_bigint, c_largeint; diff --git a/regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q15_char.sql b/regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q15_char.sql new file mode 100644 index 00000000000000..67486dde69fd33 --- /dev/null +++ b/regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q15_char.sql @@ -0,0 +1 @@ +SELECT * FROM tbl_scalar_types_unique2_bitmapindex WHERE c_char = '202.171.74.217' ORDER BY c_bigint, c_largeint; diff --git a/regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q16_varchar.sql b/regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q16_varchar.sql new file mode 100644 index 00000000000000..677bc5614a819a --- /dev/null +++ b/regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q16_varchar.sql @@ -0,0 +1 @@ +SELECT * FROM tbl_scalar_types_unique2_bitmapindex WHERE c_varchar = 'modi_a@Feedmix.biz' ORDER BY c_bigint, c_largeint; diff --git a/regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q17_string.sql b/regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q17_string.sql new file mode 100644 index 00000000000000..b02fb1d72e87dd --- /dev/null +++ b/regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q17_string.sql @@ -0,0 +1 @@ +SELECT * FROM tbl_scalar_types_unique2_bitmapindex WHERE c_string = 'Reinke Junction 63' ORDER BY c_bigint, c_largeint; diff --git a/regression-test/suites/fault_injection_p0/test_build_index_fault.groovy b/regression-test/suites/fault_injection_p0/test_build_index_fault.groovy index 99f1173acfaaa2..f5b95137c32dc5 100644 --- a/regression-test/suites/fault_injection_p0/test_build_index_fault.groovy +++ b/regression-test/suites/fault_injection_p0/test_build_index_fault.groovy @@ -73,6 +73,7 @@ suite("test_build_index_fault", "inverted_index, nonConcurrent"){ useTime = t sleep(delta_time) } + logger.info("wait_for_last_build_index_on_table_finish debug: " + alter_res) assertTrue(useTime <= OpTimeout, "wait_for_last_build_index_on_table_finish timeout") return "wait_timeout" } @@ -91,7 +92,8 @@ suite("test_build_index_fault", "inverted_index, nonConcurrent"){ useTime = t sleep(delta_time) } - assertTrue(useTime <= OpTimeout, "wait_for_last_build_index_on_table_finish timeout") + logger.info("wait_for_last_build_index_on_table_running debug: " + alter_res) + assertTrue(useTime <= OpTimeout, "wait_for_last_build_index_on_table_running timeout") return "wait_timeout" } diff --git a/regression-test/suites/index_p0/test_bitmap_index.groovy b/regression-test/suites/index_p0/test_bitmap_index.groovy index b6bd7082a69691..d3b0903842bb03 100644 --- a/regression-test/suites/index_p0/test_bitmap_index.groovy +++ b/regression-test/suites/index_p0/test_bitmap_index.groovy @@ -178,7 +178,7 @@ suite("test_bitmap_index") { } test{ sql "ALTER TABLE ${tbName2} ADD INDEX index16 (v1) USING BITMAP;" - exception "errCode = 2, detailMessage = INVERTED index only used in columns of DUP_KEYS/UNIQUE_KEYS MOW table or key columns of all table. invalid index: index16" + exception "errCode = 2, detailMessage = index should only be used in columns of DUP_KEYS/UNIQUE_KEYS table or key columns of AGG_KEYS table. invalid index: index16" } sql "insert into ${tbName2} values(1,1,1,1,'1','1','2022-05-31','2022-05-31 10:00:00',1,1.0,1,'2022-05-31','2022-05-31 10:00:00.111111','2022-05-31 10:00:00.111111','2022-05-31 10:00:00.111111',1);" @@ -340,7 +340,7 @@ suite("test_bitmap_index") { "replication_allocation" = "tag.location.default: 1", "is_being_synced" = "false", "storage_format" = "V2", - "enable_unique_key_merge_on_write" = "true", + "enable_unique_key_merge_on_write" = "false", "light_schema_change" = "true", "disable_auto_compaction" = "false", "enable_single_replica_compaction" = "false" diff --git a/regression-test/suites/inverted_index_p0/index_compaction/test_index_compaction_null.groovy b/regression-test/suites/inverted_index_p0/index_compaction/test_index_compaction_null.groovy index 6765480a4f8466..e80a97eb1af8a8 100644 --- a/regression-test/suites/inverted_index_p0/index_compaction/test_index_compaction_null.groovy +++ b/regression-test/suites/inverted_index_p0/index_compaction/test_index_compaction_null.groovy @@ -288,6 +288,7 @@ suite("test_index_compaction_null", "p0") { PROPERTIES ( "replication_allocation" = "tag.location.default: 1", "disable_auto_compaction" = "true", + "enable_unique_key_merge_on_write" = "true", "in_memory" = "false", "storage_format" = "V2" ) diff --git a/regression-test/suites/inverted_index_p0/test_bitmap_index.groovy b/regression-test/suites/inverted_index_p0/test_bitmap_index.groovy index 29289a10e096b2..b3b7cce15f8be5 100644 --- a/regression-test/suites/inverted_index_p0/test_bitmap_index.groovy +++ b/regression-test/suites/inverted_index_p0/test_bitmap_index.groovy @@ -158,7 +158,7 @@ suite("test_bitmap_index", "inverted_index") { } test{ sql "ALTER TABLE ${tbName2} ADD INDEX index12 (v1) USING BITMAP;" - exception "errCode = 2, detailMessage = INVERTED index only used in columns of DUP_KEYS/UNIQUE_KEYS MOW table or key columns of all table. invalid index: index12" + exception "errCode = 2, detailMessage = index should only be used in columns of DUP_KEYS/UNIQUE_KEYS table or key columns of AGG_KEYS table. invalid index: index12" } sql "insert into ${tbName2} values(1,1,1,1,'1','1','2022-05-31','2022-05-31 10:00:00',1,1.0,1,1);" diff --git a/regression-test/suites/inverted_index_p0/test_build_index.groovy b/regression-test/suites/inverted_index_p0/test_build_index.groovy index a2eb40aa794538..30d79943d62ecd 100644 --- a/regression-test/suites/inverted_index_p0/test_build_index.groovy +++ b/regression-test/suites/inverted_index_p0/test_build_index.groovy @@ -73,6 +73,7 @@ suite("test_build_index", "inverted_index"){ useTime = t sleep(delta_time) } + logger.info("wait_for_last_build_index_on_table_finish debug: " + alter_res) assertTrue(useTime <= OpTimeout, "wait_for_last_build_index_on_table_finish timeout") return "wait_timeout" } @@ -91,7 +92,8 @@ suite("test_build_index", "inverted_index"){ useTime = t sleep(delta_time) } - assertTrue(useTime <= OpTimeout, "wait_for_last_build_index_on_table_finish timeout") + logger.info("wait_for_last_build_index_on_table_running debug: " + alter_res) + assertTrue(useTime <= OpTimeout, "wait_for_last_build_index_on_table_running timeout") return "wait_timeout" } diff --git a/regression-test/suites/inverted_index_p0/test_inverted_index.groovy b/regression-test/suites/inverted_index_p0/test_inverted_index.groovy index 3cb9cc76fedca8..280782f75c57f2 100644 --- a/regression-test/suites/inverted_index_p0/test_inverted_index.groovy +++ b/regression-test/suites/inverted_index_p0/test_inverted_index.groovy @@ -178,7 +178,7 @@ suite("test_inverted_index", "inverted_index") { } test{ sql "ALTER TABLE ${tbName2} ADD INDEX index16 (v1) USING INVERTED;" - exception "errCode = 2, detailMessage = INVERTED index only used in columns of DUP_KEYS/UNIQUE_KEYS MOW table or key columns of all table. invalid index: index16" + exception "errCode = 2, detailMessage = index should only be used in columns of DUP_KEYS/UNIQUE_KEYS table or key columns of AGG_KEYS table. invalid index: index16" } sql "insert into ${tbName2} values(1,1,1,1,'1','1','2022-05-31','2022-05-31 10:00:00',1,1.0,1,'2022-05-31','2022-05-31 10:00:00.111111','2022-05-31 10:00:00.111111','2022-05-31 10:00:00.111111',1);" diff --git a/regression-test/suites/inverted_index_p0/test_inverted_index_mor.groovy b/regression-test/suites/inverted_index_p0/test_inverted_index_mor.groovy new file mode 100644 index 00000000000000..1905067d433e5f --- /dev/null +++ b/regression-test/suites/inverted_index_p0/test_inverted_index_mor.groovy @@ -0,0 +1,78 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + +suite("test_inverted_index_mor", "p0"){ + def timeout = 60000 + def delta_time = 1000 + def alter_res = "null" + def useTime = 0 + + def indexTblName = "test_inverted_index_mor" + + sql "DROP TABLE IF EXISTS ${indexTblName}" + + sql """ + CREATE TABLE IF NOT EXISTS ${indexTblName}( + `k1` INT, + `k2` INT, + `c_int` INT, + `c_float` FLOAT, + `c_string` STRING, + INDEX idx_k2(`k2`) USING INVERTED COMMENT '', + INDEX idx_c_int(`c_int`) USING INVERTED COMMENT '' + ) ENGINE=OLAP + UNIQUE KEY(`k1`, `k2`) + COMMENT 'OLAP' + DISTRIBUTED BY HASH(`k1`, `k2`) BUCKETS 1 + PROPERTIES( + "replication_allocation" = "tag.location.default: 1", + "enable_unique_key_merge_on_write" = "false", + "disable_auto_compaction" = "true" + ); + """ + + sql """ INSERT INTO $indexTblName VALUES (1, 2, 12, 1.2, '1 2'), (3, 4, 34, 3.4, '3 4'); """ + sql """ INSERT INTO $indexTblName VALUES (11, 12, 1112, 11.12, '11 22'), (13, 14, 1314, 13.14, '13 14'); """ + + // original data + qt_11 """ SELECT * FROM $indexTblName ORDER BY k1,k2 """ + + // + qt_12 """ SELECT * FROM $indexTblName WHERE k2 > 2 OR c_int = 12 ORDER BY k1,k2 """ + + sql """ INSERT INTO $indexTblName VALUES (1, 2, 112, 11.2, '112'); """ + qt_21 """ SELECT * FROM $indexTblName ORDER BY k1,k2 """ + qt_22 """ SELECT * FROM $indexTblName WHERE k2 > 2 OR c_int = 12 ORDER BY k1,k2 """ + qt_23 """ SELECT * FROM $indexTblName WHERE k2 > 2 OR c_int = 112 ORDER BY k1,k2 """ + + // can not add INVERTED INDEX with parser + test{ + sql """ ALTER TABLE ${indexTblName} ADD INDEX idx_c_string(`c_string`) USING INVERTED PROPERTIES("parser"="english"); """ + exception "errCode = 2, detailMessage = INVERTED index with parser can NOT be used in value columns of UNIQUE_KEYS table with merge_on_write disable. invalid index: idx_c_string" + } + + // can add INVERTED INDEX without parser + def success = false; + try { + sql """ ALTER TABLE ${indexTblName} ADD INDEX idx_c_string(`c_string`) USING INVERTED; """ + success = true + } catch(Exception ex) { + logger.info("ALTER TABLE ${indexTblName} ADD INDEX idx_c_string without parser exception: " + ex) + } + assertTrue(success) +} \ No newline at end of file diff --git a/regression-test/suites/inverted_index_p0/test_show_data.groovy b/regression-test/suites/inverted_index_p0/test_show_data.groovy index e59624be80124b..92e16a14fd3b4a 100644 --- a/regression-test/suites/inverted_index_p0/test_show_data.groovy +++ b/regression-test/suites/inverted_index_p0/test_show_data.groovy @@ -145,6 +145,7 @@ suite("test_show_data", "p0") { useTime = t Thread.sleep(delta_time) } + logger.info("wait_for_last_build_index_on_table_finish debug: " + alter_res) assertTrue(useTime <= OpTimeout, "wait_for_last_build_index_on_table_finish timeout, useTime=${useTime}") return "wait_timeout" } @@ -314,6 +315,7 @@ suite("test_show_data_for_bkd", "p0") { useTime = t Thread.sleep(delta_time) } + logger.info("wait_for_last_build_index_on_table_finish debug: " + alter_res) assertTrue(useTime <= OpTimeout, "wait_for_last_build_index_on_table_finish timeout, useTime=${useTime}") return "wait_timeout" } diff --git a/regression-test/suites/segcompaction_p2/test_segcompaction_unique_keys_index.groovy b/regression-test/suites/segcompaction_p2/test_segcompaction_unique_keys_index.groovy index 4e716f48951b82..eb39c48e0b79a5 100644 --- a/regression-test/suites/segcompaction_p2/test_segcompaction_unique_keys_index.groovy +++ b/regression-test/suites/segcompaction_p2/test_segcompaction_unique_keys_index.groovy @@ -48,28 +48,39 @@ suite("test_segcompaction_unique_keys_index") { } sql """ DROP TABLE IF EXISTS ${tableName} """ - test { - sql """ - CREATE TABLE IF NOT EXISTS ${tableName} ( - `col_0` BIGINT NOT NULL,`col_1` VARCHAR(20),`col_2` VARCHAR(20),`col_3` VARCHAR(20),`col_4` VARCHAR(20), - `col_5` VARCHAR(20),`col_6` VARCHAR(20),`col_7` VARCHAR(20),`col_8` VARCHAR(20),`col_9` VARCHAR(20), - `col_10` VARCHAR(20),`col_11` VARCHAR(20),`col_12` VARCHAR(20),`col_13` VARCHAR(20),`col_14` VARCHAR(20), - `col_15` VARCHAR(20),`col_16` VARCHAR(20),`col_17` VARCHAR(20),`col_18` VARCHAR(20),`col_19` VARCHAR(20), - `col_20` VARCHAR(20),`col_21` VARCHAR(20),`col_22` VARCHAR(20),`col_23` VARCHAR(20),`col_24` VARCHAR(20), - `col_25` VARCHAR(20),`col_26` VARCHAR(20),`col_27` VARCHAR(20),`col_28` VARCHAR(20),`col_29` VARCHAR(20), - `col_30` VARCHAR(20),`col_31` VARCHAR(20),`col_32` VARCHAR(20),`col_33` VARCHAR(20),`col_34` VARCHAR(20), - `col_35` VARCHAR(20),`col_36` VARCHAR(20),`col_37` VARCHAR(20),`col_38` VARCHAR(20),`col_39` VARCHAR(20), - `col_40` VARCHAR(20),`col_41` VARCHAR(20),`col_42` VARCHAR(20),`col_43` VARCHAR(20),`col_44` VARCHAR(20), - `col_45` VARCHAR(20),`col_46` VARCHAR(20),`col_47` VARCHAR(20),`col_48` VARCHAR(20),`col_49` VARCHAR(20), - INDEX index_col_0 (`col_0`) USING INVERTED COMMENT '', - INDEX index_col_1 (`col_1`) USING INVERTED PROPERTIES("parser" = "english") COMMENT '', - INDEX index_col_2 (`col_2`) USING INVERTED COMMENT '' - ) - UNIQUE KEY(`col_0`) DISTRIBUTED BY HASH(`col_0`) BUCKETS 1 - PROPERTIES ( "replication_num" = "1" ); - """ - exception "INVERTED index only used in columns of DUP_KEYS/UNIQUE_KEYS MOW table or key columns of all table. invalid indexName: index_col_1" - } + sql """ + CREATE TABLE IF NOT EXISTS ${tableName} ( + `col_0` BIGINT NOT NULL,`col_1` VARCHAR(20),`col_2` VARCHAR(20),`col_3` VARCHAR(20),`col_4` VARCHAR(20), + `col_5` VARCHAR(20),`col_6` VARCHAR(20),`col_7` VARCHAR(20),`col_8` VARCHAR(20),`col_9` VARCHAR(20), + `col_10` VARCHAR(20),`col_11` VARCHAR(20),`col_12` VARCHAR(20),`col_13` VARCHAR(20),`col_14` VARCHAR(20), + `col_15` VARCHAR(20),`col_16` VARCHAR(20),`col_17` VARCHAR(20),`col_18` VARCHAR(20),`col_19` VARCHAR(20), + `col_20` VARCHAR(20),`col_21` VARCHAR(20),`col_22` VARCHAR(20),`col_23` VARCHAR(20),`col_24` VARCHAR(20), + `col_25` VARCHAR(20),`col_26` VARCHAR(20),`col_27` VARCHAR(20),`col_28` VARCHAR(20),`col_29` VARCHAR(20), + `col_30` VARCHAR(20),`col_31` VARCHAR(20),`col_32` VARCHAR(20),`col_33` VARCHAR(20),`col_34` VARCHAR(20), + `col_35` VARCHAR(20),`col_36` VARCHAR(20),`col_37` VARCHAR(20),`col_38` VARCHAR(20),`col_39` VARCHAR(20), + `col_40` VARCHAR(20),`col_41` VARCHAR(20),`col_42` VARCHAR(20),`col_43` VARCHAR(20),`col_44` VARCHAR(20), + `col_45` VARCHAR(20),`col_46` VARCHAR(20),`col_47` VARCHAR(20),`col_48` VARCHAR(20),`col_49` VARCHAR(20), + INDEX index_col_0 (`col_0`) USING INVERTED COMMENT '', + INDEX index_col_2 (`col_2`) USING INVERTED COMMENT '' + ) + UNIQUE KEY(`col_0`) DISTRIBUTED BY HASH(`col_0`) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1", + "enable_unique_key_merge_on_write" = "false" + ); + """ + + def uuid = UUID.randomUUID().toString().replace("-", "0") + def path = "oss://$bucket/regression/segcompaction_test/segcompaction_test.orc" + + def columns = "col_0, col_1, col_2, col_3, col_4, col_5, col_6, col_7, col_8, col_9, col_10, col_11, col_12, col_13, col_14, col_15, col_16, col_17, col_18, col_19, col_20, col_21, col_22, col_23, col_24, col_25, col_26, col_27, col_28, col_29, col_30, col_31, col_32, col_33, col_34, col_35, col_36, col_37, col_38, col_39, col_40, col_41, col_42, col_43, col_44, col_45, col_46, col_47, col_48, col_49" + String columns_str = ("$columns" != "") ? "($columns)" : ""; + + qt_select_default """ SELECT * FROM ${tableName} WHERE col_0=47 order by col_1; """ + qt_select_default """ SELECT COUNT(*) FROM ${tableName} WHERE col_1 MATCH_ANY 'lemon'; """ + qt_select_default """ SELECT COUNT(*) FROM ${tableName} WHERE col_2 MATCH_ANY 'Lemon'; """ + + String[][] tablets = sql """ show tablets from ${tableName}; """ } finally { try_sql("DROP TABLE IF EXISTS ${tableName}") }