Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion be/src/olap/compaction.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -600,7 +600,8 @@ Status Compaction::construct_output_rowset_writer(RowsetWriterContext& ctx, bool
ctx.newest_write_timestamp = _newest_write_timestamp;
ctx.write_type = DataWriteType::TYPE_COMPACTION;
if (config::inverted_index_compaction_enable &&
((_tablet->keys_type() == KeysType::UNIQUE_KEYS ||
(((_tablet->keys_type() == KeysType::UNIQUE_KEYS &&
_tablet->enable_unique_key_merge_on_write()) ||
_tablet->keys_type() == KeysType::DUP_KEYS))) {
for (const auto& index : _cur_tablet_schema->indexes()) {
if (index.index_type() == IndexType::INVERTED) {
Expand Down
22 changes: 18 additions & 4 deletions be/src/olap/rowset/beta_rowset_reader.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -163,10 +163,24 @@ Status BetaRowsetReader::get_segment_iterators(RowsetReaderContext* read_context
}

if (_read_context->predicates_except_leafnode_of_andnode != nullptr) {
_read_options.column_predicates_except_leafnode_of_andnode.insert(
_read_options.column_predicates_except_leafnode_of_andnode.end(),
_read_context->predicates_except_leafnode_of_andnode->begin(),
_read_context->predicates_except_leafnode_of_andnode->end());
bool should_push_down = true;
bool should_push_down_value_predicates = _should_push_down_value_predicates();
for (auto pred : *_read_context->predicates_except_leafnode_of_andnode) {
if (_rowset->keys_type() == UNIQUE_KEYS && !should_push_down_value_predicates &&
!_read_context->tablet_schema->column(pred->column_id()).is_key()) {
VLOG_DEBUG << "do not push down except_leafnode_of_andnode value pred "
<< pred->debug_string();
should_push_down = false;
break;
}
}

if (should_push_down) {
_read_options.column_predicates_except_leafnode_of_andnode.insert(
_read_options.column_predicates_except_leafnode_of_andnode.end(),
_read_context->predicates_except_leafnode_of_andnode->begin(),
_read_context->predicates_except_leafnode_of_andnode->end());
}
}

// Take a delete-bitmap for each segment, the bitmap contains all deletes
Expand Down
13 changes: 10 additions & 3 deletions be/src/olap/rowset/segment_v2/segment_iterator.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -497,12 +497,16 @@ Status SegmentIterator::_get_row_ranges_by_column_conditions() {
it != _remaining_conjunct_roots.end();) {
_pred_except_leafnode_of_andnode_evaluate_result.clear();
auto res = _execute_predicates_except_leafnode_of_andnode(*it);
VLOG_DEBUG << "_execute_predicates_except_leafnode_of_andnode expr: "
<< (*it)->debug_string() << " res: " << res;
if (res.ok() && _pred_except_leafnode_of_andnode_evaluate_result.size() == 1) {
_row_bitmap &= _pred_except_leafnode_of_andnode_evaluate_result[0];
// Delete expr after it obtains the final result.
{
std::erase_if(_common_expr_ctxs_push_down,
[&it](const auto& iter) { return iter->root() == *it; });
VLOG_DEBUG << "_remaining_conjunct_roots erase expr: "
<< (*it)->debug_string();
it = _remaining_conjunct_roots.erase(it);
}
} else {
Expand Down Expand Up @@ -771,21 +775,24 @@ Status SegmentIterator::_execute_compound_fn(const std::string& function_name) {
auto size = _pred_except_leafnode_of_andnode_evaluate_result.size();
if (function_name == "and") {
if (size < 2) {
return Status::Uninitialized("execute and logic compute error.");
return Status::InvalidArgument("_execute_compound_fn {} arg num {} < 2", function_name,
size);
}
_pred_except_leafnode_of_andnode_evaluate_result.at(size - 2) &=
_pred_except_leafnode_of_andnode_evaluate_result.at(size - 1);
_pred_except_leafnode_of_andnode_evaluate_result.pop_back();
} else if (function_name == "or") {
if (size < 2) {
return Status::Uninitialized("execute or logic compute error.");
return Status::InvalidArgument("_execute_compound_fn {} arg num {} < 2", function_name,
size);
}
_pred_except_leafnode_of_andnode_evaluate_result.at(size - 2) |=
_pred_except_leafnode_of_andnode_evaluate_result.at(size - 1);
_pred_except_leafnode_of_andnode_evaluate_result.pop_back();
} else if (function_name == "not") {
if (size < 1) {
return Status::Uninitialized("execute not logic compute error.");
return Status::InvalidArgument("_execute_compound_fn {} arg num {} < 1", function_name,
size);
}
roaring::Roaring tmp = _row_bitmap;
tmp -= _pred_except_leafnode_of_andnode_evaluate_result.at(size - 1);
Expand Down
2 changes: 1 addition & 1 deletion fe/fe-core/src/main/cup/sql_parser.cup
Original file line number Diff line number Diff line change
Expand Up @@ -3812,7 +3812,7 @@ opt_index_type ::=
:}
| KW_USING KW_BITMAP
{:
RESULT = IndexDef.IndexType.INVERTED;
RESULT = IndexDef.IndexType.BITMAP;
:}
| KW_USING KW_NGRAM_BF
{:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -224,12 +224,16 @@ public void checkColumn(Column column, KeysType keysType, boolean enableUniqueKe
throw new AnalysisException(colType + " is not supported in " + indexType.toString() + " index. "
+ "invalid index: " + indexName);
}
if (!column.isKey()
&& ((keysType == KeysType.UNIQUE_KEYS && !enableUniqueKeyMergeOnWrite)
|| keysType == KeysType.AGG_KEYS)) {
throw new AnalysisException(indexType.toString()
+ " index only used in columns of DUP_KEYS/UNIQUE_KEYS MOW table or key columns of all table."
+ " invalid index: " + indexName);
if (!column.isKey()) {
if (keysType == KeysType.AGG_KEYS) {
throw new AnalysisException("index should only be used in columns of DUP_KEYS/UNIQUE_KEYS table"
+ " or key columns of AGG_KEYS table. invalid index: " + indexName);
} else if (keysType == KeysType.UNIQUE_KEYS && !enableUniqueKeyMergeOnWrite
&& indexType == IndexType.INVERTED && properties != null
&& properties.containsKey(InvertedIndexUtil.INVERTED_INDEX_PARSER_KEY)) {
throw new AnalysisException("INVERTED index with parser can NOT be used in value columns of"
+ " UNIQUE_KEYS table with merge_on_write disable. invalid index: " + indexName);
}
}

if (indexType == IndexType.INVERTED) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -525,8 +525,8 @@ public static Set<String> analyzeBloomFilterColumns(Map<String, String> properti
found = true;
break;
} else {
throw new AnalysisException("Bloom filter index only used in columns of"
+ " UNIQUE_KEYS/DUP_KEYS table or key columns of AGG_KEYS table."
throw new AnalysisException("Bloom filter index should only be used in columns"
+ " of UNIQUE_KEYS/DUP_KEYS table or key columns of AGG_KEYS table."
+ " invalid column: " + bfColumn);
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2537,10 +2537,6 @@ public IndexDefinition visitIndexDef(IndexDefContext ctx) {
Map<String, String> properties = visitPropertyItemList(ctx.properties);
String indexType = ctx.indexType != null ? ctx.indexType.getText().toUpperCase() : null;
String comment = ctx.comment != null ? ctx.comment.getText() : "";
// change BITMAP index to INVERTED index
if (indexType.equalsIgnoreCase("BITMAP")) {
indexType = "INVERTED";
}
return new IndexDefinition(indexName, indexCols, indexType, properties, comment);
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -110,12 +110,16 @@ public void checkColumn(ColumnDefinition column, KeysType keysType,
throw new AnalysisException(colType + " is not supported in " + indexType.toString()
+ " index. " + "invalid index: " + name);
}
if (!column.isKey()
&& ((keysType == KeysType.UNIQUE_KEYS && !enableUniqueKeyMergeOnWrite)
|| keysType == KeysType.AGG_KEYS)) {
throw new AnalysisException(indexType.toString()
+ " index only used in columns of DUP_KEYS/UNIQUE_KEYS MOW table or key columns of all table."
+ " invalid index: " + name);
if (!column.isKey()) {
if (keysType == KeysType.AGG_KEYS) {
throw new AnalysisException("index should only be used in columns of DUP_KEYS/UNIQUE_KEYS table"
+ " or key columns of AGG_KEYS table. invalid index: " + name);
} else if (keysType == KeysType.UNIQUE_KEYS && !enableUniqueKeyMergeOnWrite
&& indexType == IndexType.INVERTED && properties != null
&& properties.containsKey(InvertedIndexUtil.INVERTED_INDEX_PARSER_KEY)) {
throw new AnalysisException("INVERTED index with parser can NOT be used in value columns of"
+ " UNIQUE_KEYS table with merge_on_write disable. invalid index: " + name);
}
}

if (indexType == IndexType.INVERTED) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -122,7 +122,7 @@ public void testBfColumnsError() {
try {
PropertyAnalyzer.analyzeBloomFilterColumns(properties, columns, KeysType.AGG_KEYS);
} catch (AnalysisException e) {
Assert.assertTrue(e.getMessage().contains("Bloom filter index only used in"));
Assert.assertTrue(e.getMessage().contains("Bloom filter index should only be used"));
}

// reduplicated column
Expand Down
6 changes: 3 additions & 3 deletions regression-test/data/index_p0/test_bitmap_index.out
Original file line number Diff line number Diff line change
Expand Up @@ -65,9 +65,9 @@ v1 INT Yes false \N NONE
create_time DATETIME No true \N
vid VARCHAR(64) No true \N
report_time DATETIME Yes true \N
block_version INT Yes false \N NONE
vehicle_mode INT Yes false \N NONE
usage_mode INT Yes false \N NONE
block_version INT Yes false \N REPLACE
vehicle_mode INT Yes false \N REPLACE
usage_mode INT Yes false \N REPLACE

-- !sql --
2
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
-- This file is automatically generated. You should know what you did if you want to edit this
-- !11 --
1 2 12 1.2 1 2
3 4 34 3.4 3 4
11 12 1112 11.12 11 22
13 14 1314 13.14 13 14

-- !12 --
1 2 12 1.2 1 2
3 4 34 3.4 3 4
11 12 1112 11.12 11 22
13 14 1314 13.14 13 14

-- !21 --
1 2 112 11.2 112
3 4 34 3.4 3 4
11 12 1112 11.12 11 22
13 14 1314 13.14 13 14

-- !22 --
3 4 34 3.4 3 4
11 12 1112 11.12 11 22
13 14 1314 13.14 13 14

-- !23 --
1 2 112 11.2 112
3 4 34 3.4 3 4
11 12 1112 11.12 11 22
13 14 1314 13.14 13 14

Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
-- This file is automatically generated. You should know what you did if you want to edit this
-- !select_default --

-- !select_default --
0

-- !select_default --
0

54 changes: 54 additions & 0 deletions regression-test/suites/datatype_p0/scalar_types/load.groovy
Original file line number Diff line number Diff line change
Expand Up @@ -302,6 +302,60 @@ suite("test_scalar_types_load", "p0") {
`c_float`, `c_double`, `c_decimal`, `c_date`, `c_datetime`, `c_datev2`,
`c_char`, `c_varchar`, `c_string` FROM tbl_scalar_types_dup"""

// define unique key table2 enable mow
testTable = "tbl_scalar_types_unique2_bitmapindex"
sql "DROP TABLE IF EXISTS ${testTable}"
sql """
CREATE TABLE IF NOT EXISTS ${testTable} (
`c_datetimev2` datetimev2(0) NULL,
`c_bigint` bigint(20) NULL,
`c_decimalv3` decimalv3(20, 3) NULL,
`c_bool` boolean NULL,
`c_tinyint` tinyint(4) NULL,
`c_smallint` smallint(6) NULL,
`c_int` int(11) NULL,
`c_largeint` largeint(40) NULL,
`c_float` float NULL,
`c_double` double NULL,
`c_decimal` decimal(20, 3) NULL,
`c_date` date NULL,
`c_datetime` datetime NULL,
`c_datev2` datev2 NULL,
`c_char` char(15) NULL,
`c_varchar` varchar(100) NULL,
`c_string` text NULL,
INDEX idx_c_bool (c_bool) USING BITMAP,
INDEX idx_c_tinyint (c_tinyint) USING BITMAP,
INDEX idx_c_smallint (c_smallint) USING BITMAP,
INDEX idx_c_int (c_int) USING BITMAP,
INDEX idx_c_bigint (c_bigint) USING BITMAP,
INDEX idx_c_largeint (c_largeint) USING BITMAP,
INDEX idx_c_decimal (c_decimal) USING BITMAP,
INDEX idx_c_decimalv3 (c_decimalv3) USING BITMAP,
INDEX idx_c_date (c_date) USING BITMAP,
INDEX idx_c_datetime (c_datetime) USING BITMAP,
INDEX idx_c_datev2 (c_datev2) USING BITMAP,
INDEX idx_c_datetimev2 (c_datetimev2) USING BITMAP,
INDEX idx_c_char (c_char) USING BITMAP,
INDEX idx_c_varchar (c_varchar) USING BITMAP,
INDEX idx_c_string (c_string) USING BITMAP
) ENGINE=OLAP
UNIQUE KEY(`c_datetimev2`, `c_bigint`, `c_decimalv3`)
COMMENT 'OLAP'
DISTRIBUTED BY HASH(`c_bigint`) BUCKETS 10
PROPERTIES("replication_num" = "1", "enable_unique_key_merge_on_write" = "false");
"""

// insert data into unique key table2 2 times
sql """INSERT INTO ${testTable} SELECT `c_datetimev2`, `c_bigint`, `c_decimalv3`,
`c_bool`, `c_tinyint`, `c_smallint`, `c_int`, `c_largeint`,
`c_float`, `c_double`, `c_decimal`, `c_date`, `c_datetime`, `c_datev2`,
`c_char`, `c_varchar`, `c_string` FROM tbl_scalar_types_dup"""
sql """INSERT INTO ${testTable} SELECT `c_datetimev2`, `c_bigint`, `c_decimalv3`,
`c_bool`, `c_tinyint`, `c_smallint`, `c_int`, `c_largeint`,
`c_float`, `c_double`, `c_decimal`, `c_date`, `c_datetime`, `c_datev2`,
`c_char`, `c_varchar`, `c_string` FROM tbl_scalar_types_dup"""


// define dup key table with index
testTable = "tbl_scalar_types_dup_inverted_index"
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
SELECT count() FROM tbl_scalar_types_unique2_bitmapindex WHERE c_bool = 1;
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
SELECT count() FROM tbl_scalar_types_unique2_bitmapindex WHERE c_tinyint = 14;
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
SELECT * FROM tbl_scalar_types_unique2_bitmapindex WHERE c_smallint = 12321 ORDER BY c_bigint, c_largeint;
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
SELECT * FROM tbl_scalar_types_unique2_bitmapindex WHERE c_int = -16441 ORDER BY c_bigint, c_largeint;
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
SELECT * FROM tbl_scalar_types_unique2_bitmapindex WHERE c_bigint = 859063837 ORDER BY c_bigint, c_largeint;
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
SELECT * FROM tbl_scalar_types_unique2_bitmapindex WHERE c_largeint = 1524137702 ORDER BY c_bigint, c_largeint;
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
SELECT * FROM tbl_scalar_types_unique2_bitmapindex WHERE c_float = -13023.051 ORDER BY c_bigint, c_largeint;
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
SELECT * FROM tbl_scalar_types_unique2_bitmapindex WHERE c_double = -679666086.055166 ORDER BY c_bigint, c_largeint;
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
SELECT * FROM tbl_scalar_types_unique2_bitmapindex WHERE c_decimal = 5355625357307892.981 ORDER BY c_bigint, c_largeint;
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
SELECT * FROM tbl_scalar_types_unique2_bitmapindex WHERE c_decimalv3 = 89599956401481176.347 ORDER BY c_bigint, c_largeint;
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
SELECT * FROM tbl_scalar_types_unique2_bitmapindex WHERE c_date = '2022-10-09' ORDER BY c_bigint, c_largeint;
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
SELECT * FROM tbl_scalar_types_unique2_bitmapindex WHERE c_datetime = '2022-07-31 12:07:00' ORDER BY c_bigint, c_largeint;
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
SELECT * FROM tbl_scalar_types_unique2_bitmapindex WHERE c_datev2 = '2022-03-06' ORDER BY c_bigint, c_largeint;
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
SELECT * FROM tbl_scalar_types_unique2_bitmapindex WHERE c_datetimev2 = '2022-11-18 06:05:56' ORDER BY c_bigint, c_largeint;
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
SELECT * FROM tbl_scalar_types_unique2_bitmapindex WHERE c_char = '202.171.74.217' ORDER BY c_bigint, c_largeint;
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
SELECT * FROM tbl_scalar_types_unique2_bitmapindex WHERE c_varchar = 'modi_a@Feedmix.biz' ORDER BY c_bigint, c_largeint;
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
SELECT * FROM tbl_scalar_types_unique2_bitmapindex WHERE c_string = 'Reinke Junction 63' ORDER BY c_bigint, c_largeint;
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,7 @@ suite("test_build_index_fault", "inverted_index, nonConcurrent"){
useTime = t
sleep(delta_time)
}
logger.info("wait_for_last_build_index_on_table_finish debug: " + alter_res)
assertTrue(useTime <= OpTimeout, "wait_for_last_build_index_on_table_finish timeout")
return "wait_timeout"
}
Expand All @@ -91,7 +92,8 @@ suite("test_build_index_fault", "inverted_index, nonConcurrent"){
useTime = t
sleep(delta_time)
}
assertTrue(useTime <= OpTimeout, "wait_for_last_build_index_on_table_finish timeout")
logger.info("wait_for_last_build_index_on_table_running debug: " + alter_res)
assertTrue(useTime <= OpTimeout, "wait_for_last_build_index_on_table_running timeout")
return "wait_timeout"
}

Expand Down
4 changes: 2 additions & 2 deletions regression-test/suites/index_p0/test_bitmap_index.groovy
Original file line number Diff line number Diff line change
Expand Up @@ -178,7 +178,7 @@ suite("test_bitmap_index") {
}
test{
sql "ALTER TABLE ${tbName2} ADD INDEX index16 (v1) USING BITMAP;"
exception "errCode = 2, detailMessage = INVERTED index only used in columns of DUP_KEYS/UNIQUE_KEYS MOW table or key columns of all table. invalid index: index16"
exception "errCode = 2, detailMessage = index should only be used in columns of DUP_KEYS/UNIQUE_KEYS table or key columns of AGG_KEYS table. invalid index: index16"
}

sql "insert into ${tbName2} values(1,1,1,1,'1','1','2022-05-31','2022-05-31 10:00:00',1,1.0,1,'2022-05-31','2022-05-31 10:00:00.111111','2022-05-31 10:00:00.111111','2022-05-31 10:00:00.111111',1);"
Expand Down Expand Up @@ -340,7 +340,7 @@ suite("test_bitmap_index") {
"replication_allocation" = "tag.location.default: 1",
"is_being_synced" = "false",
"storage_format" = "V2",
"enable_unique_key_merge_on_write" = "true",
"enable_unique_key_merge_on_write" = "false",
"light_schema_change" = "true",
"disable_auto_compaction" = "false",
"enable_single_replica_compaction" = "false"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -288,6 +288,7 @@ suite("test_index_compaction_null", "p0") {
PROPERTIES (
"replication_allocation" = "tag.location.default: 1",
"disable_auto_compaction" = "true",
"enable_unique_key_merge_on_write" = "true",
"in_memory" = "false",
"storage_format" = "V2"
)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -158,7 +158,7 @@ suite("test_bitmap_index", "inverted_index") {
}
test{
sql "ALTER TABLE ${tbName2} ADD INDEX index12 (v1) USING BITMAP;"
exception "errCode = 2, detailMessage = INVERTED index only used in columns of DUP_KEYS/UNIQUE_KEYS MOW table or key columns of all table. invalid index: index12"
exception "errCode = 2, detailMessage = index should only be used in columns of DUP_KEYS/UNIQUE_KEYS table or key columns of AGG_KEYS table. invalid index: index12"
}

sql "insert into ${tbName2} values(1,1,1,1,'1','1','2022-05-31','2022-05-31 10:00:00',1,1.0,1,1);"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,7 @@ suite("test_build_index", "inverted_index"){
useTime = t
sleep(delta_time)
}
logger.info("wait_for_last_build_index_on_table_finish debug: " + alter_res)
assertTrue(useTime <= OpTimeout, "wait_for_last_build_index_on_table_finish timeout")
return "wait_timeout"
}
Expand All @@ -91,7 +92,8 @@ suite("test_build_index", "inverted_index"){
useTime = t
sleep(delta_time)
}
assertTrue(useTime <= OpTimeout, "wait_for_last_build_index_on_table_finish timeout")
logger.info("wait_for_last_build_index_on_table_running debug: " + alter_res)
assertTrue(useTime <= OpTimeout, "wait_for_last_build_index_on_table_running timeout")
return "wait_timeout"
}

Expand Down
Loading