From 9967b9c01bc140a93f083ff5c763731a9bd48793 Mon Sep 17 00:00:00 2001 From: zzzxl1993 Date: Tue, 4 Mar 2025 12:43:59 +0800 Subject: [PATCH] [fix](inverted index) Fix for Inaccurate match_phrase_prefix Cache in Query Processing --- .../segment_v2/inverted_index_reader.cpp | 31 +++++++++---------- .../test_index_match_phrase_prefix_1.out | 6 ++++ .../test_index_match_regexp.out | 6 ++++ .../test_index_match_phrase_prefix_1.groovy | 9 ++++++ .../test_index_match_regexp.groovy | 7 +++++ 5 files changed, 43 insertions(+), 16 deletions(-) diff --git a/be/src/olap/rowset/segment_v2/inverted_index_reader.cpp b/be/src/olap/rowset/segment_v2/inverted_index_reader.cpp index 7281c3a6fe2c7a..1c7af3fc8b1b7d 100644 --- a/be/src/olap/rowset/segment_v2/inverted_index_reader.cpp +++ b/be/src/olap/rowset/segment_v2/inverted_index_reader.cpp @@ -330,13 +330,14 @@ Status FullTextIndexReader::query(OlapReaderStatistics* stats, RuntimeState* run VLOG_DEBUG << column_name << " begin to search the fulltext index from clucene, query_str [" << search_str << "]"; + const auto& queryOptions = runtime_state->query_options(); try { InvertedIndexQueryInfo query_info; InvertedIndexQueryCache::CacheKey cache_key; auto index_file_key = _inverted_index_file_reader->get_index_file_key(&_index_meta); + // terms if (query_type == InvertedIndexQueryType::MATCH_REGEXP_QUERY) { - cache_key = {index_file_key, column_name, query_type, search_str}; query_info.terms.emplace_back(search_str); } else { if (query_type == InvertedIndexQueryType::MATCH_PHRASE_QUERY) { @@ -356,6 +357,7 @@ Status FullTextIndexReader::query(OlapReaderStatistics* stats, RuntimeState* run get_analyse_result(query_info.terms, reader.get(), analyzer.get(), column_name, query_type); } + if (query_info.terms.empty()) { auto msg = fmt::format( "token parser result is empty for query, " @@ -369,22 +371,20 @@ Status FullTextIndexReader::query(OlapReaderStatistics* stats, RuntimeState* run } } - std::unique_ptr query; + // field_name query_info.field_name = StringUtil::string_to_wstring(column_name); - if (query_type == InvertedIndexQueryType::MATCH_PHRASE_QUERY || - query_type == InvertedIndexQueryType::MATCH_PHRASE_PREFIX_QUERY || - query_type == InvertedIndexQueryType::MATCH_PHRASE_EDGE_QUERY || - query_type == InvertedIndexQueryType::MATCH_ALL_QUERY || - query_type == InvertedIndexQueryType::EQUAL_QUERY || - query_type == InvertedIndexQueryType::MATCH_ANY_QUERY) { - std::string str_tokens = join(query_info.terms, " "); - if (query_type == InvertedIndexQueryType::MATCH_PHRASE_QUERY) { - str_tokens += " " + std::to_string(query_info.slop); - str_tokens += " " + std::to_string(query_info.ordered); - } - cache_key = {index_file_key, column_name, query_type, str_tokens}; + // cache_key + std::string str_tokens = join(query_info.terms, " "); + if (query_type == InvertedIndexQueryType::MATCH_PHRASE_QUERY) { + str_tokens += " " + std::to_string(query_info.slop); + str_tokens += " " + std::to_string(query_info.ordered); + } else if (query_type == InvertedIndexQueryType::MATCH_PHRASE_PREFIX_QUERY || + query_type == InvertedIndexQueryType::MATCH_REGEXP_QUERY) { + str_tokens += " " + std::to_string(queryOptions.inverted_index_max_expansions); } + cache_key = {index_file_key, column_name, query_type, std::move(str_tokens)}; + auto* cache = InvertedIndexQueryCache::instance(); InvertedIndexQueryCacheHandle cache_handler; @@ -393,12 +393,11 @@ Status FullTextIndexReader::query(OlapReaderStatistics* stats, RuntimeState* run if (cache_status.ok()) { return Status::OK(); } - FulltextIndexSearcherPtr* searcher_ptr = nullptr; InvertedIndexCacheHandle inverted_index_cache_handle; RETURN_IF_ERROR(handle_searcher_cache(&inverted_index_cache_handle, stats)); auto searcher_variant = inverted_index_cache_handle.get_index_searcher(); - searcher_ptr = std::get_if(&searcher_variant); + auto* searcher_ptr = std::get_if(&searcher_variant); if (searcher_ptr != nullptr) { term_match_bitmap = std::make_shared(); RETURN_IF_ERROR(match_index_search(stats, runtime_state, query_type, query_info, diff --git a/regression-test/data/inverted_index_p0/test_index_match_phrase_prefix_1.out b/regression-test/data/inverted_index_p0/test_index_match_phrase_prefix_1.out index e7e934f394e05c..7ac0d5f7ec2bdd 100644 --- a/regression-test/data/inverted_index_p0/test_index_match_phrase_prefix_1.out +++ b/regression-test/data/inverted_index_p0/test_index_match_phrase_prefix_1.out @@ -11,3 +11,9 @@ -- !sql -- 6 +-- !sql -- +0 + +-- !sql -- +1 + diff --git a/regression-test/data/inverted_index_p0/test_index_match_regexp.out b/regression-test/data/inverted_index_p0/test_index_match_regexp.out index fb5d23ad266e10..2c06da4147c708 100644 --- a/regression-test/data/inverted_index_p0/test_index_match_regexp.out +++ b/regression-test/data/inverted_index_p0/test_index_match_regexp.out @@ -20,3 +20,9 @@ -- !sql -- 0 +-- !sql -- +4 + +-- !sql -- +377 + diff --git a/regression-test/suites/inverted_index_p0/test_index_match_phrase_prefix_1.groovy b/regression-test/suites/inverted_index_p0/test_index_match_phrase_prefix_1.groovy index f42462f12a624a..5ee38c9e403bdc 100644 --- a/regression-test/suites/inverted_index_p0/test_index_match_phrase_prefix_1.groovy +++ b/regression-test/suites/inverted_index_p0/test_index_match_phrase_prefix_1.groovy @@ -47,6 +47,8 @@ suite("test_index_match_phrase_prefix_1", "nonConcurrent"){ sql """ INSERT INTO ${indexTbName1} VALUES (6, "O1704361998540E2Cemx9S 123456789", "O1704361998540E2Cemx9S 123456789", "O1704361998540E2Cemx9S 123456789"); """ sql """ INSERT INTO ${indexTbName1} VALUES (7, "O1704361998540E2Cemx9S*123456789", "O1704361998540E2Cemx9S*123456789", "O1704361998540E2Cemx9S*123456789"); """ + sql """ INSERT INTO ${indexTbName1} VALUES (1, "", "s1", ""), (2, "", "s2", ""), (3, "", "s3", ""), (4, "", "s4", ""), (5, "", "tv s5", ""); """ + try { sql "sync" sql """ set enable_common_expr_pushdown = true; """ @@ -58,7 +60,14 @@ suite("test_index_match_phrase_prefix_1", "nonConcurrent"){ qt_sql """ select count() from ${indexTbName1} where c match_phrase_prefix 'O1704361998540E2Cemx9S=123456789'; """ qt_sql """ select count() from ${indexTbName1} where d match_phrase_prefix 'O1704361998540E2Cemx9S=123456789'; """ + sql """ set inverted_index_max_expansions = 3; """ + qt_sql """ select count() from ${indexTbName1} where c match_phrase_prefix 'tv s'; """ + + sql """ set inverted_index_max_expansions = 5; """ + qt_sql """ select count() from ${indexTbName1} where c match_phrase_prefix 'tv s'; """ + } finally { + sql """ set inverted_index_max_expansions = 50; """ GetDebugPoint().disableDebugPointForAllBEs("VMatchPredicate.execute") } } \ No newline at end of file diff --git a/regression-test/suites/inverted_index_p0/test_index_match_regexp.groovy b/regression-test/suites/inverted_index_p0/test_index_match_regexp.groovy index 49f0f56398909e..1f508306dbbe28 100644 --- a/regression-test/suites/inverted_index_p0/test_index_match_regexp.groovy +++ b/regression-test/suites/inverted_index_p0/test_index_match_regexp.groovy @@ -90,7 +90,14 @@ suite("test_index_match_regexp", "nonConcurrent"){ qt_sql """ select count() from test_index_match_regexp where request match_regexp '.*tickets.*'; """ qt_sql """ select count() from test_index_match_regexp where request match_regexp 'nonexistence'; """ + sql """ set inverted_index_max_expansions = 1; """ + qt_sql """ select count() from test_index_match_regexp where request match_regexp 'b'; """ + + sql """ set inverted_index_max_expansions = 50; """ + qt_sql """ select count() from test_index_match_regexp where request match_regexp 'b'; """ + } finally { + sql """ set inverted_index_max_expansions = 50; """ GetDebugPoint().disableDebugPointForAllBEs("VMatchPredicate.execute") } } \ No newline at end of file