From dbced92662572863c79fe077bc1e43112dc49871 Mon Sep 17 00:00:00 2001 From: zzzxl Date: Mon, 6 Jan 2025 16:08:46 +0800 Subject: [PATCH] [fix](inverted index) Fix for Inaccurate match_phrase_prefix Cache in Query Processing (#46310) Problem Summary: 1. different values of inverted_index_max_expansions require separate caches. --- .../segment_v2/inverted_index_reader.cpp | 33 +++++++++---------- .../test_index_match_phrase_prefix_1.out | 6 ++++ .../test_index_match_regexp.out | 6 ++++ .../test_index_match_phrase_prefix_1.groovy | 9 +++++ .../test_index_match_regexp.groovy | 7 ++++ 5 files changed, 44 insertions(+), 17 deletions(-) diff --git a/be/src/olap/rowset/segment_v2/inverted_index_reader.cpp b/be/src/olap/rowset/segment_v2/inverted_index_reader.cpp index a75a9462f913ab..a776f8f464b9dd 100644 --- a/be/src/olap/rowset/segment_v2/inverted_index_reader.cpp +++ b/be/src/olap/rowset/segment_v2/inverted_index_reader.cpp @@ -227,7 +227,7 @@ Status InvertedIndexReader::match_index_search( InvertedIndexQueryType query_type, const InvertedIndexQueryInfo& query_info, const FulltextIndexSearcherPtr& index_searcher, const std::shared_ptr& term_match_bitmap) { - TQueryOptions queryOptions = runtime_state->query_options(); + const auto& queryOptions = runtime_state->query_options(); try { SCOPED_RAW_TIMER(&stats->inverted_index_searcher_search_timer); auto query = QueryFactory::create(query_type, index_searcher, queryOptions, io_ctx); @@ -262,13 +262,14 @@ Status FullTextIndexReader::query(const io::IOContext* io_ctx, OlapReaderStatist VLOG_DEBUG << column_name << " begin to search the fulltext index from clucene, query_str [" << search_str << "]"; + const auto& queryOptions = runtime_state->query_options(); try { InvertedIndexQueryInfo query_info; InvertedIndexQueryCache::CacheKey cache_key; auto index_file_key = _inverted_index_file_reader->get_index_file_cache_key(&_index_meta); + // terms if (query_type == InvertedIndexQueryType::MATCH_REGEXP_QUERY) { - cache_key = {index_file_key, column_name, query_type, search_str}; query_info.terms.emplace_back(search_str); } else { if (query_type == InvertedIndexQueryType::MATCH_PHRASE_QUERY) { @@ -291,6 +292,7 @@ Status FullTextIndexReader::query(const io::IOContext* io_ctx, OlapReaderStatist query_info.terms = inverted_index::InvertedIndexAnalyzer::get_analyse_result( reader.get(), analyzer.get(), column_name, query_type); } + if (query_info.terms.empty()) { auto msg = fmt::format( "token parser result is empty for query, " @@ -304,22 +306,20 @@ Status FullTextIndexReader::query(const io::IOContext* io_ctx, OlapReaderStatist } } - std::unique_ptr query; + // field_name query_info.field_name = StringUtil::string_to_wstring(column_name); - if (query_type == InvertedIndexQueryType::MATCH_PHRASE_QUERY || - query_type == InvertedIndexQueryType::MATCH_PHRASE_PREFIX_QUERY || - query_type == InvertedIndexQueryType::MATCH_PHRASE_EDGE_QUERY || - query_type == InvertedIndexQueryType::MATCH_ALL_QUERY || - query_type == InvertedIndexQueryType::EQUAL_QUERY || - query_type == InvertedIndexQueryType::MATCH_ANY_QUERY) { - std::string str_tokens = join(query_info.terms, " "); - if (query_type == InvertedIndexQueryType::MATCH_PHRASE_QUERY) { - str_tokens += " " + std::to_string(query_info.slop); - str_tokens += " " + std::to_string(query_info.ordered); - } - cache_key = {index_file_key, column_name, query_type, str_tokens}; + // cache_key + std::string str_tokens = join(query_info.terms, " "); + if (query_type == InvertedIndexQueryType::MATCH_PHRASE_QUERY) { + str_tokens += " " + std::to_string(query_info.slop); + str_tokens += " " + std::to_string(query_info.ordered); + } else if (query_type == InvertedIndexQueryType::MATCH_PHRASE_PREFIX_QUERY || + query_type == InvertedIndexQueryType::MATCH_REGEXP_QUERY) { + str_tokens += " " + std::to_string(queryOptions.inverted_index_max_expansions); } + cache_key = {index_file_key, column_name, query_type, std::move(str_tokens)}; + auto* cache = InvertedIndexQueryCache::instance(); InvertedIndexQueryCacheHandle cache_handler; @@ -328,12 +328,11 @@ Status FullTextIndexReader::query(const io::IOContext* io_ctx, OlapReaderStatist if (cache_status.ok()) { return Status::OK(); } - FulltextIndexSearcherPtr* searcher_ptr = nullptr; InvertedIndexCacheHandle inverted_index_cache_handle; RETURN_IF_ERROR(handle_searcher_cache(&inverted_index_cache_handle, io_ctx, stats)); auto searcher_variant = inverted_index_cache_handle.get_index_searcher(); - searcher_ptr = std::get_if(&searcher_variant); + auto* searcher_ptr = std::get_if(&searcher_variant); if (searcher_ptr != nullptr) { term_match_bitmap = std::make_shared(); RETURN_IF_ERROR(match_index_search(io_ctx, stats, runtime_state, query_type, query_info, diff --git a/regression-test/data/inverted_index_p0/test_index_match_phrase_prefix_1.out b/regression-test/data/inverted_index_p0/test_index_match_phrase_prefix_1.out index e7e934f394e05c..7ac0d5f7ec2bdd 100644 --- a/regression-test/data/inverted_index_p0/test_index_match_phrase_prefix_1.out +++ b/regression-test/data/inverted_index_p0/test_index_match_phrase_prefix_1.out @@ -11,3 +11,9 @@ -- !sql -- 6 +-- !sql -- +0 + +-- !sql -- +1 + diff --git a/regression-test/data/inverted_index_p0/test_index_match_regexp.out b/regression-test/data/inverted_index_p0/test_index_match_regexp.out index fb5d23ad266e10..2c06da4147c708 100644 --- a/regression-test/data/inverted_index_p0/test_index_match_regexp.out +++ b/regression-test/data/inverted_index_p0/test_index_match_regexp.out @@ -20,3 +20,9 @@ -- !sql -- 0 +-- !sql -- +4 + +-- !sql -- +377 + diff --git a/regression-test/suites/inverted_index_p0/test_index_match_phrase_prefix_1.groovy b/regression-test/suites/inverted_index_p0/test_index_match_phrase_prefix_1.groovy index f42462f12a624a..5ee38c9e403bdc 100644 --- a/regression-test/suites/inverted_index_p0/test_index_match_phrase_prefix_1.groovy +++ b/regression-test/suites/inverted_index_p0/test_index_match_phrase_prefix_1.groovy @@ -47,6 +47,8 @@ suite("test_index_match_phrase_prefix_1", "nonConcurrent"){ sql """ INSERT INTO ${indexTbName1} VALUES (6, "O1704361998540E2Cemx9S 123456789", "O1704361998540E2Cemx9S 123456789", "O1704361998540E2Cemx9S 123456789"); """ sql """ INSERT INTO ${indexTbName1} VALUES (7, "O1704361998540E2Cemx9S*123456789", "O1704361998540E2Cemx9S*123456789", "O1704361998540E2Cemx9S*123456789"); """ + sql """ INSERT INTO ${indexTbName1} VALUES (1, "", "s1", ""), (2, "", "s2", ""), (3, "", "s3", ""), (4, "", "s4", ""), (5, "", "tv s5", ""); """ + try { sql "sync" sql """ set enable_common_expr_pushdown = true; """ @@ -58,7 +60,14 @@ suite("test_index_match_phrase_prefix_1", "nonConcurrent"){ qt_sql """ select count() from ${indexTbName1} where c match_phrase_prefix 'O1704361998540E2Cemx9S=123456789'; """ qt_sql """ select count() from ${indexTbName1} where d match_phrase_prefix 'O1704361998540E2Cemx9S=123456789'; """ + sql """ set inverted_index_max_expansions = 3; """ + qt_sql """ select count() from ${indexTbName1} where c match_phrase_prefix 'tv s'; """ + + sql """ set inverted_index_max_expansions = 5; """ + qt_sql """ select count() from ${indexTbName1} where c match_phrase_prefix 'tv s'; """ + } finally { + sql """ set inverted_index_max_expansions = 50; """ GetDebugPoint().disableDebugPointForAllBEs("VMatchPredicate.execute") } } \ No newline at end of file diff --git a/regression-test/suites/inverted_index_p0/test_index_match_regexp.groovy b/regression-test/suites/inverted_index_p0/test_index_match_regexp.groovy index 49f0f56398909e..1f508306dbbe28 100644 --- a/regression-test/suites/inverted_index_p0/test_index_match_regexp.groovy +++ b/regression-test/suites/inverted_index_p0/test_index_match_regexp.groovy @@ -90,7 +90,14 @@ suite("test_index_match_regexp", "nonConcurrent"){ qt_sql """ select count() from test_index_match_regexp where request match_regexp '.*tickets.*'; """ qt_sql """ select count() from test_index_match_regexp where request match_regexp 'nonexistence'; """ + sql """ set inverted_index_max_expansions = 1; """ + qt_sql """ select count() from test_index_match_regexp where request match_regexp 'b'; """ + + sql """ set inverted_index_max_expansions = 50; """ + qt_sql """ select count() from test_index_match_regexp where request match_regexp 'b'; """ + } finally { + sql """ set inverted_index_max_expansions = 50; """ GetDebugPoint().disableDebugPointForAllBEs("VMatchPredicate.execute") } } \ No newline at end of file