From abf12359a130e905ac0cc79c094c4e7124c1f641 Mon Sep 17 00:00:00 2001 From: airborne12 Date: Mon, 11 Nov 2024 14:37:23 +0800 Subject: [PATCH] [fix](segcompaction) fix coredump for inverted index file writer close in segment compaction (#43477) Related PR: #41625 Problem Summary: Fix coredump when doing segcompaction after refactor of inverted index file writer. Co-authored-by: airborne12 --- be/src/olap/rowset/segcompaction.cpp | 4 +- be/src/runtime/exec_env.cpp | 4 ++ be/src/runtime/exec_env.h | 2 + be/test/olap/segcompaction_test.cpp | 59 ++++++++++++++++++++++++++++ 4 files changed, 68 insertions(+), 1 deletion(-) diff --git a/be/src/olap/rowset/segcompaction.cpp b/be/src/olap/rowset/segcompaction.cpp index fc8baf952c1863..c9a5a77bc37487 100644 --- a/be/src/olap/rowset/segcompaction.cpp +++ b/be/src/olap/rowset/segcompaction.cpp @@ -310,7 +310,9 @@ Status SegcompactionWorker::_do_compact_segments(SegCompactionCandidatesSharedPt _writer->_num_segcompacted); } RETURN_IF_ERROR(_writer->_rename_compacted_segments(begin, end)); - + if (_inverted_index_file_writer != nullptr) { + _inverted_index_file_writer.reset(); + } if (VLOG_DEBUG_IS_ON) { _writer->vlog_buffer.clear(); for (const auto& entry : std::filesystem::directory_iterator(ctx.tablet_path)) { diff --git a/be/src/runtime/exec_env.cpp b/be/src/runtime/exec_env.cpp index e41cc982a7482c..872069ee70a1a4 100644 --- a/be/src/runtime/exec_env.cpp +++ b/be/src/runtime/exec_env.cpp @@ -45,6 +45,10 @@ ExecEnv::~ExecEnv() { } #ifdef BE_TEST +void ExecEnv::set_inverted_index_searcher_cache( + segment_v2::InvertedIndexSearcherCache* inverted_index_searcher_cache) { + _inverted_index_searcher_cache = inverted_index_searcher_cache; +} void ExecEnv::set_storage_engine(std::unique_ptr&& engine) { _storage_engine = std::move(engine); } diff --git a/be/src/runtime/exec_env.h b/be/src/runtime/exec_env.h index afd99a624f2c78..307c2586fc6fe5 100644 --- a/be/src/runtime/exec_env.h +++ b/be/src/runtime/exec_env.h @@ -271,6 +271,8 @@ class ExecEnv { } void set_storage_engine(std::unique_ptr&& engine); + void set_inverted_index_searcher_cache( + segment_v2::InvertedIndexSearcherCache* inverted_index_searcher_cache); void set_cache_manager(CacheManager* cm) { this->_cache_manager = cm; } void set_tablet_schema_cache(TabletSchemaCache* c) { this->_tablet_schema_cache = c; } void set_tablet_column_object_pool(TabletColumnObjectPool* c) { diff --git a/be/test/olap/segcompaction_test.cpp b/be/test/olap/segcompaction_test.cpp index ba0d23acb02cef..fd26f9ca0a734d 100644 --- a/be/test/olap/segcompaction_test.cpp +++ b/be/test/olap/segcompaction_test.cpp @@ -48,6 +48,7 @@ using namespace ErrorCode; static const uint32_t MAX_PATH_LEN = 1024; static StorageEngine* l_engine = nullptr; static const std::string lTestDir = "./data_test/data/segcompaction_test"; +constexpr static std::string_view tmp_dir = "./data_test/tmp"; class SegCompactionTest : public testing::Test { public: @@ -58,6 +59,7 @@ class SegCompactionTest : public testing::Test { config::tablet_map_shard_size = 1; config::txn_map_shard_size = 1; config::txn_shard_size = 1; + config::inverted_index_fd_number_limit_percent = 0; char buffer[MAX_PATH_LEN]; EXPECT_NE(getcwd(buffer, MAX_PATH_LEN), nullptr); @@ -71,6 +73,23 @@ class SegCompactionTest : public testing::Test { std::vector paths; paths.emplace_back(config::storage_root_path, -1); + // tmp dir + EXPECT_TRUE(io::global_local_filesystem()->delete_directory(tmp_dir).ok()); + EXPECT_TRUE(io::global_local_filesystem()->create_directory(tmp_dir).ok()); + paths.emplace_back(std::string(tmp_dir), 1024000000); + auto tmp_file_dirs = std::make_unique(paths); + EXPECT_TRUE(tmp_file_dirs->init().ok()); + ExecEnv::GetInstance()->set_tmp_file_dir(std::move(tmp_file_dirs)); + + // use memory limit + int64_t inverted_index_cache_limit = 0; + _inverted_index_searcher_cache = std::unique_ptr( + InvertedIndexSearcherCache::create_global_instance(inverted_index_cache_limit, + 256)); + + ExecEnv::GetInstance()->set_inverted_index_searcher_cache( + _inverted_index_searcher_cache.get()); + doris::EngineOptions options; options.store_paths = paths; @@ -98,6 +117,7 @@ class SegCompactionTest : public testing::Test { ExecEnv* exec_env = doris::ExecEnv::GetInstance(); l_engine = nullptr; exec_env->set_storage_engine(nullptr); + exec_env->set_inverted_index_searcher_cache(nullptr); } protected: @@ -139,6 +159,7 @@ class SegCompactionTest : public testing::Test { tablet_schema_pb.set_num_rows_per_row_block(1024); tablet_schema_pb.set_compress_kind(COMPRESS_NONE); tablet_schema_pb.set_next_column_unique_id(4); + tablet_schema_pb.set_inverted_index_storage_format(InvertedIndexStorageFormatPB::V2); ColumnPB* column_1 = tablet_schema_pb.add_column(); column_1->set_unique_id(1); @@ -149,6 +170,11 @@ class SegCompactionTest : public testing::Test { column_1->set_index_length(4); column_1->set_is_nullable(true); column_1->set_is_bf_column(false); + auto tablet_index_1 = tablet_schema_pb.add_index(); + tablet_index_1->set_index_id(1); + tablet_index_1->set_index_name("column_1"); + tablet_index_1->set_index_type(IndexType::INVERTED); + tablet_index_1->add_col_unique_id(1); ColumnPB* column_2 = tablet_schema_pb.add_column(); column_2->set_unique_id(2); @@ -161,6 +187,11 @@ class SegCompactionTest : public testing::Test { column_2->set_is_key(true); column_2->set_is_nullable(true); column_2->set_is_bf_column(false); + auto tablet_index_2 = tablet_schema_pb.add_index(); + tablet_index_2->set_index_id(2); + tablet_index_2->set_index_name("column_2"); + tablet_index_2->set_index_type(IndexType::INVERTED); + tablet_index_2->add_col_unique_id(2); for (int i = 1; i <= num_value_col; i++) { ColumnPB* v_column = tablet_schema_pb.add_column(); @@ -226,6 +257,7 @@ class SegCompactionTest : public testing::Test { private: std::unique_ptr _data_dir; + std::unique_ptr _inverted_index_searcher_cache; }; TEST_F(SegCompactionTest, SegCompactionThenRead) { @@ -281,6 +313,13 @@ TEST_F(SegCompactionTest, SegCompactionThenRead) { ls.push_back("10047_4.dat"); ls.push_back("10047_5.dat"); ls.push_back("10047_6.dat"); + ls.push_back("10047_0.idx"); + ls.push_back("10047_1.idx"); + ls.push_back("10047_2.idx"); + ls.push_back("10047_3.idx"); + ls.push_back("10047_4.idx"); + ls.push_back("10047_5.idx"); + ls.push_back("10047_6.idx"); EXPECT_TRUE(check_dir(ls)); } @@ -483,6 +522,13 @@ TEST_F(SegCompactionTest, SegCompactionInterleaveWithBig_ooooOOoOooooooooO) { ls.push_back("10048_4.dat"); // O ls.push_back("10048_5.dat"); // oooooooo ls.push_back("10048_6.dat"); // O + ls.push_back("10048_0.idx"); // oooo + ls.push_back("10048_1.idx"); // O + ls.push_back("10048_2.idx"); // O + ls.push_back("10048_3.idx"); // o + ls.push_back("10048_4.idx"); // O + ls.push_back("10048_5.idx"); // oooooooo + ls.push_back("10048_6.idx"); // O EXPECT_TRUE(check_dir(ls)); } } @@ -609,6 +655,11 @@ TEST_F(SegCompactionTest, SegCompactionInterleaveWithBig_OoOoO) { ls.push_back("10049_2.dat"); // O ls.push_back("10049_3.dat"); // o ls.push_back("10049_4.dat"); // O + ls.push_back("10049_0.idx"); // O + ls.push_back("10049_1.idx"); // o + ls.push_back("10049_2.idx"); // O + ls.push_back("10049_3.idx"); // o + ls.push_back("10049_4.idx"); // O EXPECT_TRUE(check_dir(ls)); } } @@ -770,6 +821,10 @@ TEST_F(SegCompactionTest, SegCompactionThenReadUniqueTableSmall) { ls.push_back("10051_1.dat"); ls.push_back("10051_2.dat"); ls.push_back("10051_3.dat"); + ls.push_back("10051_0.idx"); + ls.push_back("10051_1.idx"); + ls.push_back("10051_2.idx"); + ls.push_back("10051_3.idx"); EXPECT_TRUE(check_dir(ls)); } @@ -988,6 +1043,10 @@ TEST_F(SegCompactionTest, SegCompactionThenReadAggTableSmall) { ls.push_back("10052_1.dat"); ls.push_back("10052_2.dat"); ls.push_back("10052_3.dat"); + ls.push_back("10052_0.idx"); + ls.push_back("10052_1.idx"); + ls.push_back("10052_2.idx"); + ls.push_back("10052_3.idx"); EXPECT_TRUE(check_dir(ls)); }