From 74a016bba509a54b8b661c0abcaeecf33201c47d Mon Sep 17 00:00:00 2001 From: huangkangping Date: Fri, 21 Jun 2019 15:41:45 +0800 Subject: [PATCH 1/2] fix unused rowset bug --- be/src/olap/rowset/alpha_rowset.cpp | 9 +++++++++ be/src/olap/rowset/alpha_rowset.h | 2 ++ be/src/olap/rowset/rowset.h | 2 ++ be/src/olap/storage_engine.cpp | 7 +++++-- be/src/olap/storage_engine.h | 2 +- 5 files changed, 19 insertions(+), 3 deletions(-) diff --git a/be/src/olap/rowset/alpha_rowset.cpp b/be/src/olap/rowset/alpha_rowset.cpp index 97642e0a489c8d..bdc7a313d25840 100644 --- a/be/src/olap/rowset/alpha_rowset.cpp +++ b/be/src/olap/rowset/alpha_rowset.cpp @@ -18,6 +18,7 @@ #include "olap/rowset/alpha_rowset.h" #include "olap/rowset/alpha_rowset_meta.h" #include "olap/rowset/rowset_meta_manager.h" +#include "util/hash_util.hpp" namespace doris { @@ -514,4 +515,12 @@ OLAPStatus AlphaRowset::reset_sizeinfo() { return OLAP_SUCCESS; } +uint32_t AlphaRowset::unique_id() { + // rowset path + rowset_id is unique for a rowset + std::string unique_path = _rowset_path + std::to_string(rowset_id()); + uint32_t seed = 0; + seed = HashUtil::crc_hash(unique_path.data(), unique_path.length(), seed); + return seed; +} + } // namespace doris diff --git a/be/src/olap/rowset/alpha_rowset.h b/be/src/olap/rowset/alpha_rowset.h index fc1c227b2acdbd..292359c84eeceb 100644 --- a/be/src/olap/rowset/alpha_rowset.h +++ b/be/src/olap/rowset/alpha_rowset.h @@ -124,6 +124,8 @@ class AlphaRowset : public Rowset { // info by using segment's info OLAPStatus reset_sizeinfo(); + uint32_t unique_id() override; + private: OLAPStatus _init_segment_groups(); diff --git a/be/src/olap/rowset/rowset.h b/be/src/olap/rowset/rowset.h index 16513ad30aaeba..0e0c934603b9fc 100644 --- a/be/src/olap/rowset/rowset.h +++ b/be/src/olap/rowset/rowset.h @@ -122,6 +122,8 @@ class Rowset : public std::enable_shared_from_this { virtual bool check_path(const std::string& path) = 0; + virtual uint32_t unique_id() = 0; + bool need_delete_file() { return _need_delete_file; } diff --git a/be/src/olap/storage_engine.cpp b/be/src/olap/storage_engine.cpp index efa55e8780cd60..19bc82c89400e5 100644 --- a/be/src/olap/storage_engine.cpp +++ b/be/src/olap/storage_engine.cpp @@ -761,10 +761,13 @@ void StorageEngine::start_delete_unused_rowset() { void StorageEngine::add_unused_rowset(RowsetSharedPtr rowset) { if (rowset == nullptr) { return; } _gc_mutex.lock(); - auto it = _unused_rowsets.find(rowset->rowset_id()); + LOG(INFO) << "add unused rowset, rowset id:" << rowset->rowset_id() + << "version:" << rowset->version().first + << "-" << rowset->version().second; + auto it = _unused_rowsets.find(rowset->unique_id()); if (it == _unused_rowsets.end()) { rowset->set_need_delete_file(true); - _unused_rowsets[rowset->rowset_id()] = rowset; + _unused_rowsets[rowset->unique_id()] = rowset; } _gc_mutex.unlock(); } diff --git a/be/src/olap/storage_engine.h b/be/src/olap/storage_engine.h index 4cc61e590719a0..ff770d359392c1 100644 --- a/be/src/olap/storage_engine.h +++ b/be/src/olap/storage_engine.h @@ -298,7 +298,7 @@ class StorageEngine { static StorageEngine* _s_instance; std::unordered_map> _gc_files; - std::unordered_map _unused_rowsets; + std::unordered_map _unused_rowsets; Mutex _gc_mutex; std::thread _unused_rowset_monitor_thread; From 6b3c59b93ac0b6d5ab86f85bfa45df2b232af94e Mon Sep 17 00:00:00 2001 From: huangkangping Date: Fri, 21 Jun 2019 16:33:05 +0800 Subject: [PATCH 2/2] modify id to string --- be/src/olap/rowset/alpha_rowset.cpp | 7 ++----- be/src/olap/rowset/alpha_rowset.h | 2 +- be/src/olap/rowset/rowset.h | 2 +- be/src/olap/storage_engine.cpp | 5 +++-- be/src/olap/storage_engine.h | 2 +- 5 files changed, 8 insertions(+), 10 deletions(-) diff --git a/be/src/olap/rowset/alpha_rowset.cpp b/be/src/olap/rowset/alpha_rowset.cpp index bdc7a313d25840..b017ce48b6435c 100644 --- a/be/src/olap/rowset/alpha_rowset.cpp +++ b/be/src/olap/rowset/alpha_rowset.cpp @@ -515,12 +515,9 @@ OLAPStatus AlphaRowset::reset_sizeinfo() { return OLAP_SUCCESS; } -uint32_t AlphaRowset::unique_id() { +std::string AlphaRowset::unique_id() { // rowset path + rowset_id is unique for a rowset - std::string unique_path = _rowset_path + std::to_string(rowset_id()); - uint32_t seed = 0; - seed = HashUtil::crc_hash(unique_path.data(), unique_path.length(), seed); - return seed; + return _rowset_path + "/" + std::to_string(rowset_id()); } } // namespace doris diff --git a/be/src/olap/rowset/alpha_rowset.h b/be/src/olap/rowset/alpha_rowset.h index 292359c84eeceb..8a65b66918974d 100644 --- a/be/src/olap/rowset/alpha_rowset.h +++ b/be/src/olap/rowset/alpha_rowset.h @@ -124,7 +124,7 @@ class AlphaRowset : public Rowset { // info by using segment's info OLAPStatus reset_sizeinfo(); - uint32_t unique_id() override; + std::string unique_id() override; private: OLAPStatus _init_segment_groups(); diff --git a/be/src/olap/rowset/rowset.h b/be/src/olap/rowset/rowset.h index 0e0c934603b9fc..9eeb36fcf7f388 100644 --- a/be/src/olap/rowset/rowset.h +++ b/be/src/olap/rowset/rowset.h @@ -122,7 +122,7 @@ class Rowset : public std::enable_shared_from_this { virtual bool check_path(const std::string& path) = 0; - virtual uint32_t unique_id() = 0; + virtual std::string unique_id() = 0; bool need_delete_file() { return _need_delete_file; diff --git a/be/src/olap/storage_engine.cpp b/be/src/olap/storage_engine.cpp index 19bc82c89400e5..f4871bd4db92a8 100644 --- a/be/src/olap/storage_engine.cpp +++ b/be/src/olap/storage_engine.cpp @@ -762,8 +762,9 @@ void StorageEngine::add_unused_rowset(RowsetSharedPtr rowset) { if (rowset == nullptr) { return; } _gc_mutex.lock(); LOG(INFO) << "add unused rowset, rowset id:" << rowset->rowset_id() - << "version:" << rowset->version().first - << "-" << rowset->version().second; + << ", version:" << rowset->version().first + << "-" << rowset->version().second + << ", unique id:" << rowset->unique_id(); auto it = _unused_rowsets.find(rowset->unique_id()); if (it == _unused_rowsets.end()) { rowset->set_need_delete_file(true); diff --git a/be/src/olap/storage_engine.h b/be/src/olap/storage_engine.h index ff770d359392c1..fc5c1475401f95 100644 --- a/be/src/olap/storage_engine.h +++ b/be/src/olap/storage_engine.h @@ -298,7 +298,7 @@ class StorageEngine { static StorageEngine* _s_instance; std::unordered_map> _gc_files; - std::unordered_map _unused_rowsets; + std::unordered_map _unused_rowsets; Mutex _gc_mutex; std::thread _unused_rowset_monitor_thread;