From e1d3f2dbe04db91338949f4b8e517f4f8990ee46 Mon Sep 17 00:00:00 2001 From: chenlinzhong <490103404@qq.com> Date: Wed, 5 Apr 2023 11:05:44 +0800 Subject: [PATCH 1/5] [bugfix] Fix the issue of incorrect disk usage --- be/src/io/fs/file_system.cpp | 20 ++++++++++++++++++++ be/src/io/fs/file_system.h | 2 ++ be/src/olap/data_dir.cpp | 6 +++--- be/src/olap/rowset/rowset_meta_manager.cpp | 11 +++++++++++ be/src/olap/rowset/rowset_meta_manager.h | 1 + be/src/olap/tablet.cpp | 2 +- 6 files changed, 38 insertions(+), 4 deletions(-) diff --git a/be/src/io/fs/file_system.cpp b/be/src/io/fs/file_system.cpp index 1f34500035d69d..70cd9e003752d1 100644 --- a/be/src/io/fs/file_system.cpp +++ b/be/src/io/fs/file_system.cpp @@ -17,6 +17,8 @@ #include "io/fs/file_system.h" +#include + #include "olap/olap_define.h" #include "util/async_io.h" @@ -44,6 +46,24 @@ Status FileSystem::delete_file(const Path& file) { FILESYSTEM_M(delete_file_impl(path)); } +Status FileSystem::delete_directory_or_file(const Path& path) { + auto real_path = absolute_path(path); + if (is_dir(path)) { + FILESYSTEM_M(delete_directory_impl(path)); + } else { + FILESYSTEM_M(delete_file_impl(path)); + } +} + +bool FileSystem::is_dir(const Path& path) { + struct stat path_stat; + if (stat(path.c_str(), &path_stat) != 0) { + return false; + } else { + return S_ISDIR(path_stat.st_mode); + } +} + Status FileSystem::delete_directory(const Path& dir) { auto path = absolute_path(dir); FILESYSTEM_M(delete_directory_impl(path)); diff --git a/be/src/io/fs/file_system.h b/be/src/io/fs/file_system.h index 3dbf741a05a284..248b31833b7fd5 100644 --- a/be/src/io/fs/file_system.h +++ b/be/src/io/fs/file_system.h @@ -79,6 +79,8 @@ class FileSystem : public std::enable_shared_from_this { Status list(const Path& dir, bool only_file, std::vector* files, bool* exists); Status rename(const Path& orig_name, const Path& new_name); Status rename_dir(const Path& orig_name, const Path& new_name); + bool is_dir(const Path& path); + Status delete_directory_or_file(const Path& path); std::shared_ptr getSPtr() { return shared_from_this(); } diff --git a/be/src/olap/data_dir.cpp b/be/src/olap/data_dir.cpp index 3fe2dcd7cc6dd0..59a03d5471481c 100644 --- a/be/src/olap/data_dir.cpp +++ b/be/src/olap/data_dir.cpp @@ -723,9 +723,9 @@ void DataDir::_process_garbage_path(const std::string& path) { return; } if (exists) { - LOG(INFO) << "collect garbage dir path: " << path; - WARN_IF_ERROR(io::global_local_filesystem()->delete_directory(path), - "remove garbage dir failed"); + LOG(INFO) << "collect garbage path: " << path; + WARN_IF_ERROR(io::global_local_filesystem()->delete_directory_or_file(path), + "remove garbage failed"); } } diff --git a/be/src/olap/rowset/rowset_meta_manager.cpp b/be/src/olap/rowset/rowset_meta_manager.cpp index 7a58fa9cb64caf..d4d2e67855c590 100644 --- a/be/src/olap/rowset/rowset_meta_manager.cpp +++ b/be/src/olap/rowset/rowset_meta_manager.cpp @@ -42,6 +42,17 @@ bool RowsetMetaManager::check_rowset_meta(OlapMeta* meta, TabletUid tablet_uid, return meta->key_may_exist(META_COLUMN_FAMILY_INDEX, key, &value); } +bool RowsetMetaManager::exists(OlapMeta* meta, TabletUid tablet_uid, const RowsetId& rowset_id) { + std::string key = ROWSET_PREFIX + tablet_uid.to_string() + "_" + rowset_id.to_string(); + std::string value; + Status s = meta->get(META_COLUMN_FAMILY_INDEX, key, &value); + if (s.is()) { + return false; + } else { + return true; + } +} + Status RowsetMetaManager::get_rowset_meta(OlapMeta* meta, TabletUid tablet_uid, const RowsetId& rowset_id, RowsetMetaSharedPtr rowset_meta) { diff --git a/be/src/olap/rowset/rowset_meta_manager.h b/be/src/olap/rowset/rowset_meta_manager.h index 8c8f3144e03dbe..e4f83b55ac0a73 100644 --- a/be/src/olap/rowset/rowset_meta_manager.h +++ b/be/src/olap/rowset/rowset_meta_manager.h @@ -31,6 +31,7 @@ namespace doris { class RowsetMetaManager { public: static bool check_rowset_meta(OlapMeta* meta, TabletUid tablet_uid, const RowsetId& rowset_id); + static bool exists(OlapMeta* meta, TabletUid tablet_uid, const RowsetId& rowset_id); static Status get_rowset_meta(OlapMeta* meta, TabletUid tablet_uid, const RowsetId& rowset_id, RowsetMetaSharedPtr rowset_meta); diff --git a/be/src/olap/tablet.cpp b/be/src/olap/tablet.cpp index f70bc7f4c76ecd..e861c3b2cb5250 100644 --- a/be/src/olap/tablet.cpp +++ b/be/src/olap/tablet.cpp @@ -1160,7 +1160,7 @@ bool Tablet::check_rowset_id(const RowsetId& rowset_id) { return true; } } - if (RowsetMetaManager::check_rowset_meta(_data_dir->get_meta(), tablet_uid(), rowset_id)) { + if (RowsetMetaManager::exists(_data_dir->get_meta(), tablet_uid(), rowset_id)) { return true; } return false; From d079b476804f620a194f3ba96aadeff341cae079 Mon Sep 17 00:00:00 2001 From: chenlinzhong <490103404@qq.com> Date: Fri, 7 Apr 2023 19:57:33 +0800 Subject: [PATCH 2/5] [bug](GC)the issue of incorrect disk usage --- be/src/io/fs/file_system.cpp | 9 --------- be/src/io/fs/file_system.h | 1 - be/src/io/fs/local_file_system.cpp | 16 ++++++++++++++++ be/src/io/fs/local_file_system.h | 2 ++ be/src/olap/rowset/rowset_meta_manager.cpp | 8 ++------ be/src/olap/rowset/rowset_meta_manager.h | 2 +- be/src/olap/tablet.cpp | 3 ++- 7 files changed, 23 insertions(+), 18 deletions(-) diff --git a/be/src/io/fs/file_system.cpp b/be/src/io/fs/file_system.cpp index 70cd9e003752d1..248c9647d64ae1 100644 --- a/be/src/io/fs/file_system.cpp +++ b/be/src/io/fs/file_system.cpp @@ -46,15 +46,6 @@ Status FileSystem::delete_file(const Path& file) { FILESYSTEM_M(delete_file_impl(path)); } -Status FileSystem::delete_directory_or_file(const Path& path) { - auto real_path = absolute_path(path); - if (is_dir(path)) { - FILESYSTEM_M(delete_directory_impl(path)); - } else { - FILESYSTEM_M(delete_file_impl(path)); - } -} - bool FileSystem::is_dir(const Path& path) { struct stat path_stat; if (stat(path.c_str(), &path_stat) != 0) { diff --git a/be/src/io/fs/file_system.h b/be/src/io/fs/file_system.h index 248b31833b7fd5..1aa26789305fa5 100644 --- a/be/src/io/fs/file_system.h +++ b/be/src/io/fs/file_system.h @@ -80,7 +80,6 @@ class FileSystem : public std::enable_shared_from_this { Status rename(const Path& orig_name, const Path& new_name); Status rename_dir(const Path& orig_name, const Path& new_name); bool is_dir(const Path& path); - Status delete_directory_or_file(const Path& path); std::shared_ptr getSPtr() { return shared_from_this(); } diff --git a/be/src/io/fs/local_file_system.cpp b/be/src/io/fs/local_file_system.cpp index c82295ae277337..87faa92b86668e 100644 --- a/be/src/io/fs/local_file_system.cpp +++ b/be/src/io/fs/local_file_system.cpp @@ -115,6 +115,22 @@ Status LocalFileSystem::delete_directory_impl(const Path& dir) { return Status::OK(); } +Status LocalFileSystem::delete_directory_or_file(const Path& path) { + bool is_dir; + Status ret = is_directory(path, &is_dir); + if (ret.ok()) { + Status s; + if (is_dir) { + s = delete_directory_impl(path); + } else { + s = delete_file_impl(path); + } + return s; + } else { + return ret; + } +} + Status LocalFileSystem::batch_delete_impl(const std::vector& files) { for (auto& file : files) { RETURN_IF_ERROR(delete_file_impl(file)); diff --git a/be/src/io/fs/local_file_system.h b/be/src/io/fs/local_file_system.h index 308cd5e1fcd9e2..9b235a3a758ba3 100644 --- a/be/src/io/fs/local_file_system.h +++ b/be/src/io/fs/local_file_system.h @@ -56,6 +56,8 @@ class LocalFileSystem final : public FileSystem { Status copy_dirs(const Path& src, const Path& dest); // return true if parent path contain sub path static bool contain_path(const Path& parent, const Path& sub); + // delete dir or file + Status delete_directory_or_file(const Path& path); protected: Status create_file_impl(const Path& file, FileWriterPtr* writer) override; diff --git a/be/src/olap/rowset/rowset_meta_manager.cpp b/be/src/olap/rowset/rowset_meta_manager.cpp index d4d2e67855c590..ac9ae5764e9c5b 100644 --- a/be/src/olap/rowset/rowset_meta_manager.cpp +++ b/be/src/olap/rowset/rowset_meta_manager.cpp @@ -42,15 +42,11 @@ bool RowsetMetaManager::check_rowset_meta(OlapMeta* meta, TabletUid tablet_uid, return meta->key_may_exist(META_COLUMN_FAMILY_INDEX, key, &value); } -bool RowsetMetaManager::exists(OlapMeta* meta, TabletUid tablet_uid, const RowsetId& rowset_id) { +Status RowsetMetaManager::exists(OlapMeta* meta, TabletUid tablet_uid, const RowsetId& rowset_id) { std::string key = ROWSET_PREFIX + tablet_uid.to_string() + "_" + rowset_id.to_string(); std::string value; Status s = meta->get(META_COLUMN_FAMILY_INDEX, key, &value); - if (s.is()) { - return false; - } else { - return true; - } + return s; } Status RowsetMetaManager::get_rowset_meta(OlapMeta* meta, TabletUid tablet_uid, diff --git a/be/src/olap/rowset/rowset_meta_manager.h b/be/src/olap/rowset/rowset_meta_manager.h index e4f83b55ac0a73..cca2304ce3c50c 100644 --- a/be/src/olap/rowset/rowset_meta_manager.h +++ b/be/src/olap/rowset/rowset_meta_manager.h @@ -31,7 +31,7 @@ namespace doris { class RowsetMetaManager { public: static bool check_rowset_meta(OlapMeta* meta, TabletUid tablet_uid, const RowsetId& rowset_id); - static bool exists(OlapMeta* meta, TabletUid tablet_uid, const RowsetId& rowset_id); + static Status exists(OlapMeta* meta, TabletUid tablet_uid, const RowsetId& rowset_id); static Status get_rowset_meta(OlapMeta* meta, TabletUid tablet_uid, const RowsetId& rowset_id, RowsetMetaSharedPtr rowset_meta); diff --git a/be/src/olap/tablet.cpp b/be/src/olap/tablet.cpp index e861c3b2cb5250..02fbd64ec0ef03 100644 --- a/be/src/olap/tablet.cpp +++ b/be/src/olap/tablet.cpp @@ -1160,7 +1160,8 @@ bool Tablet::check_rowset_id(const RowsetId& rowset_id) { return true; } } - if (RowsetMetaManager::exists(_data_dir->get_meta(), tablet_uid(), rowset_id)) { + Status s = RowsetMetaManager::exists(_data_dir->get_meta(), tablet_uid(), rowset_id); + if (!s.is()) { return true; } return false; From 45c3694fb018150f3843d64a1725a30fda384f6f Mon Sep 17 00:00:00 2001 From: chenlinzhong <490103404@qq.com> Date: Fri, 7 Apr 2023 19:58:39 +0800 Subject: [PATCH 3/5] [bug](GC)the issue of incorrect disk usage --- be/src/io/fs/file_system.cpp | 9 --------- be/src/io/fs/file_system.h | 1 - 2 files changed, 10 deletions(-) diff --git a/be/src/io/fs/file_system.cpp b/be/src/io/fs/file_system.cpp index 248c9647d64ae1..0b51a5ab97ff66 100644 --- a/be/src/io/fs/file_system.cpp +++ b/be/src/io/fs/file_system.cpp @@ -46,15 +46,6 @@ Status FileSystem::delete_file(const Path& file) { FILESYSTEM_M(delete_file_impl(path)); } -bool FileSystem::is_dir(const Path& path) { - struct stat path_stat; - if (stat(path.c_str(), &path_stat) != 0) { - return false; - } else { - return S_ISDIR(path_stat.st_mode); - } -} - Status FileSystem::delete_directory(const Path& dir) { auto path = absolute_path(dir); FILESYSTEM_M(delete_directory_impl(path)); diff --git a/be/src/io/fs/file_system.h b/be/src/io/fs/file_system.h index 1aa26789305fa5..3dbf741a05a284 100644 --- a/be/src/io/fs/file_system.h +++ b/be/src/io/fs/file_system.h @@ -79,7 +79,6 @@ class FileSystem : public std::enable_shared_from_this { Status list(const Path& dir, bool only_file, std::vector* files, bool* exists); Status rename(const Path& orig_name, const Path& new_name); Status rename_dir(const Path& orig_name, const Path& new_name); - bool is_dir(const Path& path); std::shared_ptr getSPtr() { return shared_from_this(); } From 11db9e49c54173fb074157c68fe2f7b1b8f7bf6d Mon Sep 17 00:00:00 2001 From: chenlinzhong <490103404@qq.com> Date: Fri, 7 Apr 2023 19:59:38 +0800 Subject: [PATCH 4/5] [bug](GC)the issue of incorrect disk usage --- be/src/io/fs/file_system.cpp | 3 --- 1 file changed, 3 deletions(-) diff --git a/be/src/io/fs/file_system.cpp b/be/src/io/fs/file_system.cpp index 0b51a5ab97ff66..4770b38feb054b 100644 --- a/be/src/io/fs/file_system.cpp +++ b/be/src/io/fs/file_system.cpp @@ -16,9 +16,6 @@ // under the License. #include "io/fs/file_system.h" - -#include - #include "olap/olap_define.h" #include "util/async_io.h" From 23c68903491797090ebdb713fad269a53fa775f1 Mon Sep 17 00:00:00 2001 From: chenlinzhong <490103404@qq.com> Date: Fri, 7 Apr 2023 20:00:04 +0800 Subject: [PATCH 5/5] [bug](GC)the issue of incorrect disk usage --- be/src/io/fs/file_system.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/be/src/io/fs/file_system.cpp b/be/src/io/fs/file_system.cpp index 4770b38feb054b..1f34500035d69d 100644 --- a/be/src/io/fs/file_system.cpp +++ b/be/src/io/fs/file_system.cpp @@ -16,6 +16,7 @@ // under the License. #include "io/fs/file_system.h" + #include "olap/olap_define.h" #include "util/async_io.h"