From f73e1ab98e617984c9ebef5045f86caaab1d3e2b Mon Sep 17 00:00:00 2001 From: Yukang-Lian Date: Fri, 3 Jan 2025 19:57:25 +0800 Subject: [PATCH 1/6] 1 --- cloud/src/recycler/hdfs_accessor.cpp | 29 ++++- cloud/test/hdfs_accessor_test.cpp | 177 +++++++++++++++++++-------- 2 files changed, 152 insertions(+), 54 deletions(-) diff --git a/cloud/src/recycler/hdfs_accessor.cpp b/cloud/src/recycler/hdfs_accessor.cpp index 97a4670d2bfc6b..ffbc5788486b64 100644 --- a/cloud/src/recycler/hdfs_accessor.cpp +++ b/cloud/src/recycler/hdfs_accessor.cpp @@ -20,6 +20,8 @@ #include #include +#include + #include "common/stopwatch.h" #include "recycler/util.h" @@ -342,6 +344,19 @@ std::string HdfsAccessor::to_uri(const std::string& relative_path) { return uri_ + '/' + relative_path; } +// extract table path from prefix +// e.g. +// data/492211/02000000008a012957476a3e174dfdaa71ee5f80a3abafa3_ -> data/492211/ +std::string extract_tablet_path(const std::string& path) { + // Find the last '/' + size_t last_slash = path.find_last_of('/'); + if (last_slash == std::string::npos) { + LOG_WARNING("no '/' found in path").tag("path", path); + return ""; + } + return path.substr(0, last_slash + 1); +} + int HdfsAccessor::init() { // TODO(plat1ko): Cache hdfsFS fs_ = HDFSBuilder::create_fs(info_.build_conf()); @@ -357,7 +372,15 @@ int HdfsAccessor::delete_prefix(const std::string& path_prefix, int64_t expirati auto uri = to_uri(path_prefix); LOG(INFO) << "delete prefix, uri=" << uri; std::unique_ptr list_iter; - int ret = list_all(&list_iter); + auto tablet_path = extract_tablet_path(path_prefix); + if (tablet_path.empty()) { + LOG_WARNING("extract tablet path failed").tag("path prefix", path_prefix); + return -1; + } + LOG_INFO("extract tablet path success") + .tag("path prefix", path_prefix) + .tag("tablet path", tablet_path); + int ret = list_directory(tablet_path, &list_iter); if (ret != 0) { LOG(WARNING) << "delete prefix, failed to list" << uri; return ret; @@ -372,6 +395,10 @@ int HdfsAccessor::delete_prefix(const std::string& path_prefix, int64_t expirati } ++num_deleted; } + if (num_deleted == 0) { + LOG_WARNING("recycler delete prefix num = 0, maybe there are some problems?") + .tag("path prefix", path_prefix); + } LOG(INFO) << "delete prefix " << (ret != 0 ? "failed" : "succ") << " ret=" << ret << " uri=" << uri << " num_listed=" << num_listed << " num_deleted=" << num_deleted; return ret; diff --git a/cloud/test/hdfs_accessor_test.cpp b/cloud/test/hdfs_accessor_test.cpp index cabd07f8c9a051..d0ea8a4ae0517e 100644 --- a/cloud/test/hdfs_accessor_test.cpp +++ b/cloud/test/hdfs_accessor_test.cpp @@ -22,6 +22,7 @@ #include #include +#include #include "common/config.h" #include "common/logging.h" @@ -59,20 +60,20 @@ TEST(HdfsAccessorTest, normal) { HdfsAccessor accessor(info); int ret = accessor.init(); - ASSERT_EQ(ret, 0); + EXPECT_EQ(ret, 0); std::string file1 = "data/10000/1_0.dat"; ret = accessor.delete_directory(""); - ASSERT_NE(ret, 0); + EXPECT_NE(ret, 0); ret = accessor.delete_all(); - ASSERT_EQ(ret, 0); + EXPECT_EQ(ret, 0); ret = accessor.put_file(file1, ""); - ASSERT_EQ(ret, 0); + EXPECT_EQ(ret, 0); ret = accessor.exists(file1); - ASSERT_EQ(ret, 0); + EXPECT_EQ(ret, 0); auto* sp = SyncPoint::get_instance(); sp->enable_processing(); @@ -90,46 +91,46 @@ TEST(HdfsAccessorTest, normal) { std::unique_ptr iter; ret = accessor.list_directory("data", &iter); - ASSERT_EQ(ret, 0); - ASSERT_TRUE(iter); - ASSERT_TRUE(iter->is_valid()); - ASSERT_TRUE(iter->has_next()); - ASSERT_EQ(iter->next()->path, file1); - ASSERT_FALSE(iter->has_next()); + EXPECT_EQ(ret, 0); + EXPECT_TRUE(iter); + EXPECT_TRUE(iter->is_valid()); + EXPECT_TRUE(iter->has_next()); + EXPECT_EQ(iter->next()->path, file1); + EXPECT_FALSE(iter->has_next()); iter.reset(); - ASSERT_EQ(alloc_entries, 0); + EXPECT_EQ(alloc_entries, 0); ret = accessor.list_directory("data/", &iter); - ASSERT_EQ(ret, 0); - ASSERT_TRUE(iter->is_valid()); - ASSERT_TRUE(iter->has_next()); - ASSERT_EQ(iter->next()->path, file1); - ASSERT_FALSE(iter->has_next()); - ASSERT_FALSE(iter->next()); + EXPECT_EQ(ret, 0); + EXPECT_TRUE(iter->is_valid()); + EXPECT_TRUE(iter->has_next()); + EXPECT_EQ(iter->next()->path, file1); + EXPECT_FALSE(iter->has_next()); + EXPECT_FALSE(iter->next()); iter.reset(); - ASSERT_EQ(alloc_entries, 0); + EXPECT_EQ(alloc_entries, 0); ret = accessor.list_directory("data/100", &iter); - ASSERT_EQ(ret, 0); - ASSERT_FALSE(iter->has_next()); - ASSERT_FALSE(iter->next()); + EXPECT_EQ(ret, 0); + EXPECT_FALSE(iter->has_next()); + EXPECT_FALSE(iter->next()); iter.reset(); - ASSERT_EQ(alloc_entries, 0); + EXPECT_EQ(alloc_entries, 0); ret = accessor.delete_file(file1); - ASSERT_EQ(ret, 0); + EXPECT_EQ(ret, 0); ret = accessor.exists(file1); - ASSERT_EQ(ret, 1); + EXPECT_EQ(ret, 1); ret = accessor.list_directory("", &iter); - ASSERT_NE(ret, 0); + EXPECT_NE(ret, 0); ret = accessor.list_all(&iter); - ASSERT_EQ(ret, 0); - ASSERT_FALSE(iter->has_next()); - ASSERT_FALSE(iter->next()); + EXPECT_EQ(ret, 0); + EXPECT_FALSE(iter->has_next()); + EXPECT_FALSE(iter->next()); iter.reset(); - ASSERT_EQ(alloc_entries, 0); + EXPECT_EQ(alloc_entries, 0); ret = accessor.delete_file(file1); - ASSERT_EQ(ret, 0); + EXPECT_EQ(ret, 0); std::vector files; for (int dir = 10000; dir < 10005; ++dir) { @@ -140,18 +141,18 @@ TEST(HdfsAccessorTest, normal) { for (auto&& file : files) { ret = accessor.put_file(file, ""); - ASSERT_EQ(ret, 0); + EXPECT_EQ(ret, 0); } std::unordered_set list_files; ret = accessor.list_all(&iter); - ASSERT_EQ(ret, 0); + EXPECT_EQ(ret, 0); for (auto file = iter->next(); file.has_value(); file = iter->next()) { list_files.insert(std::move(file->path)); } iter.reset(); - ASSERT_EQ(alloc_entries, 0); - ASSERT_EQ(list_files.size(), files.size()); + EXPECT_EQ(alloc_entries, 0); + EXPECT_EQ(list_files.size(), files.size()); for (auto&& file : files) { EXPECT_TRUE(list_files.contains(file)); } @@ -163,69 +164,139 @@ TEST(HdfsAccessorTest, normal) { files.pop_back(); } ret = accessor.delete_files(to_delete_files); - ASSERT_EQ(ret, 0); + EXPECT_EQ(ret, 0); ret = accessor.list_all(&iter); - ASSERT_EQ(ret, 0); + EXPECT_EQ(ret, 0); list_files.clear(); for (auto file = iter->next(); file.has_value(); file = iter->next()) { list_files.insert(std::move(file->path)); } iter.reset(); - ASSERT_EQ(alloc_entries, 0); - ASSERT_EQ(list_files.size(), files.size()); + EXPECT_EQ(alloc_entries, 0); + EXPECT_EQ(list_files.size(), files.size()); for (auto&& file : files) { EXPECT_TRUE(list_files.contains(file)); } std::string to_delete_dir = "data/10001"; ret = accessor.delete_directory(to_delete_dir); - ASSERT_EQ(ret, 0); + EXPECT_EQ(ret, 0); ret = accessor.list_directory(to_delete_dir, &iter); - ASSERT_EQ(ret, 0); - ASSERT_FALSE(iter->has_next()); + EXPECT_EQ(ret, 0); + EXPECT_FALSE(iter->has_next()); files.erase(std::remove_if(files.begin(), files.end(), [&](auto&& file) { return file.starts_with(to_delete_dir); }), files.end()); ret = accessor.list_all(&iter); - ASSERT_EQ(ret, 0); + EXPECT_EQ(ret, 0); list_files.clear(); for (auto file = iter->next(); file.has_value(); file = iter->next()) { list_files.insert(std::move(file->path)); } iter.reset(); - ASSERT_EQ(alloc_entries, 0); - ASSERT_EQ(list_files.size(), files.size()); + EXPECT_EQ(alloc_entries, 0); + EXPECT_EQ(list_files.size(), files.size()); for (auto&& file : files) { EXPECT_TRUE(list_files.contains(file)); } std::string to_delete_prefix = "data/10003/"; ret = accessor.delete_directory(to_delete_prefix); - ASSERT_EQ(ret, 0); + EXPECT_EQ(ret, 0); files.erase(std::remove_if(files.begin(), files.end(), [&](auto&& file) { return file.starts_with(to_delete_prefix); }), files.end()); ret = accessor.list_all(&iter); - ASSERT_EQ(ret, 0); + EXPECT_EQ(ret, 0); list_files.clear(); for (auto file = iter->next(); file.has_value(); file = iter->next()) { list_files.insert(std::move(file->path)); } iter.reset(); - ASSERT_EQ(alloc_entries, 0); - ASSERT_EQ(list_files.size(), files.size()); + EXPECT_EQ(alloc_entries, 0); + EXPECT_EQ(list_files.size(), files.size()); for (auto&& file : files) { EXPECT_TRUE(list_files.contains(file)); } ret = accessor.delete_all(); - ASSERT_EQ(ret, 0); + EXPECT_EQ(ret, 0); ret = accessor.list_all(&iter); - ASSERT_EQ(ret, 0); - ASSERT_FALSE(iter->has_next()); - ASSERT_FALSE(iter->next()); + EXPECT_EQ(ret, 0); + EXPECT_FALSE(iter->has_next()); + EXPECT_FALSE(iter->next()); +} + +TEST(HdfsAccessorTest, delete_prefix) { + HdfsVaultInfo info; + info.set_prefix(config::test_hdfs_prefix + "/HdfsAccessorTest/" + butil::GenerateGUID()); + auto* conf = info.mutable_build_conf(); + conf->set_fs_name(config::test_hdfs_fs_name); + + HdfsAccessor accessor(info); + int ret = accessor.init(); + EXPECT_EQ(ret, 0); + + auto put_and_verify = [&accessor](const std::string& file) { + int ret = accessor.put_file(file, ""); + EXPECT_EQ(ret, 0); + ret = accessor.exists(file); + EXPECT_EQ(ret, 0); + }; + + ret = accessor.delete_directory(""); + EXPECT_NE(ret, 0); + ret = accessor.delete_all(); + EXPECT_EQ(ret, 0); + + put_and_verify("data/10000/1_0.dat"); + put_and_verify("data/10000/2_0.dat"); + put_and_verify("data/10000/20000/1_0.dat"); + put_and_verify("data/10000/20000/30000/1_0.dat"); + put_and_verify("data/20000/1_0.dat"); + put_and_verify("data111/10000/1_0.dat"); + + ret = accessor.delete_prefix("data/10000/1_"); + EXPECT_EQ(ret, 0); + ret = accessor.delete_prefix("data/10000/2_"); + EXPECT_EQ(ret, 0); + + std::unordered_set list_files; + std::unique_ptr iter; + ret = accessor.list_all(&iter); + EXPECT_EQ(ret, 0); + list_files.clear(); + for (auto file = iter->next(); file.has_value(); file = iter->next()) { + list_files.insert(std::move(file->path)); + } + EXPECT_EQ(list_files.size(), 4); + EXPECT_TRUE(list_files.contains("data/10000/20000/1_0.dat")); + EXPECT_TRUE(list_files.contains("data/10000/20000/30000/1_0.dat")); + EXPECT_TRUE(list_files.contains("data/20000/1_0.dat")); + EXPECT_TRUE(list_files.contains("data111/10000/1_0.dat")); + + ret = accessor.delete_prefix("data/10000"); + EXPECT_EQ(ret, -1); + ret = accessor.delete_prefix("data111/10000"); + EXPECT_EQ(ret, -1); + ret = accessor.delete_prefix("data/20000/1_"); + EXPECT_EQ(ret, 0); + ret = accessor.delete_prefix("data/10000/1"); + EXPECT_EQ(ret, -1); + + iter.reset(); + ret = accessor.list_all(&iter); + EXPECT_EQ(ret, 0); + list_files.clear(); + for (auto file = iter->next(); file.has_value(); file = iter->next()) { + list_files.insert(std::move(file->path)); + } + EXPECT_EQ(list_files.size(), 3); + EXPECT_TRUE(list_files.contains("data/10000/20000/1_0.dat")); + EXPECT_TRUE(list_files.contains("data/10000/20000/30000/1_0.dat")); + EXPECT_TRUE(list_files.contains("data111/10000/1_0.dat")); } } // namespace doris::cloud From 52edc3921027919c5a9ed3b7a0cead9a3f59a629 Mon Sep 17 00:00:00 2001 From: Yukang-Lian Date: Thu, 9 Jan 2025 20:18:10 +0800 Subject: [PATCH 2/6] 2 --- cloud/src/recycler/hdfs_accessor.cpp | 8 ++++++++ cloud/test/hdfs_accessor_test.cpp | 12 +++++------- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/cloud/src/recycler/hdfs_accessor.cpp b/cloud/src/recycler/hdfs_accessor.cpp index ffbc5788486b64..5394c39c12245e 100644 --- a/cloud/src/recycler/hdfs_accessor.cpp +++ b/cloud/src/recycler/hdfs_accessor.cpp @@ -368,6 +368,14 @@ int HdfsAccessor::init() { return 0; } + // Currently, the hdfs accessor deletes files on hdfs through a prefix parameter. + // The format of the prefix parameter is data/{tablet id}/{rowset_prefix}, + // for example: data/492211/02000000008a012957476a3e174dfdaa71ee5f80a3abafa3_. + // Since the hdfs cpp sdk doesn't provide an interface for deleting by prefix, + // we need to extract the tablet id path from the given prefix, + // traverse all files in the tablet id path, and delete the files that match the prefix. + // This implementation is not ideal because the hdfs accessor needs to be aware of the path structure. + // We will optimize this in the future. int HdfsAccessor::delete_prefix(const std::string& path_prefix, int64_t expiration_time) { auto uri = to_uri(path_prefix); LOG(INFO) << "delete prefix, uri=" << uri; diff --git a/cloud/test/hdfs_accessor_test.cpp b/cloud/test/hdfs_accessor_test.cpp index d0ea8a4ae0517e..72188e4c339a24 100644 --- a/cloud/test/hdfs_accessor_test.cpp +++ b/cloud/test/hdfs_accessor_test.cpp @@ -277,14 +277,12 @@ TEST(HdfsAccessorTest, delete_prefix) { EXPECT_TRUE(list_files.contains("data/20000/1_0.dat")); EXPECT_TRUE(list_files.contains("data111/10000/1_0.dat")); - ret = accessor.delete_prefix("data/10000"); - EXPECT_EQ(ret, -1); - ret = accessor.delete_prefix("data111/10000"); - EXPECT_EQ(ret, -1); + ret = accessor.delete_prefix("data/10000/1_"); + EXPECT_EQ(ret, 0); + ret = accessor.delete_prefix("data/10000/2_"); + EXPECT_EQ(ret, 0); ret = accessor.delete_prefix("data/20000/1_"); EXPECT_EQ(ret, 0); - ret = accessor.delete_prefix("data/10000/1"); - EXPECT_EQ(ret, -1); iter.reset(); ret = accessor.list_all(&iter); @@ -294,8 +292,8 @@ TEST(HdfsAccessorTest, delete_prefix) { list_files.insert(std::move(file->path)); } EXPECT_EQ(list_files.size(), 3); - EXPECT_TRUE(list_files.contains("data/10000/20000/1_0.dat")); EXPECT_TRUE(list_files.contains("data/10000/20000/30000/1_0.dat")); + EXPECT_TRUE(list_files.contains("data/10000/20000/1_0.dat")); EXPECT_TRUE(list_files.contains("data111/10000/1_0.dat")); } From 1c7c95a8c348bc053cb718df2cf2a49f70684514 Mon Sep 17 00:00:00 2001 From: Yukang-Lian Date: Thu, 9 Jan 2025 20:19:36 +0800 Subject: [PATCH 3/6] 3 --- cloud/src/recycler/hdfs_accessor.cpp | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/cloud/src/recycler/hdfs_accessor.cpp b/cloud/src/recycler/hdfs_accessor.cpp index 5394c39c12245e..596465975b03bb 100644 --- a/cloud/src/recycler/hdfs_accessor.cpp +++ b/cloud/src/recycler/hdfs_accessor.cpp @@ -368,14 +368,14 @@ int HdfsAccessor::init() { return 0; } - // Currently, the hdfs accessor deletes files on hdfs through a prefix parameter. - // The format of the prefix parameter is data/{tablet id}/{rowset_prefix}, - // for example: data/492211/02000000008a012957476a3e174dfdaa71ee5f80a3abafa3_. - // Since the hdfs cpp sdk doesn't provide an interface for deleting by prefix, - // we need to extract the tablet id path from the given prefix, - // traverse all files in the tablet id path, and delete the files that match the prefix. - // This implementation is not ideal because the hdfs accessor needs to be aware of the path structure. - // We will optimize this in the future. +// Currently, the hdfs accessor deletes files on hdfs through a prefix parameter. +// The format of the prefix parameter is data/{tablet id}/{rowset_prefix}, +// for example: data/492211/02000000008a012957476a3e174dfdaa71ee5f80a3abafa3_. +// Since the hdfs cpp sdk doesn't provide an interface for deleting by prefix, +// we need to extract the tablet id path from the given prefix, +// traverse all files in the tablet id path, and delete the files that match the prefix. +// This implementation is not ideal because the hdfs accessor needs to be aware of the path structure. +// We will optimize this in the future. int HdfsAccessor::delete_prefix(const std::string& path_prefix, int64_t expiration_time) { auto uri = to_uri(path_prefix); LOG(INFO) << "delete prefix, uri=" << uri; From aeff74ab825c7db281bdab1c59a2a215e310a03d Mon Sep 17 00:00:00 2001 From: Yukang-Lian Date: Fri, 10 Jan 2025 18:07:27 +0800 Subject: [PATCH 4/6] 4 --- cloud/src/recycler/hdfs_accessor.cpp | 39 ++++++++++++++++------------ cloud/test/hdfs_accessor_test.cpp | 25 ++++++++++++++++++ 2 files changed, 48 insertions(+), 16 deletions(-) diff --git a/cloud/src/recycler/hdfs_accessor.cpp b/cloud/src/recycler/hdfs_accessor.cpp index 596465975b03bb..98c35dace9d460 100644 --- a/cloud/src/recycler/hdfs_accessor.cpp +++ b/cloud/src/recycler/hdfs_accessor.cpp @@ -344,10 +344,10 @@ std::string HdfsAccessor::to_uri(const std::string& relative_path) { return uri_ + '/' + relative_path; } -// extract table path from prefix +// extract parent path from prefix // e.g. // data/492211/02000000008a012957476a3e174dfdaa71ee5f80a3abafa3_ -> data/492211/ -std::string extract_tablet_path(const std::string& path) { +std::string extract_parent_path(const std::string& path) { // Find the last '/' size_t last_slash = path.find_last_of('/'); if (last_slash == std::string::npos) { @@ -368,27 +368,33 @@ int HdfsAccessor::init() { return 0; } -// Currently, the hdfs accessor deletes files on hdfs through a prefix parameter. -// The format of the prefix parameter is data/{tablet id}/{rowset_prefix}, -// for example: data/492211/02000000008a012957476a3e174dfdaa71ee5f80a3abafa3_. -// Since the hdfs cpp sdk doesn't provide an interface for deleting by prefix, -// we need to extract the tablet id path from the given prefix, -// traverse all files in the tablet id path, and delete the files that match the prefix. -// This implementation is not ideal because the hdfs accessor needs to be aware of the path structure. -// We will optimize this in the future. int HdfsAccessor::delete_prefix(const std::string& path_prefix, int64_t expiration_time) { auto uri = to_uri(path_prefix); LOG(INFO) << "delete prefix, uri=" << uri; + // If path prefix exists, assume it is a dir or a file. + if (exists(path_prefix) == 0) { + // try to delete path prefix as a dir or a file. + if (delete_directory(path_prefix) == 0) { + LOG(INFO) << "delete prefix succ" + << ", is dir or file = true" + << ", uri=" << uri; + return 0; + } + } + // If path prefix is not a dir or a file, + // for example: data/492211/02000000008a012957476a3e174dfdaa71ee5f80a3abafa3_. + // Then we need to extract the parent id path from the given prefix, + // traverse all files in the parent id path, and delete the files that match the prefix. std::unique_ptr list_iter; - auto tablet_path = extract_tablet_path(path_prefix); - if (tablet_path.empty()) { - LOG_WARNING("extract tablet path failed").tag("path prefix", path_prefix); + auto parent_path = extract_parent_path(path_prefix); + if (parent_path.empty()) { + LOG_WARNING("extract parent path failed").tag("path prefix", path_prefix); return -1; } - LOG_INFO("extract tablet path success") + LOG_INFO("path prefix is not a dir, extract parent path success") .tag("path prefix", path_prefix) - .tag("tablet path", tablet_path); - int ret = list_directory(tablet_path, &list_iter); + .tag("parent path", parent_path); + int ret = list_directory(parent_path, &list_iter); if (ret != 0) { LOG(WARNING) << "delete prefix, failed to list" << uri; return ret; @@ -417,6 +423,7 @@ int HdfsAccessor::delete_directory_impl(const std::string& dir_path) { // `hdfsDelete`'s return value or errno to avoid exist rpc? int ret = exists(dir_path); if (ret == 1) { + // dir does not exist return 0; } else if (ret < 0) { return ret; diff --git a/cloud/test/hdfs_accessor_test.cpp b/cloud/test/hdfs_accessor_test.cpp index 72188e4c339a24..11c0af3853b2cd 100644 --- a/cloud/test/hdfs_accessor_test.cpp +++ b/cloud/test/hdfs_accessor_test.cpp @@ -295,6 +295,31 @@ TEST(HdfsAccessorTest, delete_prefix) { EXPECT_TRUE(list_files.contains("data/10000/20000/30000/1_0.dat")); EXPECT_TRUE(list_files.contains("data/10000/20000/1_0.dat")); EXPECT_TRUE(list_files.contains("data111/10000/1_0.dat")); + + ret = accessor.delete_prefix("data/10000/20000"); + EXPECT_EQ(ret, 0); + + iter.reset(); + ret = accessor.list_all(&iter); + EXPECT_EQ(ret, 0); + list_files.clear(); + for (auto file = iter->next(); file.has_value(); file = iter->next()) { + list_files.insert(std::move(file->path)); + } + EXPECT_EQ(list_files.size(), 1); + EXPECT_TRUE(list_files.contains("data111/10000/1_0.dat")); + + ret = accessor.delete_prefix("data111/10000/1_0.dat"); + EXPECT_EQ(ret, 0); + + iter.reset(); + ret = accessor.list_all(&iter); + EXPECT_EQ(ret, 0); + list_files.clear(); + for (auto file = iter->next(); file.has_value(); file = iter->next()) { + list_files.insert(std::move(file->path)); + } + EXPECT_EQ(list_files.size(), 0); } } // namespace doris::cloud From c74e5f02f5af61f55383c97989157d5da1cb1e6f Mon Sep 17 00:00:00 2001 From: Yukang-Lian Date: Fri, 10 Jan 2025 18:08:10 +0800 Subject: [PATCH 5/6] 5 --- cloud/src/recycler/hdfs_accessor.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/cloud/src/recycler/hdfs_accessor.cpp b/cloud/src/recycler/hdfs_accessor.cpp index 98c35dace9d460..0872164bec67a4 100644 --- a/cloud/src/recycler/hdfs_accessor.cpp +++ b/cloud/src/recycler/hdfs_accessor.cpp @@ -20,8 +20,6 @@ #include #include -#include - #include "common/stopwatch.h" #include "recycler/util.h" From 2cb3baf35646ad0e23100f88bfbbdae8371eec7e Mon Sep 17 00:00:00 2001 From: Yukang-Lian Date: Fri, 10 Jan 2025 21:47:27 +0800 Subject: [PATCH 6/6] 6 --- cloud/src/recycler/hdfs_accessor.cpp | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/cloud/src/recycler/hdfs_accessor.cpp b/cloud/src/recycler/hdfs_accessor.cpp index 0872164bec67a4..1999bcfa16543a 100644 --- a/cloud/src/recycler/hdfs_accessor.cpp +++ b/cloud/src/recycler/hdfs_accessor.cpp @@ -371,13 +371,18 @@ int HdfsAccessor::delete_prefix(const std::string& path_prefix, int64_t expirati LOG(INFO) << "delete prefix, uri=" << uri; // If path prefix exists, assume it is a dir or a file. if (exists(path_prefix) == 0) { - // try to delete path prefix as a dir or a file. + // If it exists, then it is a dir or a file. + // delete_directory func can delete a dir or a file. if (delete_directory(path_prefix) == 0) { LOG(INFO) << "delete prefix succ" << ", is dir or file = true" << ", uri=" << uri; return 0; } + // delete failed, return err + LOG_WARNING("delete prefix failed, this is a dir or a file") + .tag("path prefix", path_prefix); + return -1; } // If path prefix is not a dir or a file, // for example: data/492211/02000000008a012957476a3e174dfdaa71ee5f80a3abafa3_.