From d5297b6ac2b65383bdaf67ed88b8c42063ef5849 Mon Sep 17 00:00:00 2001 From: morningman Date: Tue, 4 Mar 2025 23:57:04 +0800 Subject: [PATCH 1/6] [fix](kerberos) fix renew kerberos issue --- be/src/common/config.cpp | 7 ++++--- be/src/common/kerberos/kerberos_ticket_cache.cpp | 6 +++++- be/src/common/kerberos/kerberos_ticket_cache.h | 6 +++++- be/src/io/fs/file_handle_cache.cpp | 4 ++++ be/src/io/fs/hdfs/hdfs_mgr.cpp | 13 ++++++++++--- be/src/io/fs/hdfs_file_reader.cpp | 2 +- be/src/io/hdfs_builder.cpp | 1 + be/src/io/hdfs_util.h | 2 ++ be/src/util/lru_multi_cache.inline.h | 4 +++- 9 files changed, 35 insertions(+), 10 deletions(-) diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp index b349c7430b85b4..233b46cf0876e3 100644 --- a/be/src/common/config.cpp +++ b/be/src/common/config.cpp @@ -1143,8 +1143,9 @@ DEFINE_Bool(enable_feature_binlog, "false"); // enable set in BitmapValue DEFINE_Bool(enable_set_in_bitmap_value, "true"); -DEFINE_Int64(max_hdfs_file_handle_cache_num, "1000"); -DEFINE_Int32(max_hdfs_file_handle_cache_time_sec, "3600"); +DEFINE_Int64(max_hdfs_file_handle_cache_num, "20000"); +// This time should less than kerberos_refresh_interval_second +DEFINE_Int32(max_hdfs_file_handle_cache_time_sec, "28800"); DEFINE_Int64(max_external_file_meta_cache_num, "1000"); DEFINE_mInt32(common_obj_lru_cache_stale_sweep_time_sec, "900"); // Apply delete pred in cumu compaction @@ -1157,7 +1158,7 @@ DEFINE_mBool(allow_zero_date, "false"); DEFINE_Bool(allow_invalid_decimalv2_literal, "false"); DEFINE_mString(kerberos_ccache_path, "/tmp/"); DEFINE_mString(kerberos_krb5_conf_path, "/etc/krb5.conf"); -DEFINE_mInt32(kerberos_refresh_interval_second, "3600"); +DEFINE_mInt32(kerberos_refresh_interval_second, "43200"); DEFINE_mString(get_stack_trace_tool, "libunwind"); DEFINE_mString(dwarf_location_info_mode, "FAST"); diff --git a/be/src/common/kerberos/kerberos_ticket_cache.cpp b/be/src/common/kerberos/kerberos_ticket_cache.cpp index 982dee22a5be7a..eb87ff368fff37 100644 --- a/be/src/common/kerberos/kerberos_ticket_cache.cpp +++ b/be/src/common/kerberos/kerberos_ticket_cache.cpp @@ -23,6 +23,7 @@ #include #include "common/config.h" +#include "util/time.h" namespace doris::kerberos { @@ -119,6 +120,8 @@ Status KerberosTicketCache::login() { } return status; } + _ticket_lifetime_sec = static_cast(creds.times.endtime) - + static_cast(creds.times.starttime); // init ccache file status = _krb5_interface->cc_initialize(_context, temp_ccache, _principal); @@ -214,7 +217,8 @@ void KerberosTicketCache::start_periodic_refresh() { // ignore and continue LOG(WARNING) << st.to_string(); } else { - LOG(INFO) << "refresh kerberos ticket cache: " << _ticket_cache_path; + LOG(INFO) << "refresh kerberos ticket cache: " << _ticket_cache_path + << ", lifetime sec: " << _ticket_lifetime_sec; } } } diff --git a/be/src/common/kerberos/kerberos_ticket_cache.h b/be/src/common/kerberos/kerberos_ticket_cache.h index 2c2c4f94fe1a35..9ca595bd5fa4a3 100644 --- a/be/src/common/kerberos/kerberos_ticket_cache.h +++ b/be/src/common/kerberos/kerberos_ticket_cache.h @@ -104,6 +104,8 @@ class KerberosTicketCache : public std::enable_shared_from_this get_ticket_info(); + int64_t get_ticket_lifetime_sec() const { return _ticket_lifetime_sec; } + private: // Initialize the ticket cache file path using principal and keytab information Status _init_ticket_cache_path(); @@ -125,6 +127,8 @@ class KerberosTicketCache : public std::enable_shared_from_this _refresh_thread; @@ -133,7 +137,7 @@ class KerberosTicketCache : public std::enable_shared_from_this _should_stop_refresh {false}; // Sleep time between refresh checks (in milliseconds) - std::chrono::milliseconds _refresh_thread_sleep_time {10}; + std::chrono::milliseconds _refresh_thread_sleep_time {5000}; // Interface for KRB5 operations std::unique_ptr _krb5_interface; diff --git a/be/src/io/fs/file_handle_cache.cpp b/be/src/io/fs/file_handle_cache.cpp index 8104b9d572e651..e2268982d2f0da 100644 --- a/be/src/io/fs/file_handle_cache.cpp +++ b/be/src/io/fs/file_handle_cache.cpp @@ -125,6 +125,10 @@ FileHandleCache::FileHandleCache(size_t capacity, size_t num_partitions, for (FileHandleCachePartition& p : _cache_partitions) { p.cache.set_capacity(partition_capacity); } + Status st = init(); + if (!st) { + LOG(FATAL) << "failed to start file handle cache thread: " << st.to_string(); + } } FileHandleCache::~FileHandleCache() { diff --git a/be/src/io/fs/hdfs/hdfs_mgr.cpp b/be/src/io/fs/hdfs/hdfs_mgr.cpp index d5cde499a5d46e..91367ebbdf3e43 100644 --- a/be/src/io/fs/hdfs/hdfs_mgr.cpp +++ b/be/src/io/fs/hdfs/hdfs_mgr.cpp @@ -74,13 +74,20 @@ void HdfsMgr::_cleanup_loop() { // Find expired handlers for (const auto& entry : _fs_handlers) { - if (current_time - entry.second->last_access_time >= - _instance_timeout_seconds) { + bool is_expired = current_time - entry.second->last_access_time >= + _instance_timeout_seconds; + bool is_krb_expired = + entry.second->is_kerberos_auth && + (current_time - entry.second->create_time >= + entry.second->ticket_cache->get_ticket_lifetime_sec() / 2); + if (is_expired || is_krb_expired) { LOG(INFO) << "Found expired HDFS handler, hash_code=" << entry.first << ", last_access_time=" << entry.second->last_access_time << ", is_kerberos=" << entry.second->is_kerberos_auth << ", principal=" << entry.second->principal - << ", fs_name=" << entry.second->fs_name; + << ", fs_name=" << entry.second->fs_name + << ", is_expired=" << is_expired + << ", is_krb_expire=" << is_krb_expired; to_remove.push_back(entry.first); handlers_to_cleanup.push_back(entry.second); } diff --git a/be/src/io/fs/hdfs_file_reader.cpp b/be/src/io/fs/hdfs_file_reader.cpp index d43cfae1c28228..977ed3d51e339d 100644 --- a/be/src/io/fs/hdfs_file_reader.cpp +++ b/be/src/io/fs/hdfs_file_reader.cpp @@ -49,7 +49,7 @@ namespace { Result get_file(const hdfsFS& fs, const Path& file, int64_t mtime, int64_t file_size) { static FileHandleCache cache(config::max_hdfs_file_handle_cache_num, 16, - config::max_hdfs_file_handle_cache_time_sec * 1000L); + config::max_hdfs_file_handle_cache_time_sec); bool cache_hit; FileHandleCache::Accessor accessor; RETURN_IF_ERROR_RESULT(cache.get_file_handle(fs, file.native(), mtime, file_size, false, diff --git a/be/src/io/hdfs_builder.cpp b/be/src/io/hdfs_builder.cpp index cced8627546c04..94e60f5bee5e14 100644 --- a/be/src/io/hdfs_builder.cpp +++ b/be/src/io/hdfs_builder.cpp @@ -156,6 +156,7 @@ Status HDFSCommonBuilder::set_kerberos_ticket_cache() { // ATTN, can't use ticket_cache->get_ticket_cache_path() directly, // it may cause the kerberos ticket cache path in libhdfs is empty, kerberos_ticket_path = ticket_cache->get_ticket_cache_path(); + hdfsBuilderSetUserName(hdfs_builder, hdfs_kerberos_principal.c_str()); hdfsBuilderSetKerbTicketCachePath(hdfs_builder, kerberos_ticket_path.c_str()); hdfsBuilderSetForceNewInstance(hdfs_builder); LOG(INFO) << "get kerberos ticket path: " << kerberos_ticket_path diff --git a/be/src/io/hdfs_util.h b/be/src/io/hdfs_util.h index 2994a26bb49203..c53a41c7166bcf 100644 --- a/be/src/io/hdfs_util.h +++ b/be/src/io/hdfs_util.h @@ -47,6 +47,7 @@ class HdfsHandler { std::string principal; std::string keytab_path; std::string fs_name; + uint64_t create_time; std::atomic last_access_time; std::shared_ptr ticket_cache; @@ -58,6 +59,7 @@ class HdfsHandler { principal(principal_), keytab_path(keytab_path_), fs_name(fs_name_), + create_time(std::time(nullptr)), last_access_time(std::time(nullptr)), ticket_cache(ticket_cache_) {} diff --git a/be/src/util/lru_multi_cache.inline.h b/be/src/util/lru_multi_cache.inline.h index 87d098913421fa..3613a87637d8c1 100644 --- a/be/src/util/lru_multi_cache.inline.h +++ b/be/src/util/lru_multi_cache.inline.h @@ -194,7 +194,9 @@ void LruMultiCache::release(ValueType_internal* p_value_inte // Has to be currently not available DCHECK(!p_value_internal->is_available()); - p_value_internal->timestamp_seconds = MonotonicSeconds(); + // DO NOT update timestamp_seconds when release. + // Because we are about to evict cache value after a certain period. + // p_value_internal->timestamp_seconds = MonotonicSeconds(); Container& container = p_value_internal->container; From 0076ce5cc076415feda77fe1bbc39bfca66a0d4f Mon Sep 17 00:00:00 2001 From: morningman Date: Thu, 6 Mar 2025 18:01:49 +0800 Subject: [PATCH 2/6] change back to keytab --- be/src/io/fs/hdfs/hdfs_mgr.cpp | 18 ++-- be/src/io/fs/hdfs/hdfs_mgr.h | 2 +- be/src/io/hdfs_builder.cpp | 36 +++---- be/src/io/hdfs_builder.h | 9 -- be/src/io/hdfs_util.h | 12 +-- be/src/util/lru_multi_cache.inline.h | 3 +- thirdparty/build-thirdparty.sh | 136 +++++++++++++-------------- 7 files changed, 106 insertions(+), 110 deletions(-) diff --git a/be/src/io/fs/hdfs/hdfs_mgr.cpp b/be/src/io/fs/hdfs/hdfs_mgr.cpp index 91367ebbdf3e43..b1de36bd266100 100644 --- a/be/src/io/fs/hdfs/hdfs_mgr.cpp +++ b/be/src/io/fs/hdfs/hdfs_mgr.cpp @@ -76,18 +76,18 @@ void HdfsMgr::_cleanup_loop() { for (const auto& entry : _fs_handlers) { bool is_expired = current_time - entry.second->last_access_time >= _instance_timeout_seconds; - bool is_krb_expired = - entry.second->is_kerberos_auth && - (current_time - entry.second->create_time >= - entry.second->ticket_cache->get_ticket_lifetime_sec() / 2); - if (is_expired || is_krb_expired) { + // bool is_krb_expired = + // entry.second->is_kerberos_auth && + // (current_time - entry.second->create_time >= + // entry.second->ticket_cache->get_ticket_lifetime_sec() / 2); + if (is_expired) { LOG(INFO) << "Found expired HDFS handler, hash_code=" << entry.first << ", last_access_time=" << entry.second->last_access_time << ", is_kerberos=" << entry.second->is_kerberos_auth << ", principal=" << entry.second->principal << ", fs_name=" << entry.second->fs_name - << ", is_expired=" << is_expired - << ", is_krb_expire=" << is_krb_expired; + << ", is_expired=" << is_expired; + // << ", is_krb_expire=" << is_krb_expired; to_remove.push_back(entry.first); handlers_to_cleanup.push_back(entry.second); } @@ -187,8 +187,8 @@ Status HdfsMgr::_create_hdfs_fs_impl(const THdfsParams& hdfs_params, const std:: bool is_kerberos = builder.is_kerberos(); *fs_handler = std::make_shared( hdfs_fs, is_kerberos, is_kerberos ? hdfs_params.hdfs_kerberos_principal : "", - is_kerberos ? hdfs_params.hdfs_kerberos_keytab : "", fs_name, - builder.get_ticket_cache()); + is_kerberos ? hdfs_params.hdfs_kerberos_keytab : "", fs_name); + // builder.get_ticket_cache()); return Status::OK(); } diff --git a/be/src/io/fs/hdfs/hdfs_mgr.h b/be/src/io/fs/hdfs/hdfs_mgr.h index 579f6efcc32484..1ebc2ad28b9f98 100644 --- a/be/src/io/fs/hdfs/hdfs_mgr.h +++ b/be/src/io/fs/hdfs/hdfs_mgr.h @@ -83,7 +83,7 @@ class HdfsMgr { std::atomic _should_stop_cleanup_thread; std::unique_ptr _cleanup_thread; - int64_t _cleanup_interval_seconds = 3600; // Run cleanup every hour + int64_t _cleanup_interval_seconds = 300; // Run cleanup every hour int64_t _instance_timeout_seconds = 86400; // 24 hours timeout }; diff --git a/be/src/io/hdfs_builder.cpp b/be/src/io/hdfs_builder.cpp index 94e60f5bee5e14..1f002e5790c99b 100644 --- a/be/src/io/hdfs_builder.cpp +++ b/be/src/io/hdfs_builder.cpp @@ -146,21 +146,21 @@ void HDFSCommonBuilder::set_hdfs_conf_to_hdfs_builder() { } Status HDFSCommonBuilder::set_kerberos_ticket_cache() { - kerberos::KerberosConfig config; - config.set_principal_and_keytab(hdfs_kerberos_principal, hdfs_kerberos_keytab); - config.set_krb5_conf_path(config::kerberos_krb5_conf_path); - config.set_refresh_interval(config::kerberos_refresh_interval_second); - config.set_min_time_before_refresh(600); - kerberos::KerberosTicketMgr* ticket_mgr = ExecEnv::GetInstance()->kerberos_ticket_mgr(); - RETURN_IF_ERROR(ticket_mgr->get_or_set_ticket_cache(config, &ticket_cache)); - // ATTN, can't use ticket_cache->get_ticket_cache_path() directly, - // it may cause the kerberos ticket cache path in libhdfs is empty, - kerberos_ticket_path = ticket_cache->get_ticket_cache_path(); - hdfsBuilderSetUserName(hdfs_builder, hdfs_kerberos_principal.c_str()); - hdfsBuilderSetKerbTicketCachePath(hdfs_builder, kerberos_ticket_path.c_str()); - hdfsBuilderSetForceNewInstance(hdfs_builder); - LOG(INFO) << "get kerberos ticket path: " << kerberos_ticket_path - << " with principal: " << hdfs_kerberos_principal; + // kerberos::KerberosConfig config; + // config.set_principal_and_keytab(hdfs_kerberos_principal, hdfs_kerberos_keytab); + // config.set_krb5_conf_path(config::kerberos_krb5_conf_path); + // config.set_refresh_interval(config::kerberos_refresh_interval_second); + // config.set_min_time_before_refresh(600); + // kerberos::KerberosTicketMgr* ticket_mgr = ExecEnv::GetInstance()->kerberos_ticket_mgr(); + // RETURN_IF_ERROR(ticket_mgr->get_or_set_ticket_cache(config, &ticket_cache)); + // // ATTN, can't use ticket_cache->get_ticket_cache_path() directly, + // // it may cause the kerberos ticket cache path in libhdfs is empty, + // kerberos_ticket_path = ticket_cache->get_ticket_cache_path(); + // hdfsBuilderSetUserName(hdfs_builder, hdfs_kerberos_principal.c_str()); + // hdfsBuilderSetKerbTicketCachePath(hdfs_builder, kerberos_ticket_path.c_str()); + // hdfsBuilderSetForceNewInstance(hdfs_builder); + // LOG(INFO) << "get kerberos ticket path: " << kerberos_ticket_path + // << " with principal: " << hdfs_kerberos_principal; return Status::OK(); } @@ -226,7 +226,11 @@ Status create_hdfs_builder(const THdfsParams& hdfsParams, const std::string& fs_ builder->kerberos_login = true; builder->hdfs_kerberos_principal = hdfsParams.hdfs_kerberos_principal; builder->hdfs_kerberos_keytab = hdfsParams.hdfs_kerberos_keytab; - RETURN_IF_ERROR(builder->set_kerberos_ticket_cache()); + hdfsBuilderSetKerb5Conf(builder->get(), doris::config::kerberos_krb5_conf_path.c_str()); + hdfsBuilderSetPrincipal(builder->get(), builder->hdfs_kerberos_principal.c_str()); + hdfsBuilderSetKeyTabFile(builder->get(), builder->hdfs_kerberos_keytab.c_str()); + hdfsBuilderConfSetStr(builder->get(), "hadoop.kerberos.keytab.login.autorenewal.enabled", "true"); + // RETURN_IF_ERROR(builder->set_kerberos_ticket_cache()); } else { if (hdfsParams.__isset.user) { builder->hadoop_user = hdfsParams.user; diff --git a/be/src/io/hdfs_builder.h b/be/src/io/hdfs_builder.h index 320275981ad974..37e2f99b8c4227 100644 --- a/be/src/io/hdfs_builder.h +++ b/be/src/io/hdfs_builder.h @@ -37,11 +37,6 @@ const std::string HADOOP_SECURITY_AUTHENTICATION = "hadoop.security.authenticati const std::string FALLBACK_TO_SIMPLE_AUTH_ALLOWED = "ipc.client.fallback-to-simple-auth-allowed"; const std::string TRUE_VALUE = "true"; -namespace kerberos { -class KerberosTicketCache; -class KerberosConfig; -}; // namespace kerberos - class HDFSCommonBuilder { friend Status create_hdfs_builder(const THdfsParams& hdfsParams, const std::string& fs_name, HDFSCommonBuilder* builder); @@ -71,8 +66,6 @@ class HDFSCommonBuilder { std::string get_hdfs_conf_value(const std::string& key, const std::string& default_val) const; void set_hdfs_conf_to_hdfs_builder(); - std::shared_ptr get_ticket_cache() { return ticket_cache; } - private: hdfsBuilder* hdfs_builder = nullptr; bool kerberos_login {false}; @@ -83,8 +76,6 @@ class HDFSCommonBuilder { std::string hadoop_user; std::string hdfs_kerberos_keytab; std::string hdfs_kerberos_principal; - std::string kerberos_ticket_path; - std::shared_ptr ticket_cache; std::unordered_map hdfs_conf; }; diff --git a/be/src/io/hdfs_util.h b/be/src/io/hdfs_util.h index c53a41c7166bcf..1b5d4ab3d014bd 100644 --- a/be/src/io/hdfs_util.h +++ b/be/src/io/hdfs_util.h @@ -25,7 +25,7 @@ #include #include -#include "common/kerberos/kerberos_ticket_cache.h" +// #include "common/kerberos/kerberos_ticket_cache.h" #include "common/status.h" #include "io/fs/hdfs.h" #include "io/fs/path.h" @@ -49,19 +49,19 @@ class HdfsHandler { std::string fs_name; uint64_t create_time; std::atomic last_access_time; - std::shared_ptr ticket_cache; + // std::shared_ptr ticket_cache; HdfsHandler(hdfsFS fs, bool is_kerberos, const std::string& principal_, - const std::string& keytab_path_, const std::string& fs_name_, - std::shared_ptr ticket_cache_) + const std::string& keytab_path_, const std::string& fs_name_) + // std::shared_ptr ticket_cache_) : hdfs_fs(fs), is_kerberos_auth(is_kerberos), principal(principal_), keytab_path(keytab_path_), fs_name(fs_name_), create_time(std::time(nullptr)), - last_access_time(std::time(nullptr)), - ticket_cache(ticket_cache_) {} + last_access_time(std::time(nullptr)) {} + // ticket_cache(ticket_cache_) {} ~HdfsHandler() { // The ticket_cache will be automatically released when the last reference is gone diff --git a/be/src/util/lru_multi_cache.inline.h b/be/src/util/lru_multi_cache.inline.h index 3613a87637d8c1..c1c81d12b530e1 100644 --- a/be/src/util/lru_multi_cache.inline.h +++ b/be/src/util/lru_multi_cache.inline.h @@ -196,7 +196,7 @@ void LruMultiCache::release(ValueType_internal* p_value_inte // DO NOT update timestamp_seconds when release. // Because we are about to evict cache value after a certain period. - // p_value_internal->timestamp_seconds = MonotonicSeconds(); + p_value_internal->timestamp_seconds = MonotonicSeconds(); Container& container = p_value_internal->container; @@ -289,6 +289,7 @@ void LruMultiCache::evict_older_than(uint64_t oldest_allowed // - cache size is below capacity and the oldest object is not older than the limit while (!_lru_list.empty() && (_size > _capacity || _lru_list.back().timestamp_seconds < oldest_allowed_timestamp)) { + LOG(INFO) << "yy debug evict file handle cache with timestamp: " << _lru_list.back().timestamp_seconds; _evict_one(_lru_list.back()); } } diff --git a/thirdparty/build-thirdparty.sh b/thirdparty/build-thirdparty.sh index 71dd5a1ae747d7..65792a50ca7570 100755 --- a/thirdparty/build-thirdparty.sh +++ b/thirdparty/build-thirdparty.sh @@ -1859,74 +1859,74 @@ build_icu() { if [[ "${#packages[@]}" -eq 0 ]]; then packages=( - odbc - openssl - libevent - zlib - lz4 - bzip - lzo2 - zstd - boost # must before thrift - abseil - gflags - gtest - glog - protobuf # after gtest - rapidjson - snappy - gperftools - curl - re2 - hyperscan - thrift - leveldb - brpc - jemalloc_doris - rocksdb - krb5 # before cyrus_sasl - cyrus_sasl - librdkafka - flatbuffers - orc - cares - grpc # after cares, protobuf - arrow - s2 - bitshuffle - croaringbitmap - fmt - parallel_hashmap - pdqsort - libdivide - cctz - tsan_header - mysql - aws_sdk - js_and_css - lzma - xml2 - idn - gsasl - hdfs3 - benchmark - simdjson - nlohmann_json - libbacktrace - sse2neon - xxhash - concurrentqueue - fast_float - libunwind - avx2neon - libdeflate - streamvbyte - ali_sdk - base64 - azure - dragonbox - brotli - icu + #odbc + #openssl + #libevent + #zlib + #lz4 + #bzip + #lzo2 + #zstd + #boost # must before thrift + #abseil + #gflags + #gtest + #glog + #protobuf # after gtest + #rapidjson + #snappy + #gperftools + #curl + #re2 + #hyperscan + #thrift + #leveldb + #brpc + #jemalloc_doris + #rocksdb + #krb5 # before cyrus_sasl + #cyrus_sasl + #librdkafka + #flatbuffers + #orc + #cares + #grpc # after cares, protobuf + #arrow + #s2 + #bitshuffle + #croaringbitmap + #fmt + #parallel_hashmap + #pdqsort + #libdivide + #cctz + #tsan_header + #mysql + #aws_sdk + #js_and_css + #lzma + #xml2 + #idn + #gsasl + #hdfs3 + #benchmark + #simdjson + #nlohmann_json + #libbacktrace + #sse2neon + #xxhash + #concurrentqueue + #fast_float + #libunwind + #avx2neon + #libdeflate + #streamvbyte + #ali_sdk + #base64 + #azure + #dragonbox + #brotli + #icu ) if [[ "$(uname -s)" == 'Darwin' ]]; then read -r -a packages <<<"binutils gettext ${packages[*]}" From 2b7fc318f0b717453e0912cb168881aaf76edde6 Mon Sep 17 00:00:00 2001 From: morningman Date: Tue, 18 Mar 2025 12:11:40 +0800 Subject: [PATCH 3/6] 1 --- be/src/io/fs/hdfs/hdfs_mgr.cpp | 4 +- be/src/io/fs/hdfs/hdfs_mgr.h | 2 +- be/src/io/hdfs_builder.cpp | 4 +- be/src/io/hdfs_util.h | 4 +- be/src/util/lru_multi_cache.inline.h | 1 - thirdparty/build-thirdparty.sh | 136 +++++++++++++-------------- 6 files changed, 76 insertions(+), 75 deletions(-) diff --git a/be/src/io/fs/hdfs/hdfs_mgr.cpp b/be/src/io/fs/hdfs/hdfs_mgr.cpp index b1de36bd266100..e9b4315507c685 100644 --- a/be/src/io/fs/hdfs/hdfs_mgr.cpp +++ b/be/src/io/fs/hdfs/hdfs_mgr.cpp @@ -87,7 +87,7 @@ void HdfsMgr::_cleanup_loop() { << ", principal=" << entry.second->principal << ", fs_name=" << entry.second->fs_name << ", is_expired=" << is_expired; - // << ", is_krb_expire=" << is_krb_expired; + // << ", is_krb_expire=" << is_krb_expired; to_remove.push_back(entry.first); handlers_to_cleanup.push_back(entry.second); } @@ -188,7 +188,7 @@ Status HdfsMgr::_create_hdfs_fs_impl(const THdfsParams& hdfs_params, const std:: *fs_handler = std::make_shared( hdfs_fs, is_kerberos, is_kerberos ? hdfs_params.hdfs_kerberos_principal : "", is_kerberos ? hdfs_params.hdfs_kerberos_keytab : "", fs_name); - // builder.get_ticket_cache()); + // builder.get_ticket_cache()); return Status::OK(); } diff --git a/be/src/io/fs/hdfs/hdfs_mgr.h b/be/src/io/fs/hdfs/hdfs_mgr.h index 1ebc2ad28b9f98..579f6efcc32484 100644 --- a/be/src/io/fs/hdfs/hdfs_mgr.h +++ b/be/src/io/fs/hdfs/hdfs_mgr.h @@ -83,7 +83,7 @@ class HdfsMgr { std::atomic _should_stop_cleanup_thread; std::unique_ptr _cleanup_thread; - int64_t _cleanup_interval_seconds = 300; // Run cleanup every hour + int64_t _cleanup_interval_seconds = 3600; // Run cleanup every hour int64_t _instance_timeout_seconds = 86400; // 24 hours timeout }; diff --git a/be/src/io/hdfs_builder.cpp b/be/src/io/hdfs_builder.cpp index 1f002e5790c99b..ed7fb0a34b4f8a 100644 --- a/be/src/io/hdfs_builder.cpp +++ b/be/src/io/hdfs_builder.cpp @@ -145,6 +145,7 @@ void HDFSCommonBuilder::set_hdfs_conf_to_hdfs_builder() { } } +// This method is deprecated, will be removed later Status HDFSCommonBuilder::set_kerberos_ticket_cache() { // kerberos::KerberosConfig config; // config.set_principal_and_keytab(hdfs_kerberos_principal, hdfs_kerberos_keytab); @@ -229,7 +230,8 @@ Status create_hdfs_builder(const THdfsParams& hdfsParams, const std::string& fs_ hdfsBuilderSetKerb5Conf(builder->get(), doris::config::kerberos_krb5_conf_path.c_str()); hdfsBuilderSetPrincipal(builder->get(), builder->hdfs_kerberos_principal.c_str()); hdfsBuilderSetKeyTabFile(builder->get(), builder->hdfs_kerberos_keytab.c_str()); - hdfsBuilderConfSetStr(builder->get(), "hadoop.kerberos.keytab.login.autorenewal.enabled", "true"); + hdfsBuilderConfSetStr(builder->get(), "hadoop.kerberos.keytab.login.autorenewal.enabled", + "true"); // RETURN_IF_ERROR(builder->set_kerberos_ticket_cache()); } else { if (hdfsParams.__isset.user) { diff --git a/be/src/io/hdfs_util.h b/be/src/io/hdfs_util.h index 1b5d4ab3d014bd..8d63a19ec9292a 100644 --- a/be/src/io/hdfs_util.h +++ b/be/src/io/hdfs_util.h @@ -53,7 +53,7 @@ class HdfsHandler { HdfsHandler(hdfsFS fs, bool is_kerberos, const std::string& principal_, const std::string& keytab_path_, const std::string& fs_name_) - // std::shared_ptr ticket_cache_) + // std::shared_ptr ticket_cache_) : hdfs_fs(fs), is_kerberos_auth(is_kerberos), principal(principal_), @@ -61,7 +61,7 @@ class HdfsHandler { fs_name(fs_name_), create_time(std::time(nullptr)), last_access_time(std::time(nullptr)) {} - // ticket_cache(ticket_cache_) {} + // ticket_cache(ticket_cache_) {} ~HdfsHandler() { // The ticket_cache will be automatically released when the last reference is gone diff --git a/be/src/util/lru_multi_cache.inline.h b/be/src/util/lru_multi_cache.inline.h index c1c81d12b530e1..5fe85399aac96b 100644 --- a/be/src/util/lru_multi_cache.inline.h +++ b/be/src/util/lru_multi_cache.inline.h @@ -289,7 +289,6 @@ void LruMultiCache::evict_older_than(uint64_t oldest_allowed // - cache size is below capacity and the oldest object is not older than the limit while (!_lru_list.empty() && (_size > _capacity || _lru_list.back().timestamp_seconds < oldest_allowed_timestamp)) { - LOG(INFO) << "yy debug evict file handle cache with timestamp: " << _lru_list.back().timestamp_seconds; _evict_one(_lru_list.back()); } } diff --git a/thirdparty/build-thirdparty.sh b/thirdparty/build-thirdparty.sh index 65792a50ca7570..71dd5a1ae747d7 100755 --- a/thirdparty/build-thirdparty.sh +++ b/thirdparty/build-thirdparty.sh @@ -1859,74 +1859,74 @@ build_icu() { if [[ "${#packages[@]}" -eq 0 ]]; then packages=( - #odbc - #openssl - #libevent - #zlib - #lz4 - #bzip - #lzo2 - #zstd - #boost # must before thrift - #abseil - #gflags - #gtest - #glog - #protobuf # after gtest - #rapidjson - #snappy - #gperftools - #curl - #re2 - #hyperscan - #thrift - #leveldb - #brpc - #jemalloc_doris - #rocksdb - #krb5 # before cyrus_sasl - #cyrus_sasl - #librdkafka - #flatbuffers - #orc - #cares - #grpc # after cares, protobuf - #arrow - #s2 - #bitshuffle - #croaringbitmap - #fmt - #parallel_hashmap - #pdqsort - #libdivide - #cctz - #tsan_header - #mysql - #aws_sdk - #js_and_css - #lzma - #xml2 - #idn - #gsasl - #hdfs3 - #benchmark - #simdjson - #nlohmann_json - #libbacktrace - #sse2neon - #xxhash - #concurrentqueue - #fast_float - #libunwind - #avx2neon - #libdeflate - #streamvbyte - #ali_sdk - #base64 - #azure - #dragonbox - #brotli - #icu + odbc + openssl + libevent + zlib + lz4 + bzip + lzo2 + zstd + boost # must before thrift + abseil + gflags + gtest + glog + protobuf # after gtest + rapidjson + snappy + gperftools + curl + re2 + hyperscan + thrift + leveldb + brpc + jemalloc_doris + rocksdb + krb5 # before cyrus_sasl + cyrus_sasl + librdkafka + flatbuffers + orc + cares + grpc # after cares, protobuf + arrow + s2 + bitshuffle + croaringbitmap + fmt + parallel_hashmap + pdqsort + libdivide + cctz + tsan_header + mysql + aws_sdk + js_and_css + lzma + xml2 + idn + gsasl + hdfs3 + benchmark + simdjson + nlohmann_json + libbacktrace + sse2neon + xxhash + concurrentqueue + fast_float + libunwind + avx2neon + libdeflate + streamvbyte + ali_sdk + base64 + azure + dragonbox + brotli + icu ) if [[ "$(uname -s)" == 'Darwin' ]]; then read -r -a packages <<<"binutils gettext ${packages[*]}" From acafd92d08313a3b2d7f36b2567f090dd171a0f3 Mon Sep 17 00:00:00 2001 From: morningman Date: Tue, 18 Mar 2025 12:18:18 +0800 Subject: [PATCH 4/6] 3 --- be/src/common/config.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp index 233b46cf0876e3..c0b832517190cc 100644 --- a/be/src/common/config.cpp +++ b/be/src/common/config.cpp @@ -1144,7 +1144,6 @@ DEFINE_Bool(enable_feature_binlog, "false"); DEFINE_Bool(enable_set_in_bitmap_value, "true"); DEFINE_Int64(max_hdfs_file_handle_cache_num, "20000"); -// This time should less than kerberos_refresh_interval_second DEFINE_Int32(max_hdfs_file_handle_cache_time_sec, "28800"); DEFINE_Int64(max_external_file_meta_cache_num, "1000"); DEFINE_mInt32(common_obj_lru_cache_stale_sweep_time_sec, "900"); @@ -1158,6 +1157,7 @@ DEFINE_mBool(allow_zero_date, "false"); DEFINE_Bool(allow_invalid_decimalv2_literal, "false"); DEFINE_mString(kerberos_ccache_path, "/tmp/"); DEFINE_mString(kerberos_krb5_conf_path, "/etc/krb5.conf"); +// Deprecated DEFINE_mInt32(kerberos_refresh_interval_second, "43200"); DEFINE_mString(get_stack_trace_tool, "libunwind"); From f6fc49aa0633281128fe09665d2701210f1d1793 Mon Sep 17 00:00:00 2001 From: morningman Date: Tue, 18 Mar 2025 22:46:26 +0800 Subject: [PATCH 5/6] 2 --- be/test/io/fs/hdfs/hdfs_mgr_test.cpp | 258 +++++++++--------- .../kerberos/test_single_hive_kerberos.groovy | 14 +- .../kerberos/test_two_hive_kerberos.groovy | 26 +- 3 files changed, 149 insertions(+), 149 deletions(-) diff --git a/be/test/io/fs/hdfs/hdfs_mgr_test.cpp b/be/test/io/fs/hdfs/hdfs_mgr_test.cpp index 7f05428acf015a..65f8c6766c22f1 100644 --- a/be/test/io/fs/hdfs/hdfs_mgr_test.cpp +++ b/be/test/io/fs/hdfs/hdfs_mgr_test.cpp @@ -100,7 +100,7 @@ class HdfsMgrTest : public testing::Test { .WillByDefault([](const THdfsParams& params, const std::string& fs_name, std::shared_ptr* fs_handler) { *fs_handler = - std::make_shared(nullptr, false, "", "", fs_name, nullptr); + std::make_shared(nullptr, false, "", "", fs_name); return Status::OK(); }); } @@ -230,133 +230,133 @@ TEST_F(HdfsMgrTest, ConcurrentAccess) { } // Test sharing of KerberosTicketCache between handlers -TEST_F(HdfsMgrTest, SharedKerberosTicketCache) { - // Create handlers with same Kerberos credentials - THdfsParams params = create_test_params("user1", "principal1", "keytab1"); - std::shared_ptr handler1; - std::shared_ptr handler2; - - // Create a shared ticket cache that will be used by both handlers - auto shared_ticket_mgr = std::make_shared>("/tmp/kerberos"); - // Set cleanup interval to 1 second for testing - shared_ticket_mgr->set_cleanup_interval(std::chrono::seconds(1)); - - // Setup mock to create handlers with Kerberos - ON_CALL(*_hdfs_mgr, _create_hdfs_fs_impl(_, _, _)) - .WillByDefault([shared_ticket_mgr](const THdfsParams& params, - const std::string& fs_name, - std::shared_ptr* fs_handler) { - kerberos::KerberosConfig config; - config.set_principal_and_keytab(params.hdfs_kerberos_principal, - params.hdfs_kerberos_keytab); - std::shared_ptr ticket_cache; - RETURN_IF_ERROR(shared_ticket_mgr->get_or_set_ticket_cache(config, &ticket_cache)); - *fs_handler = std::make_shared( - nullptr, true, params.hdfs_kerberos_principal, params.hdfs_kerberos_keytab, - fs_name, ticket_cache); - return Status::OK(); - }); - - // Create first handler - ASSERT_TRUE(_hdfs_mgr->get_or_create_fs(params, "test_fs1", &handler1).ok()); - ASSERT_EQ(MockKerberosTicketCache::get_instance_count(), 1); - - // Create second handler with same credentials - ASSERT_TRUE(_hdfs_mgr->get_or_create_fs(params, "test_fs2", &handler2).ok()); - ASSERT_EQ(MockKerberosTicketCache::get_instance_count(), 1); - - // Verify both handlers share the same ticket cache - ASSERT_EQ(handler1->ticket_cache, handler2->ticket_cache); - // ASSERT_EQ(handler1->ticket_cache, shared_ticket_cache); -} - -// Test cleanup of KerberosTicketCache when handlers are destroyed -TEST_F(HdfsMgrTest, KerberosTicketCacheCleanup) { - THdfsParams params = create_test_params("user1", "principal1", "keytab1"); - - // Create a ticket manager that will be used by the handler - auto ticket_mgr = std::make_shared>("/tmp/kerberos"); - // Set cleanup interval to 1 second for testing - ticket_mgr->set_cleanup_interval(std::chrono::seconds(1)); - - // Setup mock to create handler with Kerberos - ON_CALL(*_hdfs_mgr, _create_hdfs_fs_impl(_, _, _)) - .WillByDefault([ticket_mgr](const THdfsParams& params, const std::string& fs_name, - std::shared_ptr* fs_handler) { - kerberos::KerberosConfig config; - config.set_principal_and_keytab(params.hdfs_kerberos_principal, - params.hdfs_kerberos_keytab); - std::shared_ptr ticket_cache; - - RETURN_IF_ERROR(ticket_mgr->get_or_set_ticket_cache(config, &ticket_cache)); - *fs_handler = std::make_shared( - nullptr, true, params.hdfs_kerberos_principal, params.hdfs_kerberos_keytab, - fs_name, ticket_cache); - return Status::OK(); - }); - - // Create handler - std::shared_ptr handler; - ASSERT_TRUE(_hdfs_mgr->get_or_create_fs(params, "test_fs", &handler).ok()); - std::shared_ptr ticket_cache_holder = handler->ticket_cache; - handler.reset(); - ASSERT_EQ(MockKerberosTicketCache::get_instance_count(), 1); - - // Wait for cleanup - ticket_cache_holder.reset(); - std::this_thread::sleep_for(std::chrono::seconds(6)); - - // Verify handler and ticket cache are cleaned up - ASSERT_EQ(_hdfs_mgr->get_fs_handlers_size(), 0); - ASSERT_EQ(MockKerberosTicketCache::get_instance_count(), 0); -} - -// Test multiple handlers with different Kerberos credentials -TEST_F(HdfsMgrTest, DifferentKerberosCredentials) { - // Create a ticket manager that will be used by both handlers - auto ticket_mgr = std::make_shared>("/tmp/kerberos"); - // Set cleanup interval to 1 second for testing - ticket_mgr->set_cleanup_interval(std::chrono::seconds(1)); - - // Setup mock to create handlers with Kerberos - ON_CALL(*_hdfs_mgr, _create_hdfs_fs_impl(_, _, _)) - .WillByDefault([ticket_mgr](const THdfsParams& params, const std::string& fs_name, - std::shared_ptr* fs_handler) { - kerberos::KerberosConfig config; - config.set_principal_and_keytab(params.hdfs_kerberos_principal, - params.hdfs_kerberos_keytab); - std::shared_ptr ticket_cache; - - RETURN_IF_ERROR(ticket_mgr->get_or_set_ticket_cache(config, &ticket_cache)); - *fs_handler = std::make_shared( - nullptr, true, params.hdfs_kerberos_principal, params.hdfs_kerberos_keytab, - fs_name, ticket_cache); - return Status::OK(); - }); - - // Create handlers with different credentials - // std::cout << "xxx 6 MockKerberosTicketCache::get_instance_count(): " << MockKerberosTicketCache::get_instance_count() << std::endl; - THdfsParams params1 = create_test_params("user1", "principal1", "keytab1"); - THdfsParams params2 = create_test_params("user2", "principal2", "keytab2"); - std::shared_ptr handler1; - std::shared_ptr handler2; - ASSERT_TRUE(_hdfs_mgr->get_or_create_fs(params1, "test_fs1", &handler1).ok()); - ASSERT_TRUE(_hdfs_mgr->get_or_create_fs(params2, "test_fs2", &handler2).ok()); - - // Verify each handler has its own ticket cache - ASSERT_EQ(MockKerberosTicketCache::get_instance_count(), 2); - ASSERT_NE(handler1->ticket_cache, handler2->ticket_cache); - - // Wait for cleanup - // Also need to reset this 2 temp references - handler1.reset(); - handler2.reset(); - std::this_thread::sleep_for(std::chrono::seconds(6)); - - // Verify all handlers and ticket caches are cleaned up - ASSERT_EQ(_hdfs_mgr->get_fs_handlers_size(), 0); - - ASSERT_EQ(MockKerberosTicketCache::get_instance_count(), 0); -} +// TEST_F(HdfsMgrTest, SharedKerberosTicketCache) { +// // Create handlers with same Kerberos credentials +// THdfsParams params = create_test_params("user1", "principal1", "keytab1"); +// std::shared_ptr handler1; +// std::shared_ptr handler2; +// +// // Create a shared ticket cache that will be used by both handlers +// auto shared_ticket_mgr = std::make_shared>("/tmp/kerberos"); +// // Set cleanup interval to 1 second for testing +// shared_ticket_mgr->set_cleanup_interval(std::chrono::seconds(1)); +// +// // Setup mock to create handlers with Kerberos +// ON_CALL(*_hdfs_mgr, _create_hdfs_fs_impl(_, _, _)) +// .WillByDefault([shared_ticket_mgr](const THdfsParams& params, +// const std::string& fs_name, +// std::shared_ptr* fs_handler) { +// kerberos::KerberosConfig config; +// config.set_principal_and_keytab(params.hdfs_kerberos_principal, +// params.hdfs_kerberos_keytab); +// std::shared_ptr ticket_cache; +// RETURN_IF_ERROR(shared_ticket_mgr->get_or_set_ticket_cache(config, &ticket_cache)); +// *fs_handler = std::make_shared( +// nullptr, true, params.hdfs_kerberos_principal, params.hdfs_kerberos_keytab, +// fs_name, ticket_cache); +// return Status::OK(); +// }); +// +// // Create first handler +// ASSERT_TRUE(_hdfs_mgr->get_or_create_fs(params, "test_fs1", &handler1).ok()); +// ASSERT_EQ(MockKerberosTicketCache::get_instance_count(), 1); +// +// // Create second handler with same credentials +// ASSERT_TRUE(_hdfs_mgr->get_or_create_fs(params, "test_fs2", &handler2).ok()); +// ASSERT_EQ(MockKerberosTicketCache::get_instance_count(), 1); +// +// // Verify both handlers share the same ticket cache +// ASSERT_EQ(handler1->ticket_cache, handler2->ticket_cache); +// // ASSERT_EQ(handler1->ticket_cache, shared_ticket_cache); +// } +// +// // Test cleanup of KerberosTicketCache when handlers are destroyed +// TEST_F(HdfsMgrTest, KerberosTicketCacheCleanup) { +// THdfsParams params = create_test_params("user1", "principal1", "keytab1"); +// +// // Create a ticket manager that will be used by the handler +// auto ticket_mgr = std::make_shared>("/tmp/kerberos"); +// // Set cleanup interval to 1 second for testing +// ticket_mgr->set_cleanup_interval(std::chrono::seconds(1)); +// +// // Setup mock to create handler with Kerberos +// ON_CALL(*_hdfs_mgr, _create_hdfs_fs_impl(_, _, _)) +// .WillByDefault([ticket_mgr](const THdfsParams& params, const std::string& fs_name, +// std::shared_ptr* fs_handler) { +// kerberos::KerberosConfig config; +// config.set_principal_and_keytab(params.hdfs_kerberos_principal, +// params.hdfs_kerberos_keytab); +// std::shared_ptr ticket_cache; +// +// RETURN_IF_ERROR(ticket_mgr->get_or_set_ticket_cache(config, &ticket_cache)); +// *fs_handler = std::make_shared( +// nullptr, true, params.hdfs_kerberos_principal, params.hdfs_kerberos_keytab, +// fs_name, ticket_cache); +// return Status::OK(); +// }); +// +// // Create handler +// std::shared_ptr handler; +// ASSERT_TRUE(_hdfs_mgr->get_or_create_fs(params, "test_fs", &handler).ok()); +// std::shared_ptr ticket_cache_holder = handler->ticket_cache; +// handler.reset(); +// ASSERT_EQ(MockKerberosTicketCache::get_instance_count(), 1); +// +// // Wait for cleanup +// ticket_cache_holder.reset(); +// std::this_thread::sleep_for(std::chrono::seconds(6)); +// +// // Verify handler and ticket cache are cleaned up +// ASSERT_EQ(_hdfs_mgr->get_fs_handlers_size(), 0); +// ASSERT_EQ(MockKerberosTicketCache::get_instance_count(), 0); +// } +// +// // Test multiple handlers with different Kerberos credentials +// TEST_F(HdfsMgrTest, DifferentKerberosCredentials) { +// // Create a ticket manager that will be used by both handlers +// auto ticket_mgr = std::make_shared>("/tmp/kerberos"); +// // Set cleanup interval to 1 second for testing +// ticket_mgr->set_cleanup_interval(std::chrono::seconds(1)); +// +// // Setup mock to create handlers with Kerberos +// ON_CALL(*_hdfs_mgr, _create_hdfs_fs_impl(_, _, _)) +// .WillByDefault([ticket_mgr](const THdfsParams& params, const std::string& fs_name, +// std::shared_ptr* fs_handler) { +// kerberos::KerberosConfig config; +// config.set_principal_and_keytab(params.hdfs_kerberos_principal, +// params.hdfs_kerberos_keytab); +// std::shared_ptr ticket_cache; +// +// RETURN_IF_ERROR(ticket_mgr->get_or_set_ticket_cache(config, &ticket_cache)); +// *fs_handler = std::make_shared( +// nullptr, true, params.hdfs_kerberos_principal, params.hdfs_kerberos_keytab, +// fs_name, ticket_cache); +// return Status::OK(); +// }); +// +// // Create handlers with different credentials +// // std::cout << "xxx 6 MockKerberosTicketCache::get_instance_count(): " << MockKerberosTicketCache::get_instance_count() << std::endl; +// THdfsParams params1 = create_test_params("user1", "principal1", "keytab1"); +// THdfsParams params2 = create_test_params("user2", "principal2", "keytab2"); +// std::shared_ptr handler1; +// std::shared_ptr handler2; +// ASSERT_TRUE(_hdfs_mgr->get_or_create_fs(params1, "test_fs1", &handler1).ok()); +// ASSERT_TRUE(_hdfs_mgr->get_or_create_fs(params2, "test_fs2", &handler2).ok()); +// +// // Verify each handler has its own ticket cache +// ASSERT_EQ(MockKerberosTicketCache::get_instance_count(), 2); +// ASSERT_NE(handler1->ticket_cache, handler2->ticket_cache); +// +// // Wait for cleanup +// // Also need to reset this 2 temp references +// handler1.reset(); +// handler2.reset(); +// std::this_thread::sleep_for(std::chrono::seconds(6)); +// +// // Verify all handlers and ticket caches are cleaned up +// ASSERT_EQ(_hdfs_mgr->get_fs_handlers_size(), 0); +// +// ASSERT_EQ(MockKerberosTicketCache::get_instance_count(), 0); +// } } // namespace doris::io diff --git a/regression-test/suites/external_table_p0/kerberos/test_single_hive_kerberos.groovy b/regression-test/suites/external_table_p0/kerberos/test_single_hive_kerberos.groovy index 382d0a7b33762d..c3c97f0ca0c99f 100644 --- a/regression-test/suites/external_table_p0/kerberos/test_single_hive_kerberos.groovy +++ b/regression-test/suites/external_table_p0/kerberos/test_single_hive_kerberos.groovy @@ -115,12 +115,12 @@ suite("test_single_hive_kerberos", "p0,external,kerberos,external_docker,externa // test information_schema.backend_kerberos_ticket_cache // switch to a normal catalog - sql "switch internal"; - List> backends = sql "show backends" - int beNum = backends.size(); - test { - sql """select * from information_schema.backend_kerberos_ticket_cache where PRINCIPAL="presto-server/presto-master.docker.cluster@LABS.TERADATA.COM" and KEYTAB = "${keytab_root_dir}/presto-server.keytab";""" - rowNum beNum - } + // sql "switch internal"; + // List> backends = sql "show backends" + // int beNum = backends.size(); + // test { + // sql """select * from information_schema.backend_kerberos_ticket_cache where PRINCIPAL="presto-server/presto-master.docker.cluster@LABS.TERADATA.COM" and KEYTAB = "${keytab_root_dir}/presto-server.keytab";""" + // rowNum beNum + // } } } diff --git a/regression-test/suites/external_table_p0/kerberos/test_two_hive_kerberos.groovy b/regression-test/suites/external_table_p0/kerberos/test_two_hive_kerberos.groovy index 1d0fa2478ce1e8..4053102b14514e 100644 --- a/regression-test/suites/external_table_p0/kerberos/test_two_hive_kerberos.groovy +++ b/regression-test/suites/external_table_p0/kerberos/test_two_hive_kerberos.groovy @@ -136,19 +136,19 @@ suite("test_two_hive_kerberos", "p0,external,kerberos,external_docker,external_d thread1.join() thread2.join() - // test information_schema.backend_kerberos_ticket_cache - sql """switch internal""" - List> backends = sql "show backends" - int beNum = backends.size(); - test { - sql """select * from information_schema.backend_kerberos_ticket_cache where PRINCIPAL="hive/presto-master.docker.cluster@LABS.TERADATA.COM" and KEYTAB = "${keytab_root_dir}/hive-presto-master.keytab";""" - rowNum beNum - } - - test { - sql """select * from information_schema.backend_kerberos_ticket_cache where PRINCIPAL="hive/presto-master.docker.cluster@OTHERREALM.COM" and KEYTAB = "${keytab_root_dir}/other-hive-presto-master.keytab";""" - rowNum beNum - } + // // test information_schema.backend_kerberos_ticket_cache + // sql """switch internal""" + // List> backends = sql "show backends" + // int beNum = backends.size(); + // test { + // sql """select * from information_schema.backend_kerberos_ticket_cache where PRINCIPAL="hive/presto-master.docker.cluster@LABS.TERADATA.COM" and KEYTAB = "${keytab_root_dir}/hive-presto-master.keytab";""" + // rowNum beNum + // } + + // test { + // sql """select * from information_schema.backend_kerberos_ticket_cache where PRINCIPAL="hive/presto-master.docker.cluster@OTHERREALM.COM" and KEYTAB = "${keytab_root_dir}/other-hive-presto-master.keytab";""" + // rowNum beNum + // } // sql """drop catalog ${hms_catalog_name};""" // sql """drop catalog other_${hms_catalog_name};""" From 2438ac0c26abd5038533bb69db4311625f069e50 Mon Sep 17 00:00:00 2001 From: morningman Date: Wed, 19 Mar 2025 10:18:45 +0800 Subject: [PATCH 6/6] fix --- be/test/io/fs/hdfs/hdfs_mgr_test.cpp | 43 ++++++++++++++-------------- 1 file changed, 21 insertions(+), 22 deletions(-) diff --git a/be/test/io/fs/hdfs/hdfs_mgr_test.cpp b/be/test/io/fs/hdfs/hdfs_mgr_test.cpp index 65f8c6766c22f1..0ecc9dd030f942 100644 --- a/be/test/io/fs/hdfs/hdfs_mgr_test.cpp +++ b/be/test/io/fs/hdfs/hdfs_mgr_test.cpp @@ -99,8 +99,7 @@ class HdfsMgrTest : public testing::Test { ON_CALL(*_hdfs_mgr, _create_hdfs_fs_impl(_, _, _)) .WillByDefault([](const THdfsParams& params, const std::string& fs_name, std::shared_ptr* fs_handler) { - *fs_handler = - std::make_shared(nullptr, false, "", "", fs_name); + *fs_handler = std::make_shared(nullptr, false, "", "", fs_name); return Status::OK(); }); } @@ -235,12 +234,12 @@ TEST_F(HdfsMgrTest, ConcurrentAccess) { // THdfsParams params = create_test_params("user1", "principal1", "keytab1"); // std::shared_ptr handler1; // std::shared_ptr handler2; -// +// // // Create a shared ticket cache that will be used by both handlers // auto shared_ticket_mgr = std::make_shared>("/tmp/kerberos"); // // Set cleanup interval to 1 second for testing // shared_ticket_mgr->set_cleanup_interval(std::chrono::seconds(1)); -// +// // // Setup mock to create handlers with Kerberos // ON_CALL(*_hdfs_mgr, _create_hdfs_fs_impl(_, _, _)) // .WillByDefault([shared_ticket_mgr](const THdfsParams& params, @@ -256,29 +255,29 @@ TEST_F(HdfsMgrTest, ConcurrentAccess) { // fs_name, ticket_cache); // return Status::OK(); // }); -// +// // // Create first handler // ASSERT_TRUE(_hdfs_mgr->get_or_create_fs(params, "test_fs1", &handler1).ok()); // ASSERT_EQ(MockKerberosTicketCache::get_instance_count(), 1); -// +// // // Create second handler with same credentials // ASSERT_TRUE(_hdfs_mgr->get_or_create_fs(params, "test_fs2", &handler2).ok()); // ASSERT_EQ(MockKerberosTicketCache::get_instance_count(), 1); -// +// // // Verify both handlers share the same ticket cache // ASSERT_EQ(handler1->ticket_cache, handler2->ticket_cache); // // ASSERT_EQ(handler1->ticket_cache, shared_ticket_cache); // } -// +// // // Test cleanup of KerberosTicketCache when handlers are destroyed // TEST_F(HdfsMgrTest, KerberosTicketCacheCleanup) { // THdfsParams params = create_test_params("user1", "principal1", "keytab1"); -// +// // // Create a ticket manager that will be used by the handler // auto ticket_mgr = std::make_shared>("/tmp/kerberos"); // // Set cleanup interval to 1 second for testing // ticket_mgr->set_cleanup_interval(std::chrono::seconds(1)); -// +// // // Setup mock to create handler with Kerberos // ON_CALL(*_hdfs_mgr, _create_hdfs_fs_impl(_, _, _)) // .WillByDefault([ticket_mgr](const THdfsParams& params, const std::string& fs_name, @@ -287,37 +286,37 @@ TEST_F(HdfsMgrTest, ConcurrentAccess) { // config.set_principal_and_keytab(params.hdfs_kerberos_principal, // params.hdfs_kerberos_keytab); // std::shared_ptr ticket_cache; -// +// // RETURN_IF_ERROR(ticket_mgr->get_or_set_ticket_cache(config, &ticket_cache)); // *fs_handler = std::make_shared( // nullptr, true, params.hdfs_kerberos_principal, params.hdfs_kerberos_keytab, // fs_name, ticket_cache); // return Status::OK(); // }); -// +// // // Create handler // std::shared_ptr handler; // ASSERT_TRUE(_hdfs_mgr->get_or_create_fs(params, "test_fs", &handler).ok()); // std::shared_ptr ticket_cache_holder = handler->ticket_cache; // handler.reset(); // ASSERT_EQ(MockKerberosTicketCache::get_instance_count(), 1); -// +// // // Wait for cleanup // ticket_cache_holder.reset(); // std::this_thread::sleep_for(std::chrono::seconds(6)); -// +// // // Verify handler and ticket cache are cleaned up // ASSERT_EQ(_hdfs_mgr->get_fs_handlers_size(), 0); // ASSERT_EQ(MockKerberosTicketCache::get_instance_count(), 0); // } -// +// // // Test multiple handlers with different Kerberos credentials // TEST_F(HdfsMgrTest, DifferentKerberosCredentials) { // // Create a ticket manager that will be used by both handlers // auto ticket_mgr = std::make_shared>("/tmp/kerberos"); // // Set cleanup interval to 1 second for testing // ticket_mgr->set_cleanup_interval(std::chrono::seconds(1)); -// +// // // Setup mock to create handlers with Kerberos // ON_CALL(*_hdfs_mgr, _create_hdfs_fs_impl(_, _, _)) // .WillByDefault([ticket_mgr](const THdfsParams& params, const std::string& fs_name, @@ -326,14 +325,14 @@ TEST_F(HdfsMgrTest, ConcurrentAccess) { // config.set_principal_and_keytab(params.hdfs_kerberos_principal, // params.hdfs_kerberos_keytab); // std::shared_ptr ticket_cache; -// +// // RETURN_IF_ERROR(ticket_mgr->get_or_set_ticket_cache(config, &ticket_cache)); // *fs_handler = std::make_shared( // nullptr, true, params.hdfs_kerberos_principal, params.hdfs_kerberos_keytab, // fs_name, ticket_cache); // return Status::OK(); // }); -// +// // // Create handlers with different credentials // // std::cout << "xxx 6 MockKerberosTicketCache::get_instance_count(): " << MockKerberosTicketCache::get_instance_count() << std::endl; // THdfsParams params1 = create_test_params("user1", "principal1", "keytab1"); @@ -342,20 +341,20 @@ TEST_F(HdfsMgrTest, ConcurrentAccess) { // std::shared_ptr handler2; // ASSERT_TRUE(_hdfs_mgr->get_or_create_fs(params1, "test_fs1", &handler1).ok()); // ASSERT_TRUE(_hdfs_mgr->get_or_create_fs(params2, "test_fs2", &handler2).ok()); -// +// // // Verify each handler has its own ticket cache // ASSERT_EQ(MockKerberosTicketCache::get_instance_count(), 2); // ASSERT_NE(handler1->ticket_cache, handler2->ticket_cache); -// +// // // Wait for cleanup // // Also need to reset this 2 temp references // handler1.reset(); // handler2.reset(); // std::this_thread::sleep_for(std::chrono::seconds(6)); -// +// // // Verify all handlers and ticket caches are cleaned up // ASSERT_EQ(_hdfs_mgr->get_fs_handlers_size(), 0); -// +// // ASSERT_EQ(MockKerberosTicketCache::get_instance_count(), 0); // }