From 47aaea81b989fc8dbaf7d1bc028be411a2a65908 Mon Sep 17 00:00:00 2001 From: morningman Date: Wed, 26 Feb 2025 11:25:43 +0800 Subject: [PATCH 1/3] [fix](libhdfs) fix the lifecycle issue of libhdfs config --- be/src/io/hdfs_builder.cpp | 20 ++++++++++++++++---- be/src/io/hdfs_builder.h | 7 +++++++ 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/be/src/io/hdfs_builder.cpp b/be/src/io/hdfs_builder.cpp index 99ee89596ed9ac..a0c0331e9510ee 100644 --- a/be/src/io/hdfs_builder.cpp +++ b/be/src/io/hdfs_builder.cpp @@ -106,6 +106,16 @@ Status HDFSCommonBuilder::init_hdfs_builder() { return Status::OK(); } +void HDFSCommonBuilder::set_hdfs_conf(const std::string& key, const std::string& val) { + hdfs_conf[key] = val; +} + +void HDFSCommonBuilder::set_hdfs_conf_to_hdfs_builder() { + for (const auto& pair : hdfs_conf) { + hdfsBuilderConfSetStr(hdfs_builder, pair.first.c_str(), pair.second.c_str()); + } +} + Status HDFSCommonBuilder::check_krb_params() { std::string ticket_path = doris::config::kerberos_ccache_path; if (!ticket_path.empty()) { @@ -164,13 +174,13 @@ Status create_hdfs_builder(const THdfsParams& hdfsParams, const std::string& fs_ builder->hdfs_kerberos_keytab = hdfsParams.hdfs_kerberos_keytab; #ifdef USE_HADOOP_HDFS hdfsBuilderSetKerb5Conf(builder->get(), doris::config::kerberos_krb5_conf_path.c_str()); - hdfsBuilderSetKeyTabFile(builder->get(), hdfsParams.hdfs_kerberos_keytab.c_str()); + hdfsBuilderSetKeyTabFile(builder->get(), builder->hdfs_kerberos_keytab.c_str()); #endif } if (hdfsParams.__isset.hdfs_kerberos_principal) { builder->kerberos_login = true; builder->hdfs_kerberos_principal = hdfsParams.hdfs_kerberos_principal; - hdfsBuilderSetPrincipal(builder->get(), hdfsParams.hdfs_kerberos_principal.c_str()); + hdfsBuilderSetPrincipal(builder->get(), builder->hdfs_kerberos_principal.c_str()); } else if (hdfsParams.__isset.user) { hdfsBuilderSetUserName(builder->get(), hdfsParams.user.c_str()); #ifdef USE_HADOOP_HDFS @@ -181,15 +191,17 @@ Status create_hdfs_builder(const THdfsParams& hdfsParams, const std::string& fs_ // set other conf if (hdfsParams.__isset.hdfs_conf) { for (const THdfsConf& conf : hdfsParams.hdfs_conf) { - hdfsBuilderConfSetStr(builder->get(), conf.key.c_str(), conf.value.c_str()); + builder->set_hdfs_conf(conf.key, conf.value); LOG(INFO) << "set hdfs config: " << conf.key << ", value: " << conf.value; #ifdef USE_HADOOP_HDFS // Set krb5.conf, we should define java.security.krb5.conf in catalog properties if (strcmp(conf.key.c_str(), "java.security.krb5.conf") == 0) { - hdfsBuilderSetKerb5Conf(builder->get(), conf.value.c_str()); + builder->krb5_conf_path = conf.value; + hdfsBuilderSetKerb5Conf(builder->get(), builder->krb5_conf_path.c_str()); } #endif } + builder->set_hdfs_conf_to_hdfs_builder(); } if (builder->is_kerberos()) { RETURN_IF_ERROR(builder->check_krb_params()); diff --git a/be/src/io/hdfs_builder.h b/be/src/io/hdfs_builder.h index 125f3bed82be41..8b2539f9e86bc7 100644 --- a/be/src/io/hdfs_builder.h +++ b/be/src/io/hdfs_builder.h @@ -59,11 +59,18 @@ class HDFSCommonBuilder { bool is_kerberos() const { return kerberos_login; } Status check_krb_params(); + void set_hdfs_conf(const std::string& key, const std::string& val); + void set_hdfs_conf_to_hdfs_builder(); + private: hdfsBuilder* hdfs_builder = nullptr; bool kerberos_login {false}; + std::string fs_name; + std::string hadoop_user; std::string hdfs_kerberos_keytab; std::string hdfs_kerberos_principal; + std::string krb5_conf_path; + std::unordered_map hdfs_conf; }; THdfsParams parse_properties(const std::map& properties); From 1fbbaf85b83ba71510003b7b437483976e22e0ff Mon Sep 17 00:00:00 2001 From: morningman Date: Wed, 26 Feb 2025 11:37:50 +0800 Subject: [PATCH 2/3] 2 --- be/src/io/hdfs_builder.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/be/src/io/hdfs_builder.cpp b/be/src/io/hdfs_builder.cpp index a0c0331e9510ee..604d7f387327be 100644 --- a/be/src/io/hdfs_builder.cpp +++ b/be/src/io/hdfs_builder.cpp @@ -174,7 +174,7 @@ Status create_hdfs_builder(const THdfsParams& hdfsParams, const std::string& fs_ builder->hdfs_kerberos_keytab = hdfsParams.hdfs_kerberos_keytab; #ifdef USE_HADOOP_HDFS hdfsBuilderSetKerb5Conf(builder->get(), doris::config::kerberos_krb5_conf_path.c_str()); - hdfsBuilderSetKeyTabFile(builder->get(), builder->hdfs_kerberos_keytab.c_str()); + hdfsBuilderSetKeyTabFile(builder->get(), builder->hdfs_kerberos_keytab.c_str()); #endif } if (hdfsParams.__isset.hdfs_kerberos_principal) { From 547f0ae11da6731fa076d8ef64e8985c915a71f0 Mon Sep 17 00:00:00 2001 From: morningman Date: Wed, 26 Feb 2025 11:53:18 +0800 Subject: [PATCH 3/3] 2 --- be/src/io/hdfs_builder.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/be/src/io/hdfs_builder.cpp b/be/src/io/hdfs_builder.cpp index 604d7f387327be..cb27b23ca851af 100644 --- a/be/src/io/hdfs_builder.cpp +++ b/be/src/io/hdfs_builder.cpp @@ -167,7 +167,8 @@ THdfsParams parse_properties(const std::map& propertie Status create_hdfs_builder(const THdfsParams& hdfsParams, const std::string& fs_name, HDFSCommonBuilder* builder) { RETURN_IF_ERROR(builder->init_hdfs_builder()); - hdfsBuilderSetNameNode(builder->get(), fs_name.c_str()); + builder->fs_name = fs_name; + hdfsBuilderSetNameNode(builder->get(), builder->fs_name.c_str()); // set kerberos conf if (hdfsParams.__isset.hdfs_kerberos_keytab) { builder->kerberos_login = true; @@ -182,7 +183,8 @@ Status create_hdfs_builder(const THdfsParams& hdfsParams, const std::string& fs_ builder->hdfs_kerberos_principal = hdfsParams.hdfs_kerberos_principal; hdfsBuilderSetPrincipal(builder->get(), builder->hdfs_kerberos_principal.c_str()); } else if (hdfsParams.__isset.user) { - hdfsBuilderSetUserName(builder->get(), hdfsParams.user.c_str()); + builder->hadoop_user = hdfsParams.user; + hdfsBuilderSetUserName(builder->get(), builder->hadoop_user.c_str()); #ifdef USE_HADOOP_HDFS hdfsBuilderSetKerb5Conf(builder->get(), nullptr); hdfsBuilderSetKeyTabFile(builder->get(), nullptr);