From 5730582b0e4de16cb1905b7911735fe047f17803 Mon Sep 17 00:00:00 2001 From: zhangdong <493738387@qq.com> Date: Sun, 3 Sep 2023 08:09:47 +0800 Subject: [PATCH 1/8] be when priority_network is empty,not choose ipv6 --- be/src/service/backend_options.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/be/src/service/backend_options.cpp b/be/src/service/backend_options.cpp index 20f4e68c7603cb..419e820bd53bff 100644 --- a/be/src/service/backend_options.cpp +++ b/be/src/service/backend_options.cpp @@ -71,7 +71,7 @@ bool BackendOptions::init() { } else if ((*addr_it).is_loopback()) { loopback = addr_it->get_host_address(); _bind_ipv6 = addr_it->is_ipv6(); - } else { + } else if (!addr_it->is_ipv6()) { _s_localhost = addr_it->get_host_address(); _bind_ipv6 = addr_it->is_ipv6(); break; From 24c3758d2caed5debd21e76a68fb3697b9fd389e Mon Sep 17 00:00:00 2001 From: zhangdong <493738387@qq.com> Date: Sun, 3 Sep 2023 08:23:07 +0800 Subject: [PATCH 2/8] be when priority_network is empty,not choose ipv6 --- be/src/service/backend_options.cpp | 38 ++++++++++++++++++------------ 1 file changed, 23 insertions(+), 15 deletions(-) diff --git a/be/src/service/backend_options.cpp b/be/src/service/backend_options.cpp index 419e820bd53bff..773b272a132417 100644 --- a/be/src/service/backend_options.cpp +++ b/be/src/service/backend_options.cpp @@ -54,11 +54,10 @@ bool BackendOptions::init() { return false; } - std::string loopback; std::vector::iterator addr_it = hosts.begin(); - for (; addr_it != hosts.end(); ++addr_it) { - VLOG_CRITICAL << "check ip=" << addr_it->get_host_address(); - if (!_s_priority_cidrs.empty()) { + if (!_s_priority_cidrs.empty()) { + for (; addr_it != hosts.end(); ++addr_it) { + VLOG_CRITICAL << "check ip=" << addr_it->get_host_address(); // Whether to use IPV4 or IPV6, it's configured by CIDR format. // If both IPV4 and IPV6 are configured, the config order decides priority. if (is_in_prior_network(addr_it->get_host_address())) { @@ -68,22 +67,31 @@ bool BackendOptions::init() { } LOG(INFO) << "skip ip not belonged to priority networks: " << addr_it->get_host_address(); - } else if ((*addr_it).is_loopback()) { - loopback = addr_it->get_host_address(); - _bind_ipv6 = addr_it->is_ipv6(); - } else if (!addr_it->is_ipv6()) { - _s_localhost = addr_it->get_host_address(); - _bind_ipv6 = addr_it->is_ipv6(); - break; + } + if (_s_localhost.empty()) { + LOG(FATAL) << "fail to find one valid address, exit."; + return false; + } + } else { + std::string loopback; + for (; addr_it != hosts.end(); ++addr_it) { + if ((*addr_it).is_loopback()) { + loopback = addr_it->get_host_address(); + _bind_ipv6 = addr_it->is_ipv6(); + } else if (!addr_it->is_ipv6()) { + _s_localhost = addr_it->get_host_address(); + _bind_ipv6 = addr_it->is_ipv6(); + break; + } + } + if (_s_localhost.empty()) { + LOG(INFO) << "fail to find one valid non-loopback address, use loopback address."; + _s_localhost = loopback; } } if (_bind_ipv6) { _service_bind_address = "[::0]"; } - if (_s_localhost.empty()) { - LOG(INFO) << "fail to find one valid non-loopback address, use loopback address."; - _s_localhost = loopback; - } LOG(INFO) << "local host ip=" << _s_localhost; return true; } From 9caed4d7559104a53a93b2d792db80906b99ec68 Mon Sep 17 00:00:00 2001 From: zhangdong <493738387@qq.com> Date: Fri, 8 Sep 2023 18:30:50 +0800 Subject: [PATCH 3/8] add ut --- be/src/service/backend_options.cpp | 91 ++++++++++++++------------- be/src/service/backend_options.h | 6 +- be/test/util/backend_options_test.cpp | 51 +++++++++++++++ 3 files changed, 105 insertions(+), 43 deletions(-) create mode 100644 be/test/util/backend_options_test.cpp diff --git a/be/src/service/backend_options.cpp b/be/src/service/backend_options.cpp index 773b272a132417..9832b2b488ec41 100644 --- a/be/src/service/backend_options.cpp +++ b/be/src/service/backend_options.cpp @@ -38,7 +38,7 @@ bool BackendOptions::_bind_ipv6 = false; const char* _service_bind_address = "0.0.0.0"; bool BackendOptions::init() { - if (!analyze_priority_cidrs()) { + if (!analyze_priority_cidrs(config::priority_networks, &_s_priority_cidrs)) { return false; } std::vector hosts; @@ -53,41 +53,8 @@ bool BackendOptions::init() { LOG(FATAL) << "failed to get host"; return false; } - - std::vector::iterator addr_it = hosts.begin(); - if (!_s_priority_cidrs.empty()) { - for (; addr_it != hosts.end(); ++addr_it) { - VLOG_CRITICAL << "check ip=" << addr_it->get_host_address(); - // Whether to use IPV4 or IPV6, it's configured by CIDR format. - // If both IPV4 and IPV6 are configured, the config order decides priority. - if (is_in_prior_network(addr_it->get_host_address())) { - _s_localhost = addr_it->get_host_address(); - _bind_ipv6 = addr_it->is_ipv6(); - break; - } - LOG(INFO) << "skip ip not belonged to priority networks: " - << addr_it->get_host_address(); - } - if (_s_localhost.empty()) { - LOG(FATAL) << "fail to find one valid address, exit."; - return false; - } - } else { - std::string loopback; - for (; addr_it != hosts.end(); ++addr_it) { - if ((*addr_it).is_loopback()) { - loopback = addr_it->get_host_address(); - _bind_ipv6 = addr_it->is_ipv6(); - } else if (!addr_it->is_ipv6()) { - _s_localhost = addr_it->get_host_address(); - _bind_ipv6 = addr_it->is_ipv6(); - break; - } - } - if (_s_localhost.empty()) { - LOG(INFO) << "fail to find one valid non-loopback address, use loopback address."; - _s_localhost = loopback; - } + if (!analyze_localhost(_s_localhost, _bind_ipv6, &_s_priority_cidrs, &hosts)) { + return false; } if (_bind_ipv6) { _service_bind_address = "[::0]"; @@ -126,14 +93,14 @@ const char* BackendOptions::get_service_bind_address_without_bracket() { return _service_bind_address; } -bool BackendOptions::analyze_priority_cidrs() { - if (config::priority_networks == "") { +bool BackendOptions::analyze_priority_cidrs(const std::string& priority_networks, + std::vector* cidrs) { + if (priority_networks == "") { return true; } - LOG(INFO) << "priority cidrs in conf: " << config::priority_networks; + LOG(INFO) << "priority cidrs: " << priority_networks; - std::vector cidr_strs = - strings::Split(config::priority_networks, PRIORITY_CIDR_SEPARATOR); + std::vector cidr_strs = strings::Split(priority_networks, PRIORITY_CIDR_SEPARATOR); for (auto& cidr_str : cidr_strs) { CIDR cidr; @@ -141,7 +108,47 @@ bool BackendOptions::analyze_priority_cidrs() { LOG(FATAL) << "wrong cidr format. cidr_str=" << cidr_str; return false; } - _s_priority_cidrs.push_back(cidr); + cidrs->push_back(cidr); + } + return true; +} + +bool BackendOptions::analyze_localhost(std::string& localhost, bool& bind_ipv6, std::vector* cidrs, + std::vector* hosts) { + std::vector::iterator addr_it = hosts->begin(); + if (!cidrs->empty()) { + for (; addr_it != hosts->end(); ++addr_it) { + VLOG_CRITICAL << "check ip=" << addr_it->get_host_address(); + // Whether to use IPV4 or IPV6, it's configured by CIDR format. + // If both IPV4 and IPV6 are configured, the config order decides priority. + if (is_in_prior_network(addr_it->get_host_address())) { + localhost = addr_it->get_host_address(); + bind_ipv6 = addr_it->is_ipv6(); + break; + } + LOG(INFO) << "skip ip not belonged to priority networks: " + << addr_it->get_host_address(); + } + if (localhost.empty()) { + LOG(FATAL) << "fail to find one valid address, exit."; + return false; + } + } else { + std::string loopback; + for (; addr_it != hosts->end(); ++addr_it) { + if ((*addr_it).is_loopback()) { + loopback = addr_it->get_host_address(); + _bind_ipv6 = addr_it->is_ipv6(); + } else if (!addr_it->is_ipv6()) { + localhost = addr_it->get_host_address(); + _bind_ipv6 = addr_it->is_ipv6(); + break; + } + } + if (localhost.empty()) { + LOG(INFO) << "fail to find one valid non-loopback address, use loopback address."; + localhost = loopback; + } } return true; } diff --git a/be/src/service/backend_options.h b/be/src/service/backend_options.h index 3aff93403635db..4d41a02a95e0fe 100644 --- a/be/src/service/backend_options.h +++ b/be/src/service/backend_options.h @@ -23,6 +23,7 @@ #include #include "gen_cpp/Types_types.h" +#include "util/network_util.h" namespace doris { @@ -37,9 +38,12 @@ class BackendOptions { static bool is_bind_ipv6(); static const char* get_service_bind_address(); static const char* get_service_bind_address_without_bracket(); + static bool analyze_priority_cidrs(const std::string& priority_networks, + std::vector* cidrs); + static bool analyze_localhost(std::string& localhost, bool& bind_ipv6, std::vector* cidrs, + std::vector* hosts); private: - static bool analyze_priority_cidrs(); static bool is_in_prior_network(const std::string& ip); static std::string _s_localhost; diff --git a/be/test/util/backend_options_test.cpp b/be/test/util/backend_options_test.cpp new file mode 100644 index 00000000000000..c1dea5cf2cae59 --- /dev/null +++ b/be/test/util/backend_options_test.cpp @@ -0,0 +1,51 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "service/backend_options.h" + +#include +#include + +#include "gtest/gtest_pred_impl.h" +#include "util/cidr.h" + +namespace doris { + +class BackendOptionsTest : public testing::Test { +public: + BackendOptionsTest() {} + virtual ~BackendOptionsTest() {} +}; + +TEST_F(BackendOptionsTest, normal) { + std::vector hosts; + hosts.emplace_back(std::string("127.0.0.1"), AF_INET, true); + hosts.emplace_back(std::string("10.10.10.10"), AF_INET, false); + hosts.emplace_back(std::string("10.10.10.11"), AF_INET, false); + hosts.emplace_back(std::string("fe80::5054:ff:fec9:dee0"), AF_INET6, false); + hosts.emplace_back(std::string("::1"), AF_INET6, true); + + std::vector cidrs; + BackendOptions::analyze_priority_cidrs("", &cidrs); + std::string localhost; + bool bind_ipv6 = false; + BackendOptions::analyze_localhost(localhost, bind_ipv6, &cidrs, &hosts); + + EXPECT_STREQ("10.10.10.10", localhost.c_str()); +} + +} // namespace doris From 37e3b82a9002eea294ac59a47e6c4e3997f1b6a2 Mon Sep 17 00:00:00 2001 From: zhangdong <493738387@qq.com> Date: Fri, 8 Sep 2023 18:53:35 +0800 Subject: [PATCH 4/8] fix problem by repeat name --- be/src/vec/exec/scan/new_es_scan_node.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/be/src/vec/exec/scan/new_es_scan_node.cpp b/be/src/vec/exec/scan/new_es_scan_node.cpp index 088784b330b7bc..6a3ec3a73890b2 100644 --- a/be/src/vec/exec/scan/new_es_scan_node.cpp +++ b/be/src/vec/exec/scan/new_es_scan_node.cpp @@ -41,7 +41,7 @@ class VScanner; static const std::string NEW_SCAN_NODE_TYPE = "NewEsScanNode"; // Prefer to the local host -static std::string get_host_port(const std::vector& es_hosts) { +static std::string get_host_and_port(const std::vector& es_hosts) { std::string host_port; std::string localhost = doris::BackendOptions::get_localhost(); @@ -152,7 +152,7 @@ Status NewEsScanNode::_init_scanners(std::list* scanners) { } properties[ESScanReader::KEY_SHARD] = std::to_string(es_scan_range->shard_id); properties[ESScanReader::KEY_BATCH_SIZE] = std::to_string(_state->batch_size()); - properties[ESScanReader::KEY_HOST_PORT] = get_host_port(es_scan_range->es_hosts); + properties[ESScanReader::KEY_HOST_PORT] = get_host_and_port(es_scan_range->es_hosts); // push down limit to Elasticsearch // if predicate in _conjunct_ctxs can not be processed by Elasticsearch, we can not push down limit operator to Elasticsearch if (limit() != -1 && limit() <= _state->batch_size()) { From ffeccfeb6e733e43c0cd8d0645261d3aaba90467 Mon Sep 17 00:00:00 2001 From: zhangdong <493738387@qq.com> Date: Fri, 8 Sep 2023 19:45:14 +0800 Subject: [PATCH 5/8] add more ut --- be/test/util/backend_options_test.cpp | 65 +++++++++++++++++++++++++-- 1 file changed, 62 insertions(+), 3 deletions(-) diff --git a/be/test/util/backend_options_test.cpp b/be/test/util/backend_options_test.cpp index c1dea5cf2cae59..6a10b8b394b6a4 100644 --- a/be/test/util/backend_options_test.cpp +++ b/be/test/util/backend_options_test.cpp @@ -31,21 +31,80 @@ class BackendOptionsTest : public testing::Test { virtual ~BackendOptionsTest() {} }; -TEST_F(BackendOptionsTest, normal) { +// only loopback +TEST_F(BackendOptionsTest, emptyCidr1) { + std::vector hosts; + hosts.emplace_back(std::string("127.0.0.1"), AF_INET, true); + + std::vector cidrs; + BackendOptions::analyze_priority_cidrs("", &cidrs); + std::string localhost; + bool bind_ipv6 = false; + BackendOptions::analyze_localhost(localhost, bind_ipv6, &cidrs, &hosts); + EXPECT_STREQ("127.0.0.1", localhost.c_str()); +} + +// priority not loopback +TEST_F(BackendOptionsTest, emptyCidr2) { std::vector hosts; hosts.emplace_back(std::string("127.0.0.1"), AF_INET, true); hosts.emplace_back(std::string("10.10.10.10"), AF_INET, false); hosts.emplace_back(std::string("10.10.10.11"), AF_INET, false); - hosts.emplace_back(std::string("fe80::5054:ff:fec9:dee0"), AF_INET6, false); - hosts.emplace_back(std::string("::1"), AF_INET6, true); std::vector cidrs; BackendOptions::analyze_priority_cidrs("", &cidrs); std::string localhost; bool bind_ipv6 = false; BackendOptions::analyze_localhost(localhost, bind_ipv6, &cidrs, &hosts); + EXPECT_STREQ("10.10.10.10", localhost.c_str()); +} + +// not choose ipv6 +TEST_F(BackendOptionsTest, emptyCidr3) { + std::vector hosts; + hosts.emplace_back(std::string("127.0.0.1"), AF_INET, true); + hosts.emplace_back(std::string("fe80::5054:ff:fec9:dee0"), AF_INET6, false); + hosts.emplace_back(std::string("10.10.10.10"), AF_INET, false); + hosts.emplace_back(std::string("10.10.10.11"), AF_INET, false); + std::vector cidrs; + BackendOptions::analyze_priority_cidrs("", &cidrs); + std::string localhost; + bool bind_ipv6 = false; + BackendOptions::analyze_localhost(localhost, bind_ipv6, &cidrs, &hosts); EXPECT_STREQ("10.10.10.10", localhost.c_str()); } +TEST_F(BackendOptionsTest, ipv4) { + std::vector hosts; + hosts.emplace_back(std::string("127.0.0.1"), AF_INET, true); + hosts.emplace_back(std::string("10.10.10.10"), AF_INET, false); + hosts.emplace_back(std::string("10.10.10.11"), AF_INET, false); + hosts.emplace_back(std::string("fe80::5054:ff:fec9:dee0"), AF_INET6, false); + hosts.emplace_back(std::string("::1"), AF_INET6, true); + + std::vector cidrs; + BackendOptions::analyze_priority_cidrs("10.10.10.11", &cidrs); + std::string localhost; + bool bind_ipv6 = false; + BackendOptions::analyze_localhost(localhost, bind_ipv6, &cidrs, &hosts); + EXPECT_STREQ("10.10.10.11", localhost.c_str()); +} + +TEST_F(BackendOptionsTest, ipv6) { + std::vector hosts; + hosts.emplace_back(std::string("127.0.0.1"), AF_INET, true); + hosts.emplace_back(std::string("10.10.10.10"), AF_INET, false); + hosts.emplace_back(std::string("10.10.10.11"), AF_INET, false); + hosts.emplace_back(std::string("fe80::5054:ff:fec9:dee0"), AF_INET6, false); + hosts.emplace_back(std::string("::1"), AF_INET6, true); + + std::vector cidrs; + BackendOptions::analyze_priority_cidrs("fe80::5054:ff:fec9:dee0", &cidrs); + std::string localhost; + bool bind_ipv6 = false; + BackendOptions::analyze_localhost(localhost, bind_ipv6, &cidrs, &hosts); + EXPECT_STREQ("fe80::5054:ff:fec9:dee0", localhost.c_str()); +} + } // namespace doris From 375b69f309b26677f72c696e24da369a8ff8cb73 Mon Sep 17 00:00:00 2001 From: zhangdong <493738387@qq.com> Date: Fri, 8 Sep 2023 19:59:20 +0800 Subject: [PATCH 6/8] add more ut --- be/test/util/backend_options_test.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/be/test/util/backend_options_test.cpp b/be/test/util/backend_options_test.cpp index 6a10b8b394b6a4..21f14c5c57863c 100644 --- a/be/test/util/backend_options_test.cpp +++ b/be/test/util/backend_options_test.cpp @@ -84,11 +84,11 @@ TEST_F(BackendOptionsTest, ipv4) { hosts.emplace_back(std::string("::1"), AF_INET6, true); std::vector cidrs; - BackendOptions::analyze_priority_cidrs("10.10.10.11", &cidrs); + BackendOptions::analyze_priority_cidrs("127.0.0.1", &cidrs); std::string localhost; bool bind_ipv6 = false; BackendOptions::analyze_localhost(localhost, bind_ipv6, &cidrs, &hosts); - EXPECT_STREQ("10.10.10.11", localhost.c_str()); + EXPECT_STREQ("127.0.0.1", localhost.c_str()); } TEST_F(BackendOptionsTest, ipv6) { @@ -100,11 +100,11 @@ TEST_F(BackendOptionsTest, ipv6) { hosts.emplace_back(std::string("::1"), AF_INET6, true); std::vector cidrs; - BackendOptions::analyze_priority_cidrs("fe80::5054:ff:fec9:dee0", &cidrs); + BackendOptions::analyze_priority_cidrs("::1", &cidrs); std::string localhost; bool bind_ipv6 = false; BackendOptions::analyze_localhost(localhost, bind_ipv6, &cidrs, &hosts); - EXPECT_STREQ("fe80::5054:ff:fec9:dee0", localhost.c_str()); + EXPECT_STREQ("::1", localhost.c_str()); } } // namespace doris From cb7c6672dfabefde48b09f311f47d0236d0b2080 Mon Sep 17 00:00:00 2001 From: zhangdong <493738387@qq.com> Date: Fri, 8 Sep 2023 20:03:00 +0800 Subject: [PATCH 7/8] add more ut --- be/test/util/backend_options_test.cpp | 32 --------------------------- 1 file changed, 32 deletions(-) diff --git a/be/test/util/backend_options_test.cpp b/be/test/util/backend_options_test.cpp index 21f14c5c57863c..a8d0ca41513ca5 100644 --- a/be/test/util/backend_options_test.cpp +++ b/be/test/util/backend_options_test.cpp @@ -75,36 +75,4 @@ TEST_F(BackendOptionsTest, emptyCidr3) { EXPECT_STREQ("10.10.10.10", localhost.c_str()); } -TEST_F(BackendOptionsTest, ipv4) { - std::vector hosts; - hosts.emplace_back(std::string("127.0.0.1"), AF_INET, true); - hosts.emplace_back(std::string("10.10.10.10"), AF_INET, false); - hosts.emplace_back(std::string("10.10.10.11"), AF_INET, false); - hosts.emplace_back(std::string("fe80::5054:ff:fec9:dee0"), AF_INET6, false); - hosts.emplace_back(std::string("::1"), AF_INET6, true); - - std::vector cidrs; - BackendOptions::analyze_priority_cidrs("127.0.0.1", &cidrs); - std::string localhost; - bool bind_ipv6 = false; - BackendOptions::analyze_localhost(localhost, bind_ipv6, &cidrs, &hosts); - EXPECT_STREQ("127.0.0.1", localhost.c_str()); -} - -TEST_F(BackendOptionsTest, ipv6) { - std::vector hosts; - hosts.emplace_back(std::string("127.0.0.1"), AF_INET, true); - hosts.emplace_back(std::string("10.10.10.10"), AF_INET, false); - hosts.emplace_back(std::string("10.10.10.11"), AF_INET, false); - hosts.emplace_back(std::string("fe80::5054:ff:fec9:dee0"), AF_INET6, false); - hosts.emplace_back(std::string("::1"), AF_INET6, true); - - std::vector cidrs; - BackendOptions::analyze_priority_cidrs("::1", &cidrs); - std::string localhost; - bool bind_ipv6 = false; - BackendOptions::analyze_localhost(localhost, bind_ipv6, &cidrs, &hosts); - EXPECT_STREQ("::1", localhost.c_str()); -} - } // namespace doris From d09f8813f0659d2270864fc9d8d555f1d7c2d027 Mon Sep 17 00:00:00 2001 From: zhangdong <493738387@qq.com> Date: Sun, 10 Sep 2023 20:35:57 +0800 Subject: [PATCH 8/8] code format --- be/src/service/backend_options.cpp | 4 ++-- be/src/service/backend_options.h | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/be/src/service/backend_options.cpp b/be/src/service/backend_options.cpp index 9832b2b488ec41..c8325733368e71 100644 --- a/be/src/service/backend_options.cpp +++ b/be/src/service/backend_options.cpp @@ -113,8 +113,8 @@ bool BackendOptions::analyze_priority_cidrs(const std::string& priority_networks return true; } -bool BackendOptions::analyze_localhost(std::string& localhost, bool& bind_ipv6, std::vector* cidrs, - std::vector* hosts) { +bool BackendOptions::analyze_localhost(std::string& localhost, bool& bind_ipv6, + std::vector* cidrs, std::vector* hosts) { std::vector::iterator addr_it = hosts->begin(); if (!cidrs->empty()) { for (; addr_it != hosts->end(); ++addr_it) { diff --git a/be/src/service/backend_options.h b/be/src/service/backend_options.h index 4d41a02a95e0fe..72293373883471 100644 --- a/be/src/service/backend_options.h +++ b/be/src/service/backend_options.h @@ -40,7 +40,7 @@ class BackendOptions { static const char* get_service_bind_address_without_bracket(); static bool analyze_priority_cidrs(const std::string& priority_networks, std::vector* cidrs); - static bool analyze_localhost(std::string& localhost, bool& bind_ipv6, std::vector* cidrs, + static bool analyze_localhost(std::string& localhost, bool& bind_ipv6, std::vector* cidrs, std::vector* hosts); private: