From aa323331bc19265bee85dd49bebd7c4fd705f030 Mon Sep 17 00:00:00 2001 From: zhaochun Date: Tue, 10 Sep 2019 15:55:01 +0800 Subject: [PATCH 1/2] Make CpuInfo::get_current_core work --- be/CMakeLists.txt | 9 ++++++++- be/src/common/CMakeLists.txt | 3 ++- be/src/common/config.h | 5 +++-- be/src/common/env_config.h.in | 24 ++++++++++++++++++++++++ be/src/util/cpu_info.cpp | 25 ++++++++++++++++--------- 5 files changed, 53 insertions(+), 13 deletions(-) create mode 100644 be/src/common/env_config.h.in diff --git a/be/CMakeLists.txt b/be/CMakeLists.txt index a28c9b919f9ae1..9f226c99c3ea69 100644 --- a/be/CMakeLists.txt +++ b/be/CMakeLists.txt @@ -287,6 +287,14 @@ execute_process( OUTPUT_VARIABLE LLVM_MODULE_LIBS OUTPUT_STRIP_TRAILING_WHITESPACE ) + +# Check if functions are supported in this platform. All flags will generated +# in gensrc/build/common/env_config.h. +# You can check funcion here which depends on paltform. Don't forget add this +# to be/src/common/env_config.h.in +include(CheckFunctionExists) +check_function_exists(sched_getcpu HAVE_SCHED_GETCPU) + # compiler flags that are common across debug/release builds # -Wall: Enable all warnings. # -Wno-sign-compare: suppress warnings for comparison between signed and unsigned @@ -461,7 +469,6 @@ include_directories( ${THIRDPARTY_DIR}/include/event/ ) - set(WL_START_GROUP "-Wl,--start-group") set(WL_END_GROUP "-Wl,--end-group") diff --git a/be/src/common/CMakeLists.txt b/be/src/common/CMakeLists.txt index b8f592d77bc197..872bf36e5c019e 100644 --- a/be/src/common/CMakeLists.txt +++ b/be/src/common/CMakeLists.txt @@ -26,4 +26,5 @@ add_library(Common STATIC configbase.cpp ) -#ADD_BE_TEST(resource_tls_test) +# Generate env_config.h according to env_config.h.in +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/env_config.h.in ${GENSRC_DIR}/common/env_config.h) diff --git a/be/src/common/config.h b/be/src/common/config.h index 4a8812478e0edb..d0946386826110 100644 --- a/be/src/common/config.h +++ b/be/src/common/config.h @@ -344,8 +344,9 @@ namespace config { CONF_Bool(enable_prefetch, "true"); - // cpu count - CONF_Int32(flags_num_cores, "32"); + // Number of cores Doris will used, this will effect only when it's greater than 0. + // Otherwise, Doris will use all cores returned from "/proc/cpuinfo". + CONF_Int32(num_cores, "0"); CONF_Bool(thread_creation_fault_injection, "false"); diff --git a/be/src/common/env_config.h.in b/be/src/common/env_config.h.in new file mode 100644 index 00000000000000..d4d5a4434e9498 --- /dev/null +++ b/be/src/common/env_config.h.in @@ -0,0 +1,24 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +namespace doris { + +#cmakedefine HAVE_SCHED_GETCPU @HAVE_SCHED_GETCPU@ + +} diff --git a/be/src/util/cpu_info.cpp b/be/src/util/cpu_info.cpp index 5b351000f931ba..ba3e3435f51c57 100755 --- a/be/src/util/cpu_info.cpp +++ b/be/src/util/cpu_info.cpp @@ -35,6 +35,7 @@ #include #include "common/config.h" +#include "common/env_config.h" #include "gflags/gflags.h" #include "gutil/strings/substitute.h" #include "util/pretty_printer.h" @@ -156,13 +157,13 @@ void CpuInfo::init() { } else { num_cores_ = 1; } - if (config::flags_num_cores > 0) num_cores_ = config::flags_num_cores; + if (config::num_cores > 0) num_cores_ = config::num_cores; max_num_cores_ = get_nprocs_conf(); // Print a warning if something is wrong with sched_getcpu(). #ifdef HAVE_SCHED_GETCPU if (sched_getcpu() == -1) { - LOG(WARNING) << "Kernel does not support getcpu(). Performance may be impacted."; + LOG(WARNING) << "Kernel does not support sched_getcpu(). Performance may be impacted."; } #else LOG(WARNING) << "Built on a system without sched_getcpu() support. Performance may" @@ -284,15 +285,21 @@ void CpuInfo::enable_feature(long flag, bool enable) { } int CpuInfo::get_current_core() { - // sched_getcpu() is not supported on some old kernels/glibcs (like the versions that - // shipped with CentOS 5). In that case just pretend we're always running on CPU 0 - // so that we can build and run with degraded perf. + // sched_getcpu() is not supported on some old kernels/glibcs (like the versions that + // shipped with CentOS 5). In that case just pretend we're always running on CPU 0 + // so that we can build and run with degraded perf. #ifdef HAVE_SCHED_GETCPU - int cpu = sched_getcpu(); - // The syscall may not be supported even if the function exists. - return cpu == -1 ? 0 : cpu; + int cpu = sched_getcpu(); + if (cpu < 0) return 0; + if (cpu >= max_num_cores_) { + LOG_FIRST_N(WARNING, 5) << "sched_getcpu() return value " << cpu + << ", which is greater than get_nprocs_conf() retrun value " << max_num_cores_ + << ", now is " << get_nprocs_conf(); + cpu %= max_num_cores_; + } + return cpu; #else - return 0; + return 0; #endif } From a1c4ca3208c5f294a5625ffd983127fd2bfbdff5 Mon Sep 17 00:00:00 2001 From: zhaochun Date: Tue, 10 Sep 2019 16:46:23 +0800 Subject: [PATCH 2/2] Fix typo --- be/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/be/CMakeLists.txt b/be/CMakeLists.txt index 9f226c99c3ea69..0766638cbaeea6 100644 --- a/be/CMakeLists.txt +++ b/be/CMakeLists.txt @@ -290,7 +290,7 @@ execute_process( # Check if functions are supported in this platform. All flags will generated # in gensrc/build/common/env_config.h. -# You can check funcion here which depends on paltform. Don't forget add this +# You can check funcion here which depends on platform. Don't forget add this # to be/src/common/env_config.h.in include(CheckFunctionExists) check_function_exists(sched_getcpu HAVE_SCHED_GETCPU)