From 6e3fed91558acd6b6fe127204a593f11148a2480 Mon Sep 17 00:00:00 2001 From: Xinyi Zou Date: Fri, 31 May 2024 11:08:54 +0800 Subject: [PATCH 1/7] Revert "[fix](memory) Fix BE compile in Mac with no prefix Jemalloc (#34886)" This reverts commit 75fee8f03015e064d37da5422c097e0e187cb230. --- build.sh | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/build.sh b/build.sh index e4fd7d48e9ccf2..c320ba12ee1591 100755 --- a/build.sh +++ b/build.sh @@ -372,12 +372,7 @@ if [[ ! -f "${TP_INCLUDE_DIR}/jemalloc/jemalloc_doris_with_prefix.h" ]]; then fi else if [[ -z "${USE_JEMALLOC_HOOK}" ]]; then - if [[ "$(uname -s)" != 'Darwin' ]]; then - USE_JEMALLOC_HOOK='OFF' - else - # compile jemalloc on mac have default prefix `je_`, so default use prefix jemalloc to ensure code uniformity. - USE_JEMALLOC_HOOK='ON' - fi + USE_JEMALLOC_HOOK='OFF' fi # update jemalloc prefix rm -rf "${TP_INCLUDE_DIR}/jemalloc/jemalloc.h" From f174bd6157de449306d516931a04e0719ac98f38 Mon Sep 17 00:00:00 2001 From: Xinyi Zou Date: Fri, 31 May 2024 11:11:18 +0800 Subject: [PATCH 2/7] Revert "[fix](memory) Fix BE compile in Mac with Jemalloc (#34750)" This reverts commit 667f5b5e1c8959516c09da63afeec17bd6c9537d. --- be/src/runtime/CMakeLists.txt | 2 +- build.sh | 6 +----- cloud/src/common/CMakeLists.txt | 2 +- 3 files changed, 3 insertions(+), 7 deletions(-) diff --git a/be/src/runtime/CMakeLists.txt b/be/src/runtime/CMakeLists.txt index 3bfec93edfc083..70db5b4cb52cfd 100644 --- a/be/src/runtime/CMakeLists.txt +++ b/be/src/runtime/CMakeLists.txt @@ -25,7 +25,7 @@ set(EXECUTABLE_OUTPUT_PATH "${BUILD_DIR}/src/runtime") file(GLOB_RECURSE RUNTIME_FILES CONFIGURE_DEPENDS *.cpp *.cc) -if (NOT USE_JEMALLOC OR NOT USE_MEM_TRACKER OR NOT USE_JEMALLOC_HOOK) +if (NOT USE_JEMALLOC OR NOT USE_JEMALLOC_HOOK) list(REMOVE_ITEM RUNTIME_FILES ${CMAKE_CURRENT_SOURCE_DIR}/memory/jemalloc_hook.cpp) endif() diff --git a/build.sh b/build.sh index c320ba12ee1591..2c3db123055a5f 100755 --- a/build.sh +++ b/build.sh @@ -30,9 +30,6 @@ set -eo pipefail ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" export DORIS_HOME="${ROOT}" -if [[ -z "${DORIS_THIRDPARTY}" ]]; then - export DORIS_THIRDPARTY="${DORIS_HOME}/thirdparty" -fi export TP_INCLUDE_DIR="${DORIS_THIRDPARTY}/installed/include" export TP_LIB_DIR="${DORIS_THIRDPARTY}/installed/lib" @@ -359,8 +356,7 @@ if [[ -z "${USE_MEM_TRACKER}" ]]; then USE_MEM_TRACKER='OFF' fi fi -BUILD_TYPE_LOWWER=$(echo "${BUILD_TYPE}" | tr '[:upper:]' '[:lower:]') -if [[ "${BUILD_TYPE_LOWWER}" == "asan" ]]; then +if [[ "${BUILD_TYPE,,}" == "asan" ]]; then USE_JEMALLOC='OFF' elif [[ -z "${USE_JEMALLOC}" ]]; then USE_JEMALLOC='ON' diff --git a/cloud/src/common/CMakeLists.txt b/cloud/src/common/CMakeLists.txt index 0b6b2794f6fcc4..b18947b04a1e5b 100644 --- a/cloud/src/common/CMakeLists.txt +++ b/cloud/src/common/CMakeLists.txt @@ -16,7 +16,7 @@ set(COMMON_FILES network_util.cpp ) -if (USE_JEMALLOC AND USE_MEM_TRACKER AND USE_JEMALLOC_HOOK) +if (USE_JEMALLOC AND USE_JEMALLOC_HOOK) set(COMMON_FILES ${COMMON_FILES} jemalloc_hook.cpp ) From 2e985595c338e85051b8ce7c461af3e9b0948c43 Mon Sep 17 00:00:00 2001 From: Xinyi Zou Date: Fri, 31 May 2024 11:11:24 +0800 Subject: [PATCH 3/7] Revert "[fix](memory) Fix build.sh thirdparty path error for jemalloc (#34643)" This reverts commit 875b85c02b73b5c39b879dcf48ae7485d6c97fd3. --- build.sh | 10 +++++----- thirdparty/build-thirdparty.sh | 9 ++------- 2 files changed, 7 insertions(+), 12 deletions(-) diff --git a/build.sh b/build.sh index 2c3db123055a5f..999fdae6959e39 100755 --- a/build.sh +++ b/build.sh @@ -30,8 +30,10 @@ set -eo pipefail ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" export DORIS_HOME="${ROOT}" -export TP_INCLUDE_DIR="${DORIS_THIRDPARTY}/installed/include" -export TP_LIB_DIR="${DORIS_THIRDPARTY}/installed/lib" +export TP_DIR="${ROOT}/thirdparty" +export TP_INSTALL_DIR="${TP_DIR:-.}/installed" +export TP_INCLUDE_DIR="${TP_INSTALL_DIR}/include" +export TP_LIB_DIR="${TP_INSTALL_DIR}/lib" . "${DORIS_HOME}/env.sh" @@ -356,9 +358,7 @@ if [[ -z "${USE_MEM_TRACKER}" ]]; then USE_MEM_TRACKER='OFF' fi fi -if [[ "${BUILD_TYPE,,}" == "asan" ]]; then - USE_JEMALLOC='OFF' -elif [[ -z "${USE_JEMALLOC}" ]]; then +if [[ -z "${USE_JEMALLOC}" ]]; then USE_JEMALLOC='ON' fi if [[ ! -f "${TP_INCLUDE_DIR}/jemalloc/jemalloc_doris_with_prefix.h" ]]; then diff --git a/thirdparty/build-thirdparty.sh b/thirdparty/build-thirdparty.sh index 610a516bbd07c0..2bedc927706379 100755 --- a/thirdparty/build-thirdparty.sh +++ b/thirdparty/build-thirdparty.sh @@ -875,11 +875,6 @@ build_rocksdb_jemalloc_with_prefix() { cp librocksdb.a "${TP_LIB_DIR}/librocksdb_jemalloc_with_prefix.a" cp -r include/rocksdb "${TP_INCLUDE_DIR}/rocksdb_jemalloc_with_prefix" strip_lib librocksdb_jemalloc_with_prefix.a - # for compatibility with previous doris version - rm -rf "${TP_LIB_DIR}/librocksdb.a" - rm -rf "${TP_INCLUDE_DIR}/rocksdb" - cp "${TP_LIB_DIR}/librocksdb_jemalloc_with_prefix.a" "${TP_LIB_DIR}/librocksdb.a" - cp -r "${TP_INCLUDE_DIR}/rocksdb_jemalloc_with_prefix" "${TP_INCLUDE_DIR}/rocksdb" } # cyrus_sasl @@ -1890,10 +1885,10 @@ if [[ "${#packages[@]}" -eq 0 ]]; then thrift leveldb brpc - jemalloc_doris - rocksdb jemalloc_doris_with_prefix rocksdb_jemalloc_with_prefix + jemalloc_doris + rocksdb krb5 # before cyrus_sasl cyrus_sasl librdkafka From a0e78b1b230b9431342898cb408b8ee1e84b3f32 Mon Sep 17 00:00:00 2001 From: Xinyi Zou Date: Fri, 31 May 2024 11:21:52 +0800 Subject: [PATCH 4/7] Revert "[fix](memory) Fix jdk17 and jemalloc hook not compatible on some envs (#34578)" This reverts commit 673c61adef3b47529816e4b28cc35abd8dc45803. --- be/CMakeLists.txt | 5 -- be/src/common/daemon.cpp | 3 - be/src/http/action/jeprofile_actions.cpp | 4 - be/src/http/action/pprof_actions.cpp | 3 - be/src/http/default_path_handlers.cpp | 8 +- be/src/pch/pch.h | 3 - be/src/runtime/CMakeLists.txt | 2 +- be/src/runtime/memory/tcmalloc_hook.h | 3 - be/src/runtime/thread_context.h | 5 +- be/src/service/doris_main.cpp | 3 - be/src/util/mem_info.cpp | 4 - be/src/util/mem_info.h | 18 +---- bin/start_be.sh | 2 - build.sh | 36 --------- cloud/CMakeLists.txt | 3 - cloud/cmake/thirdparty.cmake | 1 - cloud/src/common/CMakeLists.txt | 2 +- .../pipeline/performance/compile.sh | 3 +- thirdparty/build-thirdparty.sh | 79 ++----------------- 19 files changed, 14 insertions(+), 173 deletions(-) diff --git a/be/CMakeLists.txt b/be/CMakeLists.txt index fafaff15fe164c..6d6ab2cec4c2f4 100644 --- a/be/CMakeLists.txt +++ b/be/CMakeLists.txt @@ -72,7 +72,6 @@ option(USE_LIBCPP "Use libc++" OFF) option(USE_MEM_TRACKER, "Use memory tracker" ON) option(USE_UNWIND "Use libunwind" ON) option(USE_JEMALLOC "Use jemalloc" ON) -option(USE_JEMALLOC_HOOK "Use jemalloc hook" ON) if (OS_MACOSX) set(GLIBC_COMPATIBILITY OFF) set(USE_LIBCPP ON) @@ -88,7 +87,6 @@ message(STATUS "GLIBC_COMPATIBILITY is ${GLIBC_COMPATIBILITY}") message(STATUS "USE_LIBCPP is ${USE_LIBCPP}") message(STATUS "USE_MEM_TRACKER is ${USE_MEM_TRACKER}") message(STATUS "USE_JEMALLOC is ${USE_JEMALLOC}") -message(STATUS "USE_JEMALLOC_HOOK is ${USE_JEMALLOC_HOOK}") message(STATUS "USE_UNWIND is ${USE_UNWIND}") message(STATUS "ENABLE_PCH is ${ENABLE_PCH}") @@ -347,9 +345,6 @@ endif() if (USE_JEMALLOC) add_definitions(-DUSE_JEMALLOC) endif() -if (USE_JEMALLOC_HOOK) - add_definitions(-DUSE_JEMALLOC_HOOK) -endif() # Compile with libunwind if (USE_UNWIND) diff --git a/be/src/common/daemon.cpp b/be/src/common/daemon.cpp index 647f0275e7a39b..059910be3ef9be 100644 --- a/be/src/common/daemon.cpp +++ b/be/src/common/daemon.cpp @@ -20,10 +20,7 @@ // IWYU pragma: no_include #include // IWYU pragma: keep #include -#if !defined(__SANITIZE_ADDRESS__) && !defined(ADDRESS_SANITIZER) && !defined(LEAK_SANITIZER) && \ - !defined(THREAD_SANITIZER) && !defined(USE_JEMALLOC) #include // IWYU pragma: keep -#endif // IWYU pragma: no_include #include #include diff --git a/be/src/http/action/jeprofile_actions.cpp b/be/src/http/action/jeprofile_actions.cpp index 703902096bc9ab..1ed763cfb747f4 100644 --- a/be/src/http/action/jeprofile_actions.cpp +++ b/be/src/http/action/jeprofile_actions.cpp @@ -60,11 +60,7 @@ void JeHeapAction::handle(HttpRequest* req) { << "." << rand() << ".heap"; const std::string& tmp_file_name_str = tmp_jeprof_file_name.str(); const char* file_name_ptr = tmp_file_name_str.c_str(); -#ifdef USE_JEMALLOC_HOOK int result = jemallctl("prof.dump", nullptr, nullptr, &file_name_ptr, sizeof(const char*)); -#else - int result = mallctl("prof.dump", nullptr, nullptr, &file_name_ptr, sizeof(const char*)); -#endif std::stringstream response; if (result == 0) { response << "Jemalloc heap dump success, dump file path: " << tmp_jeprof_file_name.str() diff --git a/be/src/http/action/pprof_actions.cpp b/be/src/http/action/pprof_actions.cpp index 2894ba01781382..1cbe2163b9374f 100644 --- a/be/src/http/action/pprof_actions.cpp +++ b/be/src/http/action/pprof_actions.cpp @@ -17,12 +17,9 @@ #include "http/action/pprof_actions.h" -#if !defined(__SANITIZE_ADDRESS__) && !defined(ADDRESS_SANITIZER) && !defined(LEAK_SANITIZER) && \ - !defined(THREAD_SANITIZER) && !defined(USE_JEMALLOC) #include // IWYU pragma: keep #include // IWYU pragma: keep #include // IWYU pragma: keep -#endif #include #include diff --git a/be/src/http/default_path_handlers.cpp b/be/src/http/default_path_handlers.cpp index f9003373f8257e..fa46ca9d04ead2 100644 --- a/be/src/http/default_path_handlers.cpp +++ b/be/src/http/default_path_handlers.cpp @@ -22,9 +22,7 @@ #include #ifdef USE_JEMALLOC #include "jemalloc/jemalloc.h" -#endif -#if !defined(__SANITIZE_ADDRESS__) && !defined(ADDRESS_SANITIZER) && !defined(LEAK_SANITIZER) && \ - !defined(THREAD_SANITIZER) && !defined(USE_JEMALLOC) +#else #include #endif @@ -114,11 +112,7 @@ void mem_usage_handler(const WebPageHandler::ArgumentMap& args, std::stringstrea auto* _opaque = static_cast(opaque); _opaque->append(buf); }; -#ifdef USE_JEMALLOC_HOOK jemalloc_stats_print(write_cb, &tmp, "a"); -#else - malloc_stats_print(write_cb, &tmp, "a"); -#endif boost::replace_all(tmp, "\n", "
"); (*output) << tmp << ""; #else diff --git a/be/src/pch/pch.h b/be/src/pch/pch.h index ca6fbea8463482..8c7ef3ea3a9fb8 100644 --- a/be/src/pch/pch.h +++ b/be/src/pch/pch.h @@ -231,14 +231,11 @@ #include #include -#if !defined(__SANITIZE_ADDRESS__) && !defined(ADDRESS_SANITIZER) && !defined(LEAK_SANITIZER) && \ - !defined(THREAD_SANITIZER) && !defined(USE_JEMALLOC) // gperftools headers #include #include #include #include -#endif // hs headers #include diff --git a/be/src/runtime/CMakeLists.txt b/be/src/runtime/CMakeLists.txt index 70db5b4cb52cfd..a0b3b799a764cb 100644 --- a/be/src/runtime/CMakeLists.txt +++ b/be/src/runtime/CMakeLists.txt @@ -25,7 +25,7 @@ set(EXECUTABLE_OUTPUT_PATH "${BUILD_DIR}/src/runtime") file(GLOB_RECURSE RUNTIME_FILES CONFIGURE_DEPENDS *.cpp *.cc) -if (NOT USE_JEMALLOC OR NOT USE_JEMALLOC_HOOK) +if (NOT USE_JEMALLOC OR NOT USE_MEM_TRACKER) list(REMOVE_ITEM RUNTIME_FILES ${CMAKE_CURRENT_SOURCE_DIR}/memory/jemalloc_hook.cpp) endif() diff --git a/be/src/runtime/memory/tcmalloc_hook.h b/be/src/runtime/memory/tcmalloc_hook.h index 9f7bee16b0fe78..9cc88be1b081bc 100644 --- a/be/src/runtime/memory/tcmalloc_hook.h +++ b/be/src/runtime/memory/tcmalloc_hook.h @@ -17,8 +17,6 @@ #pragma once -#if !defined(__SANITIZE_ADDRESS__) && !defined(ADDRESS_SANITIZER) && !defined(LEAK_SANITIZER) && \ - !defined(THREAD_SANITIZER) && !defined(USE_JEMALLOC) #include #include #include @@ -55,4 +53,3 @@ void init_hook() { // MallocHook::RemoveNewHook(&new_hook); // MallocHook::RemoveDeleteHook(&delete_hook); // } -#endif diff --git a/be/src/runtime/thread_context.h b/be/src/runtime/thread_context.h index f3698abbe004df..20346be62384d3 100644 --- a/be/src/runtime/thread_context.h +++ b/be/src/runtime/thread_context.h @@ -85,7 +85,7 @@ #define MEMORY_ORPHAN_CHECK() (void)0 #endif -#if defined(USE_MEM_TRACKER) && !defined(BE_TEST) && defined(USE_JEMALLOC_HOOK) +#if defined(USE_MEM_TRACKER) && !defined(BE_TEST) // Count a code segment memory (memory malloc - memory free) to int64_t // Usage example: int64_t scope_mem = 0; { SCOPED_MEM_COUNT_BY_HOOK(&scope_mem); xxx; xxx; } #define SCOPED_MEM_COUNT_BY_HOOK(scope_mem) \ @@ -93,7 +93,6 @@ // Count a code segment memory (memory malloc - memory free) to MemTracker. // Compared to count `scope_mem`, MemTracker is easier to observe from the outside and is thread-safe. -// Usage example: std::unique_ptr tracker = std::make_unique("first_tracker"); // { SCOPED_CONSUME_MEM_TRACKER_BY_HOOK(_mem_tracker.get()); xxx; xxx; } #define SCOPED_CONSUME_MEM_TRACKER_BY_HOOK(mem_tracker) \ auto VARNAME_LINENUM(add_mem_consumer) = doris::AddThreadMemTrackerConsumerByHook(mem_tracker) @@ -108,8 +107,6 @@ auto VARNAME_LINENUM(scope_skip_memory_check) = doris::ScopeSkipMemoryCheck() #define SKIP_LARGE_MEMORY_CHECK(...) \ - do { \ - doris::ThreadLocalHandle::create_thread_local_if_not_exits(); \ doris::thread_context()->skip_large_memory_check++; \ DEFER({ \ doris::thread_context()->skip_large_memory_check--; \ diff --git a/be/src/service/doris_main.cpp b/be/src/service/doris_main.cpp index 92d3452dcb136d..ebc515cc12810c 100644 --- a/be/src/service/doris_main.cpp +++ b/be/src/service/doris_main.cpp @@ -24,10 +24,7 @@ // IWYU pragma: no_include #include // IWYU pragma: keep #include -#if !defined(__SANITIZE_ADDRESS__) && !defined(ADDRESS_SANITIZER) && !defined(LEAK_SANITIZER) && \ - !defined(THREAD_SANITIZER) && !defined(USE_JEMALLOC) #include // IWYU pragma: keep -#endif #include #include #include diff --git a/be/src/util/mem_info.cpp b/be/src/util/mem_info.cpp index 3196115ef06fc2..72a47fa076aa7d 100644 --- a/be/src/util/mem_info.cpp +++ b/be/src/util/mem_info.cpp @@ -91,11 +91,7 @@ void MemInfo::refresh_allocator_mem() { // the current epoch number, which might be useful to log as a sanity check. uint64_t epoch = 0; size_t sz = sizeof(epoch); -#ifdef USE_JEMALLOC_HOOK jemallctl("epoch", &epoch, &sz, &epoch, sz); -#else - mallctl("epoch", &epoch, &sz, &epoch, sz); -#endif // https://jemalloc.net/jemalloc.3.html // https://www.bookstack.cn/read/aliyun-rds-core/4a0cdf677f62feb3.md diff --git a/be/src/util/mem_info.h b/be/src/util/mem_info.h index e44646bf328e71..dc4b0e0d2986a4 100644 --- a/be/src/util/mem_info.h +++ b/be/src/util/mem_info.h @@ -36,9 +36,7 @@ #include "common/logging.h" #ifdef USE_JEMALLOC #include "jemalloc/jemalloc.h" -#endif -#if !defined(__SANITIZE_ADDRESS__) && !defined(ADDRESS_SANITIZER) && !defined(LEAK_SANITIZER) && \ - !defined(THREAD_SANITIZER) && !defined(USE_JEMALLOC) +#else #include #endif #include "common/config.h" @@ -102,8 +100,7 @@ class MemInfo { } static inline int64_t get_tc_metrics(const std::string& name) { -#if !defined(__SANITIZE_ADDRESS__) && !defined(ADDRESS_SANITIZER) && !defined(LEAK_SANITIZER) && \ - !defined(THREAD_SANITIZER) && !defined(USE_JEMALLOC) +#ifndef USE_JEMALLOC size_t value = 0; MallocExtension::instance()->GetNumericProperty(name.c_str(), &value); return value; @@ -114,15 +111,9 @@ class MemInfo { #ifdef USE_JEMALLOC size_t value = 0; size_t sz = sizeof(value); -#ifdef USE_JEMALLOC_HOOK if (jemallctl(name.c_str(), &value, &sz, nullptr, 0) == 0) { return value; } -#else - if (mallctl(name.c_str(), &value, &sz, nullptr, 0) == 0) { - return value; - } -#endif #endif return 0; } @@ -143,13 +134,8 @@ class MemInfo { if (config::enable_je_purge_dirty_pages) { try { // Purge all unused dirty pages for arena , or for all arenas if equals MALLCTL_ARENAS_ALL. -#ifdef USE_JEMALLOC_HOOK jemallctl(fmt::format("arena.{}.purge", MALLCTL_ARENAS_ALL).c_str(), nullptr, nullptr, nullptr, 0); -#else - mallctl(fmt::format("arena.{}.purge", MALLCTL_ARENAS_ALL).c_str(), nullptr, nullptr, - nullptr, 0); -#endif } catch (...) { LOG(WARNING) << "Purge all unused dirty pages for all arenas failed"; } diff --git a/bin/start_be.sh b/bin/start_be.sh index 7d3b5c5679ebac..4f4eb2b1bccc85 100755 --- a/bin/start_be.sh +++ b/bin/start_be.sh @@ -387,11 +387,9 @@ fi if [[ -z ${JEMALLOC_PROF_PRFIX} ]]; then export JEMALLOC_CONF="${JEMALLOC_CONF},prof_prefix:" - export MALLOC_CONF="${JEMALLOC_CONF},prof_prefix:" else JEMALLOC_PROF_PRFIX="${DORIS_HOME}/log/${JEMALLOC_PROF_PRFIX}" export JEMALLOC_CONF="${JEMALLOC_CONF},prof_prefix:${JEMALLOC_PROF_PRFIX}" - export MALLOC_CONF="${JEMALLOC_CONF},prof_prefix:${JEMALLOC_PROF_PRFIX}" fi if [[ "${RUN_DAEMON}" -eq 1 ]]; then diff --git a/build.sh b/build.sh index 999fdae6959e39..690d617d510ec8 100755 --- a/build.sh +++ b/build.sh @@ -30,10 +30,6 @@ set -eo pipefail ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" export DORIS_HOME="${ROOT}" -export TP_DIR="${ROOT}/thirdparty" -export TP_INSTALL_DIR="${TP_DIR:-.}/installed" -export TP_INCLUDE_DIR="${TP_INSTALL_DIR}/include" -export TP_LIB_DIR="${TP_INSTALL_DIR}/lib" . "${DORIS_HOME}/env.sh" @@ -361,35 +357,6 @@ fi if [[ -z "${USE_JEMALLOC}" ]]; then USE_JEMALLOC='ON' fi -if [[ ! -f "${TP_INCLUDE_DIR}/jemalloc/jemalloc_doris_with_prefix.h" ]]; then - # compatible with old thirdparty - if [[ -z "${USE_JEMALLOC_HOOK}" ]]; then - USE_JEMALLOC_HOOK='ON' - fi -else - if [[ -z "${USE_JEMALLOC_HOOK}" ]]; then - USE_JEMALLOC_HOOK='OFF' - fi - # update jemalloc prefix - rm -rf "${TP_INCLUDE_DIR}/jemalloc/jemalloc.h" - rm -rf "${TP_LIB_DIR}/libjemalloc_doris.a" - rm -rf "${TP_LIB_DIR}/libjemalloc_doris_pic.a" - rm -rf "${TP_INCLUDE_DIR}/rocksdb" - rm -rf "${TP_LIB_DIR}/librocksdb.a" - if [[ "${USE_JEMALLOC_HOOK}" == "ON" ]]; then - cp "${TP_INCLUDE_DIR}/jemalloc/jemalloc_doris_with_prefix.h" "${TP_INCLUDE_DIR}/jemalloc/jemalloc.h" - cp "${TP_LIB_DIR}/libjemalloc_doris_with_prefix.a" "${TP_LIB_DIR}/libjemalloc_doris.a" - cp "${TP_LIB_DIR}/libjemalloc_doris_with_prefix_pic.a" "${TP_LIB_DIR}/libjemalloc_doris_pic.a" - cp "${TP_LIB_DIR}/librocksdb_jemalloc_with_prefix.a" "${TP_LIB_DIR}/librocksdb.a" - cp -r "${TP_INCLUDE_DIR}/rocksdb_jemalloc_with_prefix" "${TP_INCLUDE_DIR}/rocksdb" - else - cp "${TP_INCLUDE_DIR}/jemalloc/jemalloc_doris_no_prefix.h" "${TP_INCLUDE_DIR}/jemalloc/jemalloc.h" - cp "${TP_LIB_DIR}/libjemalloc_doris_no_prefix.a" "${TP_LIB_DIR}/libjemalloc_doris.a" - cp "${TP_LIB_DIR}/libjemalloc_doris_no_prefix_pic.a" "${TP_LIB_DIR}/libjemalloc_doris_pic.a" - cp "${TP_LIB_DIR}/librocksdb_jemalloc_no_prefix.a" "${TP_LIB_DIR}/librocksdb.a" - cp -r "${TP_INCLUDE_DIR}/rocksdb_jemalloc_no_prefix" "${TP_INCLUDE_DIR}/rocksdb" - fi -fi if [[ -z "${USE_BTHREAD_SCANNER}" ]]; then USE_BTHREAD_SCANNER='OFF' fi @@ -495,7 +462,6 @@ echo "Get params: STRIP_DEBUG_INFO -- ${STRIP_DEBUG_INFO} USE_MEM_TRACKER -- ${USE_MEM_TRACKER} USE_JEMALLOC -- ${USE_JEMALLOC} - USE_JEMALLOC_HOOK -- ${USE_JEMALLOC_HOOK} USE_BTHREAD_SCANNER -- ${USE_BTHREAD_SCANNER} ENABLE_STACKTRACE -- ${ENABLE_STACKTRACE} ENABLE_INJECTION_POINT -- ${ENABLE_INJECTION_POINT} @@ -599,7 +565,6 @@ if [[ "${BUILD_BE}" -eq 1 ]]; then -DENABLE_PCH="${ENABLE_PCH}" \ -DUSE_MEM_TRACKER="${USE_MEM_TRACKER}" \ -DUSE_JEMALLOC="${USE_JEMALLOC}" \ - -DUSE_JEMALLOC_HOOK="${USE_JEMALLOC_HOOK}" \ -DENABLE_STACKTRACE="${ENABLE_STACKTRACE}" \ -DUSE_AVX2="${USE_AVX2}" \ -DGLIBC_COMPATIBILITY="${GLIBC_COMPATIBILITY}" \ @@ -643,7 +608,6 @@ if [[ "${BUILD_CLOUD}" -eq 1 ]]; then -DSTRIP_DEBUG_INFO="${STRIP_DEBUG_INFO}" \ -DUSE_DWARF="${USE_DWARF}" \ -DUSE_JEMALLOC="${USE_JEMALLOC}" \ - -DUSE_JEMALLOC_HOOK="${USE_JEMALLOC_HOOK}" \ -DEXTRA_CXX_FLAGS="${EXTRA_CXX_FLAGS}" \ -DBUILD_CHECK_META="${BUILD_CHECK_META:-OFF}" \ "${DORIS_HOME}/cloud/" diff --git a/cloud/CMakeLists.txt b/cloud/CMakeLists.txt index 22469ef4d37b2d..6d3b72ba7ffa00 100644 --- a/cloud/CMakeLists.txt +++ b/cloud/CMakeLists.txt @@ -188,9 +188,6 @@ endif () if (USE_JEMALLOC) set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -DUSE_JEMALLOC") endif() -if (USE_JEMALLOC_HOOK) - set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -DUSE_JEMALLOC_HOOK") -endif() if (CMAKE_CXX_COMPILER_VERSION VERSION_GREATER 7.0) set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -faligned-new") diff --git a/cloud/cmake/thirdparty.cmake b/cloud/cmake/thirdparty.cmake index 0e148896bfcc1e..83c10d2be9ed42 100644 --- a/cloud/cmake/thirdparty.cmake +++ b/cloud/cmake/thirdparty.cmake @@ -61,7 +61,6 @@ add_thirdparty(thrift) add_thirdparty(crypto) add_thirdparty(openssl LIBNAME "lib/libssl.a") add_thirdparty(jemalloc LIBNAME "lib/libjemalloc_doris.a") -add_thirdparty(jemalloc_arrow LIBNAME "lib/libjemalloc_arrow.a") add_thirdparty(leveldb) # Required by brpc add_thirdparty(brpc LIB64) add_thirdparty(rocksdb) # For local storage mocking diff --git a/cloud/src/common/CMakeLists.txt b/cloud/src/common/CMakeLists.txt index b18947b04a1e5b..a2abfb075bf934 100644 --- a/cloud/src/common/CMakeLists.txt +++ b/cloud/src/common/CMakeLists.txt @@ -16,7 +16,7 @@ set(COMMON_FILES network_util.cpp ) -if (USE_JEMALLOC AND USE_JEMALLOC_HOOK) +if (USE_JEMALLOC) set(COMMON_FILES ${COMMON_FILES} jemalloc_hook.cpp ) diff --git a/regression-test/pipeline/performance/compile.sh b/regression-test/pipeline/performance/compile.sh index fbef81937317ff..76ab06574cbed3 100644 --- a/regression-test/pipeline/performance/compile.sh +++ b/regression-test/pipeline/performance/compile.sh @@ -128,13 +128,12 @@ sudo docker run -i --rm \ -v "${teamcity_build_checkoutDir}":/root/doris \ "${docker_image}" \ /bin/bash -c "mkdir -p ${git_storage_path} \ - && cp -r /root/git/* ${git_storage_path}/ \ + && cp -r /root/git/* ${git_storage_path}/ \ && cd /root/doris \ && export CCACHE_LOGFILE=/tmp/cache.debug \ && export CCACHE_REMOTE_STORAGE=file:///root/ccache \ && export EXTRA_CXX_FLAGS=-O3 \ && export USE_JEMALLOC='ON' \ - && export USE_JEMALLOC_HOOK_WITH_PREFIX='OFF' \ && export ENABLE_PCH=OFF \ && export CUSTOM_NPM_REGISTRY=https://registry.npmjs.org \ && bash build.sh --fe --be --clean 2>&1 | tee build.log" diff --git a/thirdparty/build-thirdparty.sh b/thirdparty/build-thirdparty.sh index 2bedc927706379..8b431e13371a24 100755 --- a/thirdparty/build-thirdparty.sh +++ b/thirdparty/build-thirdparty.sh @@ -831,7 +831,7 @@ build_brpc() { } # rocksdb -build_rocksdb_base() { +build_rocksdb() { check_if_source_exist "${ROCKSDB_SOURCE}" cd "${TP_SOURCE_DIR}/${ROCKSDB_SOURCE}" @@ -852,29 +852,9 @@ build_rocksdb_base() { ${warning_defaulted_function_deleted} ${warning_unused_but_set_variable} -Wno-pessimizing-move -Wno-range-loop-construct" \ LDFLAGS="${ldflags}" \ PORTABLE=1 make USE_RTTI=1 -j "${PARALLEL}" static_lib -} - -build_rocksdb() { - build_rocksdb_base - rm -rf "${TP_LIB_DIR}/librocksdb_jemalloc_no_prefix.a" - rm -rf "${TP_INCLUDE_DIR}/rocksdb_jemalloc_no_prefix" - cp librocksdb.a "${TP_LIB_DIR}/librocksdb_jemalloc_no_prefix.a" - cp -r include/rocksdb "${TP_INCLUDE_DIR}/rocksdb_jemalloc_no_prefix" - strip_lib librocksdb_jemalloc_no_prefix.a - # for compatibility with previous doris version - rm -rf "${TP_LIB_DIR}/librocksdb.a" - rm -rf "${TP_INCLUDE_DIR}/rocksdb" - cp "${TP_LIB_DIR}/librocksdb_jemalloc_no_prefix.a" "${TP_LIB_DIR}/librocksdb.a" - cp -r "${TP_INCLUDE_DIR}/rocksdb_jemalloc_no_prefix" "${TP_INCLUDE_DIR}/rocksdb" -} - -build_rocksdb_jemalloc_with_prefix() { - build_rocksdb_base - rm -rf "${TP_LIB_DIR}/librocksdb_jemalloc_with_prefix.a" - rm -rf "${TP_INCLUDE_DIR}/rocksdb_jemalloc_with_prefix" - cp librocksdb.a "${TP_LIB_DIR}/librocksdb_jemalloc_with_prefix.a" - cp -r include/rocksdb "${TP_INCLUDE_DIR}/rocksdb_jemalloc_with_prefix" - strip_lib librocksdb_jemalloc_with_prefix.a + cp librocksdb.a ../../installed/lib/librocksdb.a + cp -r include/rocksdb ../../installed/include/ + strip_lib librocksdb.a } # cyrus_sasl @@ -1513,7 +1493,7 @@ build_hdfs3() { strip_lib libhdfs3.a } -# jemalloc no prefix +# jemalloc build_jemalloc_doris() { check_if_source_exist "${JEMALLOC_DORIS_SOURCE}" cd "${TP_SOURCE_DIR}/${JEMALLOC_DORIS_SOURCE}" @@ -1534,55 +1514,12 @@ build_jemalloc_doris() { WITH_LG_PAGE='' fi - CFLAGS="${cflags}" ../configure --prefix="${TP_INSTALL_DIR}" --with-install-suffix="_doris_no_prefix" "${WITH_LG_PAGE}" \ - --enable-prof --disable-libdl --disable-shared - - make -j "${PARALLEL}" - make install - - # for compatibility with previous - rm -rf "${TP_INCLUDE_DIR}/jemalloc/jemalloc.h" - rm -rf "${TP_LIB_DIR}/libjemalloc_doris.a" - rm -rf "${TP_LIB_DIR}/libjemalloc_doris_pic.a" - cp "${TP_INCLUDE_DIR}/jemalloc/jemalloc_doris_no_prefix.h" "${TP_INCLUDE_DIR}/jemalloc/jemalloc.h" - cp "${TP_LIB_DIR}/libjemalloc_doris_no_prefix.a" "${TP_LIB_DIR}/libjemalloc_doris.a" - cp "${TP_LIB_DIR}/libjemalloc_doris_no_prefix_pic.a" "${TP_LIB_DIR}/libjemalloc_doris_pic.a" -} - -# jemalloc with prefix -build_jemalloc_doris_with_prefix() { - check_if_source_exist "${JEMALLOC_DORIS_SOURCE}" - cd "${TP_SOURCE_DIR}/${JEMALLOC_DORIS_SOURCE}" - - mkdir -p "${BUILD_DIR}" - cd "${BUILD_DIR}" - - cflags='-O3 -fno-omit-frame-pointer -fPIC -g' - # Build jemalloc --with-lg-page=16 in order to make the wheel work on both 4k and 64k page arm64 systems. - # Jemalloc compiled on a system with page size 4K can only run on a system with the same page size 4K. - # If it is run on a system with page size > 4K, an error `unsupported system page size`. - # Jemalloc compiled on a system with page size 64K can run on a system with page size < 64K, - # but this will waste more memory. Jemalloc does not support dynamic adaptation to the page size of the system. - # The reason is that jemalloc will perform some optimizations based on the page size when compiling. - if [[ "${MACHINE_TYPE}" == "aarch64" || "${MACHINE_TYPE}" == 'arm64' ]]; then - WITH_LG_PAGE='--with-lg-page=16' - else - WITH_LG_PAGE='' - fi - - CFLAGS="${cflags}" ../configure --prefix="${TP_INSTALL_DIR}" --with-install-suffix="_doris_with_prefix" "${WITH_LG_PAGE}" \ + CFLAGS="${cflags}" ../configure --prefix="${TP_INSTALL_DIR}" --with-install-suffix="_doris" "${WITH_LG_PAGE}" \ --with-jemalloc-prefix=je --enable-prof --disable-cxx --disable-libdl --disable-shared make -j "${PARALLEL}" make install - - # for compatibility with previous doris version - rm -rf "${TP_INCLUDE_DIR}/jemalloc/jemalloc.h" - rm -rf "${TP_LIB_DIR}/libjemalloc_doris.a" - rm -rf "${TP_LIB_DIR}/libjemalloc_doris_pic.a" - cp "${TP_INCLUDE_DIR}/jemalloc/jemalloc_doris_with_prefix.h" "${TP_INCLUDE_DIR}/jemalloc/jemalloc.h" - cp "${TP_LIB_DIR}/libjemalloc_doris_with_prefix.a" "${TP_LIB_DIR}/libjemalloc_doris.a" - cp "${TP_LIB_DIR}/libjemalloc_doris_with_prefix_pic.a" "${TP_LIB_DIR}/libjemalloc_doris_pic.a" + mv "${TP_INCLUDE_DIR}/jemalloc/jemalloc_doris.h" "${TP_INCLUDE_DIR}/jemalloc/jemalloc.h" } # libunwind @@ -1885,8 +1822,6 @@ if [[ "${#packages[@]}" -eq 0 ]]; then thrift leveldb brpc - jemalloc_doris_with_prefix - rocksdb_jemalloc_with_prefix jemalloc_doris rocksdb krb5 # before cyrus_sasl From b9c92e5dc520049487da6ee270b0daa4e8bb5637 Mon Sep 17 00:00:00 2001 From: Xinyi Zou Date: Fri, 31 May 2024 12:22:09 +0800 Subject: [PATCH 5/7] Revert Fix jdk17 and jemalloc hook not compatible on some envs --- be/src/common/daemon.cpp | 3 +++ be/src/http/action/pprof_actions.cpp | 3 +++ be/src/http/default_path_handlers.cpp | 4 +++- be/src/pch/pch.h | 3 +++ be/src/runtime/memory/tcmalloc_hook.h | 3 +++ be/src/runtime/thread_context.h | 3 +++ be/src/service/doris_main.cpp | 3 +++ be/src/util/mem_info.h | 4 +++- bin/start_be.sh | 2 ++ build.sh | 5 +++++ cloud/cmake/thirdparty.cmake | 1 + regression-test/pipeline/performance/compile.sh | 2 +- 12 files changed, 33 insertions(+), 3 deletions(-) diff --git a/be/src/common/daemon.cpp b/be/src/common/daemon.cpp index 059910be3ef9be..647f0275e7a39b 100644 --- a/be/src/common/daemon.cpp +++ b/be/src/common/daemon.cpp @@ -20,7 +20,10 @@ // IWYU pragma: no_include #include // IWYU pragma: keep #include +#if !defined(__SANITIZE_ADDRESS__) && !defined(ADDRESS_SANITIZER) && !defined(LEAK_SANITIZER) && \ + !defined(THREAD_SANITIZER) && !defined(USE_JEMALLOC) #include // IWYU pragma: keep +#endif // IWYU pragma: no_include #include #include diff --git a/be/src/http/action/pprof_actions.cpp b/be/src/http/action/pprof_actions.cpp index 1cbe2163b9374f..2894ba01781382 100644 --- a/be/src/http/action/pprof_actions.cpp +++ b/be/src/http/action/pprof_actions.cpp @@ -17,9 +17,12 @@ #include "http/action/pprof_actions.h" +#if !defined(__SANITIZE_ADDRESS__) && !defined(ADDRESS_SANITIZER) && !defined(LEAK_SANITIZER) && \ + !defined(THREAD_SANITIZER) && !defined(USE_JEMALLOC) #include // IWYU pragma: keep #include // IWYU pragma: keep #include // IWYU pragma: keep +#endif #include #include diff --git a/be/src/http/default_path_handlers.cpp b/be/src/http/default_path_handlers.cpp index fa46ca9d04ead2..5d0689e2a8a7a9 100644 --- a/be/src/http/default_path_handlers.cpp +++ b/be/src/http/default_path_handlers.cpp @@ -22,7 +22,9 @@ #include #ifdef USE_JEMALLOC #include "jemalloc/jemalloc.h" -#else +#endif +#if !defined(__SANITIZE_ADDRESS__) && !defined(ADDRESS_SANITIZER) && !defined(LEAK_SANITIZER) && \ + !defined(THREAD_SANITIZER) && !defined(USE_JEMALLOC) #include #endif diff --git a/be/src/pch/pch.h b/be/src/pch/pch.h index 8c7ef3ea3a9fb8..ca6fbea8463482 100644 --- a/be/src/pch/pch.h +++ b/be/src/pch/pch.h @@ -231,11 +231,14 @@ #include #include +#if !defined(__SANITIZE_ADDRESS__) && !defined(ADDRESS_SANITIZER) && !defined(LEAK_SANITIZER) && \ + !defined(THREAD_SANITIZER) && !defined(USE_JEMALLOC) // gperftools headers #include #include #include #include +#endif // hs headers #include diff --git a/be/src/runtime/memory/tcmalloc_hook.h b/be/src/runtime/memory/tcmalloc_hook.h index 9cc88be1b081bc..9f7bee16b0fe78 100644 --- a/be/src/runtime/memory/tcmalloc_hook.h +++ b/be/src/runtime/memory/tcmalloc_hook.h @@ -17,6 +17,8 @@ #pragma once +#if !defined(__SANITIZE_ADDRESS__) && !defined(ADDRESS_SANITIZER) && !defined(LEAK_SANITIZER) && \ + !defined(THREAD_SANITIZER) && !defined(USE_JEMALLOC) #include #include #include @@ -53,3 +55,4 @@ void init_hook() { // MallocHook::RemoveNewHook(&new_hook); // MallocHook::RemoveDeleteHook(&delete_hook); // } +#endif diff --git a/be/src/runtime/thread_context.h b/be/src/runtime/thread_context.h index 20346be62384d3..45f64a3739ae1a 100644 --- a/be/src/runtime/thread_context.h +++ b/be/src/runtime/thread_context.h @@ -93,6 +93,7 @@ // Count a code segment memory (memory malloc - memory free) to MemTracker. // Compared to count `scope_mem`, MemTracker is easier to observe from the outside and is thread-safe. +// Usage example: std::unique_ptr tracker = std::make_unique("first_tracker"); // { SCOPED_CONSUME_MEM_TRACKER_BY_HOOK(_mem_tracker.get()); xxx; xxx; } #define SCOPED_CONSUME_MEM_TRACKER_BY_HOOK(mem_tracker) \ auto VARNAME_LINENUM(add_mem_consumer) = doris::AddThreadMemTrackerConsumerByHook(mem_tracker) @@ -107,6 +108,8 @@ auto VARNAME_LINENUM(scope_skip_memory_check) = doris::ScopeSkipMemoryCheck() #define SKIP_LARGE_MEMORY_CHECK(...) \ + do { \ + doris::ThreadLocalHandle::create_thread_local_if_not_exits(); \ doris::thread_context()->skip_large_memory_check++; \ DEFER({ \ doris::thread_context()->skip_large_memory_check--; \ diff --git a/be/src/service/doris_main.cpp b/be/src/service/doris_main.cpp index ebc515cc12810c..92d3452dcb136d 100644 --- a/be/src/service/doris_main.cpp +++ b/be/src/service/doris_main.cpp @@ -24,7 +24,10 @@ // IWYU pragma: no_include #include // IWYU pragma: keep #include +#if !defined(__SANITIZE_ADDRESS__) && !defined(ADDRESS_SANITIZER) && !defined(LEAK_SANITIZER) && \ + !defined(THREAD_SANITIZER) && !defined(USE_JEMALLOC) #include // IWYU pragma: keep +#endif #include #include #include diff --git a/be/src/util/mem_info.h b/be/src/util/mem_info.h index dc4b0e0d2986a4..82de2856853c32 100644 --- a/be/src/util/mem_info.h +++ b/be/src/util/mem_info.h @@ -36,7 +36,9 @@ #include "common/logging.h" #ifdef USE_JEMALLOC #include "jemalloc/jemalloc.h" -#else +#endif +#if !defined(__SANITIZE_ADDRESS__) && !defined(ADDRESS_SANITIZER) && !defined(LEAK_SANITIZER) && \ + !defined(THREAD_SANITIZER) && !defined(USE_JEMALLOC) #include #endif #include "common/config.h" diff --git a/bin/start_be.sh b/bin/start_be.sh index 4f4eb2b1bccc85..7d3b5c5679ebac 100755 --- a/bin/start_be.sh +++ b/bin/start_be.sh @@ -387,9 +387,11 @@ fi if [[ -z ${JEMALLOC_PROF_PRFIX} ]]; then export JEMALLOC_CONF="${JEMALLOC_CONF},prof_prefix:" + export MALLOC_CONF="${JEMALLOC_CONF},prof_prefix:" else JEMALLOC_PROF_PRFIX="${DORIS_HOME}/log/${JEMALLOC_PROF_PRFIX}" export JEMALLOC_CONF="${JEMALLOC_CONF},prof_prefix:${JEMALLOC_PROF_PRFIX}" + export MALLOC_CONF="${JEMALLOC_CONF},prof_prefix:${JEMALLOC_PROF_PRFIX}" fi if [[ "${RUN_DAEMON}" -eq 1 ]]; then diff --git a/build.sh b/build.sh index 690d617d510ec8..321c27ee1acfa8 100755 --- a/build.sh +++ b/build.sh @@ -30,6 +30,11 @@ set -eo pipefail ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" export DORIS_HOME="${ROOT}" +if [[ -z "${DORIS_THIRDPARTY}" ]]; then + export DORIS_THIRDPARTY="${DORIS_HOME}/thirdparty" +fi +export TP_INCLUDE_DIR="${DORIS_THIRDPARTY}/installed/include" +export TP_LIB_DIR="${DORIS_THIRDPARTY}/installed/lib" . "${DORIS_HOME}/env.sh" diff --git a/cloud/cmake/thirdparty.cmake b/cloud/cmake/thirdparty.cmake index 83c10d2be9ed42..0e148896bfcc1e 100644 --- a/cloud/cmake/thirdparty.cmake +++ b/cloud/cmake/thirdparty.cmake @@ -61,6 +61,7 @@ add_thirdparty(thrift) add_thirdparty(crypto) add_thirdparty(openssl LIBNAME "lib/libssl.a") add_thirdparty(jemalloc LIBNAME "lib/libjemalloc_doris.a") +add_thirdparty(jemalloc_arrow LIBNAME "lib/libjemalloc_arrow.a") add_thirdparty(leveldb) # Required by brpc add_thirdparty(brpc LIB64) add_thirdparty(rocksdb) # For local storage mocking diff --git a/regression-test/pipeline/performance/compile.sh b/regression-test/pipeline/performance/compile.sh index 76ab06574cbed3..3a668cd3fdfdd3 100644 --- a/regression-test/pipeline/performance/compile.sh +++ b/regression-test/pipeline/performance/compile.sh @@ -128,7 +128,7 @@ sudo docker run -i --rm \ -v "${teamcity_build_checkoutDir}":/root/doris \ "${docker_image}" \ /bin/bash -c "mkdir -p ${git_storage_path} \ - && cp -r /root/git/* ${git_storage_path}/ \ + && cp -r /root/git/* ${git_storage_path}/ \ && cd /root/doris \ && export CCACHE_LOGFILE=/tmp/cache.debug \ && export CCACHE_REMOTE_STORAGE=file:///root/ccache \ From 9bfb8e175de2416f81b084886d67f2d98c40fecc Mon Sep 17 00:00:00 2001 From: Xinyi Zou Date: Fri, 31 May 2024 15:28:42 +0800 Subject: [PATCH 6/7] Revert Fix jdk17 and jemalloc hook not compatible on some envs --- be/cmake/thirdparty.cmake | 7 +++++-- build.sh | 19 ++++++++++++++++++- 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/be/cmake/thirdparty.cmake b/be/cmake/thirdparty.cmake index 78f28fe72dc4e8..e1b6b0a637ff4e 100644 --- a/be/cmake/thirdparty.cmake +++ b/be/cmake/thirdparty.cmake @@ -58,7 +58,6 @@ add_thirdparty(re2) add_thirdparty(hyperscan LIBNAME "lib64/libhs.a") add_thirdparty(odbc) add_thirdparty(pprof WHOLELIBPATH ${GPERFTOOLS_HOME}/lib/libprofiler.a) -add_thirdparty(tcmalloc WHOLELIBPATH ${GPERFTOOLS_HOME}/lib/libtcmalloc.a NOTADD) add_thirdparty(protobuf) add_thirdparty(gtest) add_thirdparty(gtest_main) @@ -77,7 +76,11 @@ add_thirdparty(libz LIBNAME "lib/libz.a") add_thirdparty(crypto) add_thirdparty(openssl LIBNAME "lib/libssl.a") add_thirdparty(leveldb) -add_thirdparty(jemalloc LIBNAME "lib/libjemalloc_doris.a") +if (USE_JEMALLOC) + add_thirdparty(jemalloc LIBNAME "lib/libjemalloc_doris.a") +else() + add_thirdparty(tcmalloc WHOLELIBPATH ${GPERFTOOLS_HOME}/lib/libtcmalloc.a NOTADD) +endif() add_thirdparty(jemalloc_arrow LIBNAME "lib/libjemalloc_arrow.a") if (WITH_MYSQL) diff --git a/build.sh b/build.sh index 321c27ee1acfa8..88a8c5bf96d211 100755 --- a/build.sh +++ b/build.sh @@ -359,9 +359,26 @@ if [[ -z "${USE_MEM_TRACKER}" ]]; then USE_MEM_TRACKER='OFF' fi fi -if [[ -z "${USE_JEMALLOC}" ]]; then +BUILD_TYPE_LOWWER=$(echo "${BUILD_TYPE}" | tr '[:upper:]' '[:lower:]') +if [[ "${BUILD_TYPE_LOWWER}" == "asan" ]]; then + USE_JEMALLOC='OFF' +elif [[ -z "${USE_JEMALLOC}" ]]; then USE_JEMALLOC='ON' fi +if [[ -f "${TP_INCLUDE_DIR}/jemalloc/jemalloc_doris_with_prefix.h" ]]; then + # compatible with old thirdparty + rm -rf "${TP_INCLUDE_DIR}/jemalloc/jemalloc.h" + rm -rf "${TP_LIB_DIR}/libjemalloc_doris.a" + rm -rf "${TP_LIB_DIR}/libjemalloc_doris_pic.a" + rm -rf "${TP_INCLUDE_DIR}/rocksdb" + rm -rf "${TP_LIB_DIR}/librocksdb.a" + + mv "${TP_INCLUDE_DIR}/jemalloc/jemalloc_doris_with_prefix.h" "${TP_INCLUDE_DIR}/jemalloc/jemalloc.h" + mv "${TP_LIB_DIR}/libjemalloc_doris_with_prefix.a" "${TP_LIB_DIR}/libjemalloc_doris.a" + mv "${TP_LIB_DIR}/libjemalloc_doris_with_prefix_pic.a" "${TP_LIB_DIR}/libjemalloc_doris_pic.a" + mv "${TP_LIB_DIR}/librocksdb_jemalloc_with_prefix.a" "${TP_LIB_DIR}/librocksdb.a" + mv -f "${TP_INCLUDE_DIR}/rocksdb_jemalloc_with_prefix" "${TP_INCLUDE_DIR}/rocksdb" +fi if [[ -z "${USE_BTHREAD_SCANNER}" ]]; then USE_BTHREAD_SCANNER='OFF' fi From ab8259c78b496952cc4883a03850673f6458b218 Mon Sep 17 00:00:00 2001 From: Xinyi Zou Date: Fri, 31 May 2024 15:38:32 +0800 Subject: [PATCH 7/7] Revert Fix jdk17 and jemalloc hook not compatible on some envs --- be/src/util/mem_info.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/be/src/util/mem_info.h b/be/src/util/mem_info.h index 82de2856853c32..ea839465977d9f 100644 --- a/be/src/util/mem_info.h +++ b/be/src/util/mem_info.h @@ -102,7 +102,8 @@ class MemInfo { } static inline int64_t get_tc_metrics(const std::string& name) { -#ifndef USE_JEMALLOC +#if !defined(__SANITIZE_ADDRESS__) && !defined(ADDRESS_SANITIZER) && !defined(LEAK_SANITIZER) && \ + !defined(THREAD_SANITIZER) && !defined(USE_JEMALLOC) size_t value = 0; MallocExtension::instance()->GetNumericProperty(name.c_str(), &value); return value;