apache · xinyiZzz · Jun 20, 2024 · May 31, 2024 · May 31, 2024 · May 31, 2024
@@ -75,7 +75,6 @@ option(USE_LIBCPP "Use libc++" OFF)
 option(USE_MEM_TRACKER, "Use memory tracker" ON)
 option(USE_UNWIND "Use libunwind" ON)
 option(USE_JEMALLOC "Use jemalloc" ON)
-option(USE_JEMALLOC_HOOK "Use jemalloc hook" ON)
 if (OS_MACOSX)
     set(GLIBC_COMPATIBILITY OFF)
     set(USE_LIBCPP ON)
@@ -91,7 +90,6 @@ message(STATUS "GLIBC_COMPATIBILITY is ${GLIBC_COMPATIBILITY}")
 message(STATUS "USE_LIBCPP is ${USE_LIBCPP}")
 message(STATUS "USE_MEM_TRACKER is ${USE_MEM_TRACKER}")
 message(STATUS "USE_JEMALLOC is ${USE_JEMALLOC}")
-message(STATUS "USE_JEMALLOC_HOOK is ${USE_JEMALLOC_HOOK}")
 message(STATUS "USE_UNWIND is ${USE_UNWIND}")
 message(STATUS "ENABLE_PCH is ${ENABLE_PCH}")
 
@@ -350,9 +348,6 @@ endif()
 if (USE_JEMALLOC)
     add_definitions(-DUSE_JEMALLOC)
 endif()
-if (USE_JEMALLOC_HOOK)
-    add_definitions(-DUSE_JEMALLOC_HOOK)
-endif()
 
 # Compile with libunwind
 if (USE_UNWIND)

diff --git a/be/cmake/thirdparty.cmake b/be/cmake/thirdparty.cmake
@@ -58,7 +58,6 @@ add_thirdparty(re2)
 add_thirdparty(hyperscan LIBNAME "lib64/libhs.a")
 add_thirdparty(odbc)
 add_thirdparty(pprof WHOLELIBPATH ${GPERFTOOLS_HOME}/lib/libprofiler.a)
-add_thirdparty(tcmalloc WHOLELIBPATH ${GPERFTOOLS_HOME}/lib/libtcmalloc.a NOTADD)
 add_thirdparty(protobuf)
 add_thirdparty(gtest)
 add_thirdparty(gtest_main)
@@ -77,7 +76,11 @@ add_thirdparty(libz LIBNAME "lib/libz.a")
 add_thirdparty(crypto)
 add_thirdparty(openssl LIBNAME "lib/libssl.a")
 add_thirdparty(leveldb)
-add_thirdparty(jemalloc LIBNAME "lib/libjemalloc_doris.a")
+if (USE_JEMALLOC)
+    add_thirdparty(jemalloc LIBNAME "lib/libjemalloc_doris.a")
+else()
+    add_thirdparty(tcmalloc WHOLELIBPATH ${GPERFTOOLS_HOME}/lib/libtcmalloc.a NOTADD)
+endif()
 add_thirdparty(jemalloc_arrow LIBNAME "lib/libjemalloc_arrow.a")
 
 if (WITH_MYSQL)

diff --git a/be/src/http/action/jeprofile_actions.cpp b/be/src/http/action/jeprofile_actions.cpp
@@ -60,11 +60,7 @@ void JeHeapAction::handle(HttpRequest* req) {
                          << "." << rand() << ".heap";
     const std::string& tmp_file_name_str = tmp_jeprof_file_name.str();
     const char* file_name_ptr = tmp_file_name_str.c_str();
-#ifdef USE_JEMALLOC_HOOK
     int result = jemallctl("prof.dump", nullptr, nullptr, &file_name_ptr, sizeof(const char*));
-#else
-    int result = mallctl("prof.dump", nullptr, nullptr, &file_name_ptr, sizeof(const char*));
-#endif
     std::stringstream response;
     if (result == 0) {
         response << "Jemalloc heap dump success, dump file path: " << tmp_jeprof_file_name.str()

diff --git a/be/src/http/default_path_handlers.cpp b/be/src/http/default_path_handlers.cpp
@@ -115,11 +115,7 @@ void mem_usage_handler(const WebPageHandler::ArgumentMap& args, std::stringstrea
         auto* _opaque = static_cast<std::string*>(opaque);
         _opaque->append(buf);
     };
-#ifdef USE_JEMALLOC_HOOK
     jemalloc_stats_print(write_cb, &tmp, "a");
-#else
-    malloc_stats_print(write_cb, &tmp, "a");
-#endif
     boost::replace_all(tmp, "\n", "<br>");
     (*output) << tmp << "</pre>";
 #else

@@ -25,7 +25,7 @@ set(EXECUTABLE_OUTPUT_PATH "${BUILD_DIR}/src/runtime")
 
 file(GLOB_RECURSE RUNTIME_FILES CONFIGURE_DEPENDS *.cpp *.cc)
 
-if (NOT USE_JEMALLOC OR NOT USE_MEM_TRACKER OR NOT USE_JEMALLOC_HOOK)
+if (NOT USE_JEMALLOC OR NOT USE_MEM_TRACKER)
     list(REMOVE_ITEM RUNTIME_FILES ${CMAKE_CURRENT_SOURCE_DIR}/memory/jemalloc_hook.cpp)
 endif()
 

diff --git a/be/src/runtime/thread_context.h b/be/src/runtime/thread_context.h
@@ -85,7 +85,7 @@
 #define MEMORY_ORPHAN_CHECK() (void)0
 #endif
 
-#if defined(USE_MEM_TRACKER) && !defined(BE_TEST) && defined(USE_JEMALLOC_HOOK)
+#if defined(USE_MEM_TRACKER) && !defined(BE_TEST)
 // Count a code segment memory (memory malloc - memory free) to int64_t
 // Usage example: int64_t scope_mem = 0; { SCOPED_MEM_COUNT_BY_HOOK(&scope_mem); xxx; xxx; }
 #define SCOPED_MEM_COUNT_BY_HOOK(scope_mem) \

diff --git a/be/src/util/mem_info.cpp b/be/src/util/mem_info.cpp
@@ -78,11 +78,7 @@ void MemInfo::refresh_allocator_mem() {
     // the current epoch number, which might be useful to log as a sanity check.
     uint64_t epoch = 0;
     size_t sz = sizeof(epoch);
-#ifdef USE_JEMALLOC_HOOK
     jemallctl("epoch", &epoch, &sz, &epoch, sz);
-#else
-    mallctl("epoch", &epoch, &sz, &epoch, sz);
-#endif
 
     // https://jemalloc.net/jemalloc.3.html
     // https://www.bookstack.cn/read/aliyun-rds-core/4a0cdf677f62feb3.md

diff --git a/be/src/util/mem_info.h b/be/src/util/mem_info.h
@@ -101,15 +101,9 @@ class MemInfo {
 #ifdef USE_JEMALLOC
         size_t value = 0;
         size_t sz = sizeof(value);
-#ifdef USE_JEMALLOC_HOOK
         if (jemallctl(name.c_str(), &value, &sz, nullptr, 0) == 0) {
             return value;
         }
-#else
-        if (mallctl(name.c_str(), &value, &sz, nullptr, 0) == 0) {
-            return value;
-        }
-#endif
 #endif
         return 0;
     }
@@ -130,13 +124,8 @@ class MemInfo {
         if (config::enable_je_purge_dirty_pages) {
             try {
                 // Purge all unused dirty pages for arena <i>, or for all arenas if <i> equals MALLCTL_ARENAS_ALL.
-#ifdef USE_JEMALLOC_HOOK
                 jemallctl(fmt::format("arena.{}.purge", MALLCTL_ARENAS_ALL).c_str(), nullptr,
                           nullptr, nullptr, 0);
-#else
-                mallctl(fmt::format("arena.{}.purge", MALLCTL_ARENAS_ALL).c_str(), nullptr, nullptr,
-                        nullptr, 0);
-#endif
             } catch (...) {
                 LOG(WARNING) << "Purge all unused dirty pages for all arenas failed";
             }

diff --git a/build.sh b/build.sh
@@ -365,39 +365,19 @@ if [[ "${BUILD_TYPE_LOWWER}" == "asan" ]]; then
 elif [[ -z "${USE_JEMALLOC}" ]]; then
     USE_JEMALLOC='ON'
 fi
-if [[ ! -f "${TP_INCLUDE_DIR}/jemalloc/jemalloc_doris_with_prefix.h" ]]; then
+if [[ -f "${TP_INCLUDE_DIR}/jemalloc/jemalloc_doris_with_prefix.h" ]]; then
     # compatible with old thirdparty
-    if [[ -z "${USE_JEMALLOC_HOOK}" ]]; then
-        USE_JEMALLOC_HOOK='ON'
-    fi
-else
-    if [[ -z "${USE_JEMALLOC_HOOK}" ]]; then
-        if [[ "$(uname -s)" != 'Darwin' ]]; then
-            USE_JEMALLOC_HOOK='OFF'
-        else
-            # compile jemalloc on mac have default prefix `je_`, so default use prefix jemalloc to ensure code uniformity.
-            USE_JEMALLOC_HOOK='ON'
-        fi
-    fi
-    # update jemalloc prefix
     rm -rf "${TP_INCLUDE_DIR}/jemalloc/jemalloc.h"
     rm -rf "${TP_LIB_DIR}/libjemalloc_doris.a"
     rm -rf "${TP_LIB_DIR}/libjemalloc_doris_pic.a"
     rm -rf "${TP_INCLUDE_DIR}/rocksdb"
     rm -rf "${TP_LIB_DIR}/librocksdb.a"
-    if [[ "${USE_JEMALLOC_HOOK}" == "ON" ]]; then
-        cp "${TP_INCLUDE_DIR}/jemalloc/jemalloc_doris_with_prefix.h" "${TP_INCLUDE_DIR}/jemalloc/jemalloc.h"
-        cp "${TP_LIB_DIR}/libjemalloc_doris_with_prefix.a" "${TP_LIB_DIR}/libjemalloc_doris.a"
-        cp "${TP_LIB_DIR}/libjemalloc_doris_with_prefix_pic.a" "${TP_LIB_DIR}/libjemalloc_doris_pic.a"
-        cp "${TP_LIB_DIR}/librocksdb_jemalloc_with_prefix.a" "${TP_LIB_DIR}/librocksdb.a"
-        cp -r "${TP_INCLUDE_DIR}/rocksdb_jemalloc_with_prefix" "${TP_INCLUDE_DIR}/rocksdb"
-    else
-        cp "${TP_INCLUDE_DIR}/jemalloc/jemalloc_doris_no_prefix.h" "${TP_INCLUDE_DIR}/jemalloc/jemalloc.h"
-        cp "${TP_LIB_DIR}/libjemalloc_doris_no_prefix.a" "${TP_LIB_DIR}/libjemalloc_doris.a"
-        cp "${TP_LIB_DIR}/libjemalloc_doris_no_prefix_pic.a" "${TP_LIB_DIR}/libjemalloc_doris_pic.a"
-        cp "${TP_LIB_DIR}/librocksdb_jemalloc_no_prefix.a" "${TP_LIB_DIR}/librocksdb.a"
-        cp -r "${TP_INCLUDE_DIR}/rocksdb_jemalloc_no_prefix" "${TP_INCLUDE_DIR}/rocksdb"
-    fi
+
+    mv "${TP_INCLUDE_DIR}/jemalloc/jemalloc_doris_with_prefix.h" "${TP_INCLUDE_DIR}/jemalloc/jemalloc.h"
+    mv "${TP_LIB_DIR}/libjemalloc_doris_with_prefix.a" "${TP_LIB_DIR}/libjemalloc_doris.a"
+    mv "${TP_LIB_DIR}/libjemalloc_doris_with_prefix_pic.a" "${TP_LIB_DIR}/libjemalloc_doris_pic.a"
+    mv "${TP_LIB_DIR}/librocksdb_jemalloc_with_prefix.a" "${TP_LIB_DIR}/librocksdb.a"
+    mv -f "${TP_INCLUDE_DIR}/rocksdb_jemalloc_with_prefix" "${TP_INCLUDE_DIR}/rocksdb"
 fi
 if [[ -z "${USE_BTHREAD_SCANNER}" ]]; then
     USE_BTHREAD_SCANNER='OFF'
@@ -504,7 +484,6 @@ echo "Get params:
     STRIP_DEBUG_INFO            -- ${STRIP_DEBUG_INFO}
     USE_MEM_TRACKER             -- ${USE_MEM_TRACKER}
     USE_JEMALLOC                -- ${USE_JEMALLOC}
-    USE_JEMALLOC_HOOK           -- ${USE_JEMALLOC_HOOK}
     USE_BTHREAD_SCANNER         -- ${USE_BTHREAD_SCANNER}
     ENABLE_STACKTRACE           -- ${ENABLE_STACKTRACE}
     ENABLE_INJECTION_POINT      -- ${ENABLE_INJECTION_POINT}
@@ -608,7 +587,6 @@ if [[ "${BUILD_BE}" -eq 1 ]]; then
         -DENABLE_PCH="${ENABLE_PCH}" \
         -DUSE_MEM_TRACKER="${USE_MEM_TRACKER}" \
         -DUSE_JEMALLOC="${USE_JEMALLOC}" \
-        -DUSE_JEMALLOC_HOOK="${USE_JEMALLOC_HOOK}" \
         -DENABLE_STACKTRACE="${ENABLE_STACKTRACE}" \
         -DUSE_AVX2="${USE_AVX2}" \
         -DGLIBC_COMPATIBILITY="${GLIBC_COMPATIBILITY}" \
@@ -652,7 +630,6 @@ if [[ "${BUILD_CLOUD}" -eq 1 ]]; then
         -DSTRIP_DEBUG_INFO="${STRIP_DEBUG_INFO}" \
         -DUSE_DWARF="${USE_DWARF}" \
         -DUSE_JEMALLOC="${USE_JEMALLOC}" \
-        -DUSE_JEMALLOC_HOOK="${USE_JEMALLOC_HOOK}" \
         -DEXTRA_CXX_FLAGS="${EXTRA_CXX_FLAGS}" \
         -DBUILD_CHECK_META="${BUILD_CHECK_META:-OFF}" \
         "${DORIS_HOME}/cloud/"

@@ -188,9 +188,6 @@ endif ()
 if (USE_JEMALLOC)
     set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -DUSE_JEMALLOC")
 endif()
-if (USE_JEMALLOC_HOOK)
-    set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -DUSE_JEMALLOC_HOOK")
-endif()
 
 if (CMAKE_CXX_COMPILER_VERSION VERSION_GREATER 7.0)
     set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -faligned-new")

@@ -16,7 +16,7 @@ set(COMMON_FILES
     network_util.cpp
 )
 
-if (USE_JEMALLOC AND USE_MEM_TRACKER AND USE_JEMALLOC_HOOK)
+if (USE_JEMALLOC)
     set(COMMON_FILES ${COMMON_FILES}
         jemalloc_hook.cpp
         )

diff --git a/regression-test/pipeline/performance/compile.sh b/regression-test/pipeline/performance/compile.sh
@@ -134,7 +134,6 @@ sudo docker run -i --rm \
                     && export CCACHE_REMOTE_STORAGE=file:///root/ccache \
                     && export EXTRA_CXX_FLAGS=-O3 \
                     && export USE_JEMALLOC='ON' \
-                    && export USE_JEMALLOC_HOOK_WITH_PREFIX='OFF' \
                     && export ENABLE_PCH=OFF \
                     && export CUSTOM_NPM_REGISTRY=https://registry.npmjs.org \
                     && bash build.sh --fe --be --clean 2>&1 | tee build.log"

diff --git a/thirdparty/build-thirdparty.sh b/thirdparty/build-thirdparty.sh
@@ -831,7 +831,7 @@ build_brpc() {
 }
 
 # rocksdb
-build_rocksdb_base() {
+build_rocksdb() {
     check_if_source_exist "${ROCKSDB_SOURCE}"
 
     cd "${TP_SOURCE_DIR}/${ROCKSDB_SOURCE}"
@@ -852,34 +852,9 @@ build_rocksdb_base() {
     ${warning_defaulted_function_deleted} ${warning_unused_but_set_variable} -Wno-pessimizing-move -Wno-range-loop-construct" \
         LDFLAGS="${ldflags}" \
         PORTABLE=1 make USE_RTTI=1 -j "${PARALLEL}" static_lib
-}
-
-build_rocksdb() {
-    build_rocksdb_base
-    rm -rf "${TP_LIB_DIR}/librocksdb_jemalloc_no_prefix.a"
-    rm -rf "${TP_INCLUDE_DIR}/rocksdb_jemalloc_no_prefix"
-    cp librocksdb.a "${TP_LIB_DIR}/librocksdb_jemalloc_no_prefix.a"
-    cp -r include/rocksdb "${TP_INCLUDE_DIR}/rocksdb_jemalloc_no_prefix"
-    strip_lib librocksdb_jemalloc_no_prefix.a
-    # for compatibility with previous doris version
-    rm -rf "${TP_LIB_DIR}/librocksdb.a"
-    rm -rf "${TP_INCLUDE_DIR}/rocksdb"
-    cp "${TP_LIB_DIR}/librocksdb_jemalloc_no_prefix.a" "${TP_LIB_DIR}/librocksdb.a"
-    cp -r "${TP_INCLUDE_DIR}/rocksdb_jemalloc_no_prefix" "${TP_INCLUDE_DIR}/rocksdb"
-}
-
-build_rocksdb_jemalloc_with_prefix() {
-    build_rocksdb_base
-    rm -rf "${TP_LIB_DIR}/librocksdb_jemalloc_with_prefix.a"
-    rm -rf "${TP_INCLUDE_DIR}/rocksdb_jemalloc_with_prefix"
-    cp librocksdb.a "${TP_LIB_DIR}/librocksdb_jemalloc_with_prefix.a"
-    cp -r include/rocksdb "${TP_INCLUDE_DIR}/rocksdb_jemalloc_with_prefix"
-    strip_lib librocksdb_jemalloc_with_prefix.a
-    # for compatibility with previous doris version
-    rm -rf "${TP_LIB_DIR}/librocksdb.a"
-    rm -rf "${TP_INCLUDE_DIR}/rocksdb"
-    cp "${TP_LIB_DIR}/librocksdb_jemalloc_with_prefix.a" "${TP_LIB_DIR}/librocksdb.a"
-    cp -r "${TP_INCLUDE_DIR}/rocksdb_jemalloc_with_prefix" "${TP_INCLUDE_DIR}/rocksdb"
+    cp librocksdb.a ../../installed/lib/librocksdb.a
+    cp -r include/rocksdb ../../installed/include/
+    strip_lib librocksdb.a
 }
 
 # cyrus_sasl
@@ -1518,7 +1493,7 @@ build_hdfs3() {
     strip_lib libhdfs3.a
 }
 
-# jemalloc no prefix
+# jemalloc
 build_jemalloc_doris() {
     check_if_source_exist "${JEMALLOC_DORIS_SOURCE}"
     cd "${TP_SOURCE_DIR}/${JEMALLOC_DORIS_SOURCE}"
@@ -1539,55 +1514,12 @@ build_jemalloc_doris() {
         WITH_LG_PAGE=''
     fi
 
-    CFLAGS="${cflags}" ../configure --prefix="${TP_INSTALL_DIR}" --with-install-suffix="_doris_no_prefix" "${WITH_LG_PAGE}" \
-        --enable-prof --disable-libdl --disable-shared
-
-    make -j "${PARALLEL}"
-    make install
-
-    # for compatibility with previous
-    rm -rf "${TP_INCLUDE_DIR}/jemalloc/jemalloc.h"
-    rm -rf "${TP_LIB_DIR}/libjemalloc_doris.a"
-    rm -rf "${TP_LIB_DIR}/libjemalloc_doris_pic.a"
-    cp "${TP_INCLUDE_DIR}/jemalloc/jemalloc_doris_no_prefix.h" "${TP_INCLUDE_DIR}/jemalloc/jemalloc.h"
-    cp "${TP_LIB_DIR}/libjemalloc_doris_no_prefix.a" "${TP_LIB_DIR}/libjemalloc_doris.a"
-    cp "${TP_LIB_DIR}/libjemalloc_doris_no_prefix_pic.a" "${TP_LIB_DIR}/libjemalloc_doris_pic.a"
-}
-
-# jemalloc with prefix
-build_jemalloc_doris_with_prefix() {
-    check_if_source_exist "${JEMALLOC_DORIS_SOURCE}"
-    cd "${TP_SOURCE_DIR}/${JEMALLOC_DORIS_SOURCE}"
-
-    mkdir -p "${BUILD_DIR}"
-    cd "${BUILD_DIR}"
-
-    cflags='-O3 -fno-omit-frame-pointer -fPIC -g'
-    # Build jemalloc --with-lg-page=16 in order to make the wheel work on both 4k and 64k page arm64 systems.
-    # Jemalloc compiled on a system with page size 4K can only run on a system with the same page size 4K.
-    # If it is run on a system with page size > 4K, an error `unsupported system page size`.
-    # Jemalloc compiled on a system with page size 64K can run on a system with page size < 64K,
-    # but this will waste more memory. Jemalloc does not support dynamic adaptation to the page size of the system.
-    # The reason is that jemalloc will perform some optimizations based on the page size when compiling.
-    if [[ "${MACHINE_TYPE}" == "aarch64" || "${MACHINE_TYPE}" == 'arm64' ]]; then
-        WITH_LG_PAGE='--with-lg-page=16'
-    else
-        WITH_LG_PAGE=''
-    fi
-
-    CFLAGS="${cflags}" ../configure --prefix="${TP_INSTALL_DIR}" --with-install-suffix="_doris_with_prefix" "${WITH_LG_PAGE}" \
+    CFLAGS="${cflags}" ../configure --prefix="${TP_INSTALL_DIR}" --with-install-suffix="_doris" "${WITH_LG_PAGE}" \
         --with-jemalloc-prefix=je --enable-prof --disable-cxx --disable-libdl --disable-shared
 
     make -j "${PARALLEL}"
     make install
-
-    # for compatibility with previous doris version
-    rm -rf "${TP_INCLUDE_DIR}/jemalloc/jemalloc.h"
-    rm -rf "${TP_LIB_DIR}/libjemalloc_doris.a"
-    rm -rf "${TP_LIB_DIR}/libjemalloc_doris_pic.a"
-    cp "${TP_INCLUDE_DIR}/jemalloc/jemalloc_doris_with_prefix.h" "${TP_INCLUDE_DIR}/jemalloc/jemalloc.h"
-    cp "${TP_LIB_DIR}/libjemalloc_doris_with_prefix.a" "${TP_LIB_DIR}/libjemalloc_doris.a"
-    cp "${TP_LIB_DIR}/libjemalloc_doris_with_prefix_pic.a" "${TP_LIB_DIR}/libjemalloc_doris_pic.a"
+    mv "${TP_INCLUDE_DIR}/jemalloc/jemalloc_doris.h" "${TP_INCLUDE_DIR}/jemalloc/jemalloc.h"
 }
 
 # libunwind
@@ -1907,8 +1839,6 @@ if [[ "${#packages[@]}" -eq 0 ]]; then
         brpc
         jemalloc_doris
         rocksdb
-        jemalloc_doris_with_prefix
-        rocksdb_jemalloc_with_prefix
         krb5 # before cyrus_sasl
         cyrus_sasl
         librdkafka