From 1796fdc944fa5c5f58e920ce89ea4c09dafdfb93 Mon Sep 17 00:00:00 2001 From: Xinyi Zou Date: Wed, 14 Sep 2022 12:28:50 +0800 Subject: [PATCH 1/2] Compile Jemalloc separately on thirdparty --- thirdparty/build-thirdparty.sh | 17 +++++++++++++++++ thirdparty/download-thirdparty.sh | 11 +++++++++++ thirdparty/patches/jemalloc_hook.patch | 11 +++++++++++ 3 files changed, 39 insertions(+) create mode 100644 thirdparty/patches/jemalloc_hook.patch diff --git a/thirdparty/build-thirdparty.sh b/thirdparty/build-thirdparty.sh index 71092f21f50a99..0bddd434b29022 100755 --- a/thirdparty/build-thirdparty.sh +++ b/thirdparty/build-thirdparty.sh @@ -1255,6 +1255,22 @@ build_hdfs3() { make install } +# jemalloc +build_jemalloc() { + check_if_source_exist "${JEMALLOC_SOURCE}" + cd "${TP_SOURCE_DIR}/${JEMALLOC_SOURCE}" + + mkdir -p "${BUILD_DIR}" + cd "${BUILD_DIR}" + + cflags='-O3 -fno-omit-frame-pointer -fPIC -g' + CFLAGS="${cflags}" ../configure --prefix=${TP_INSTALL_DIR} --with-jemalloc-prefix=je --enable-prof --disable-cxx --disable-libdl --disable-shared + + make -j "${PARALLEL}" + make install + mv ${TP_INSTALL_DIR}/lib/libjemalloc.a ${TP_INSTALL_DIR}/lib/libjemalloc_doris.a +} + # benchmark build_benchmark() { check_if_source_exist "${BENCHMARK_SOURCE}" @@ -1360,6 +1376,7 @@ build_cyrus_sasl build_librdkafka build_flatbuffers build_orc +build_jemalloc build_arrow build_s2 build_bitshuffle diff --git a/thirdparty/download-thirdparty.sh b/thirdparty/download-thirdparty.sh index 3856993c2f0918..a8a03bfeb43bc2 100755 --- a/thirdparty/download-thirdparty.sh +++ b/thirdparty/download-thirdparty.sh @@ -325,6 +325,17 @@ if [[ "${LIBRDKAFKA_SOURCE}" == "librdkafka-1.8.2" ]]; then fi echo "Finished patching ${LIBRDKAFKA_SOURCE}" +# patch jemalloc, disable JEMALLOC_MANGLE for overloading the memory API. +if [[ "${JEMALLOC_SOURCE}" = "jemalloc-5.2.1" ]]; then + cd "${TP_SOURCE_DIR}/${JEMALLOC_SOURCE}" + if [[ ! -f "${PATCHED_MARK}" ]]; then + patch -p0 < "${TP_PATCH_DIR}/jemalloc_hook.patch" + touch "${PATCHED_MARK}" + fi + cd - +fi +echo "Finished patching ${JEMALLOC_SOURCE}" + # patch hyperscan # https://github.com/intel/hyperscan/issues/292 if [[ "${HYPERSCAN_SOURCE}" == "hyperscan-5.4.0" ]]; then diff --git a/thirdparty/patches/jemalloc_hook.patch b/thirdparty/patches/jemalloc_hook.patch new file mode 100644 index 00000000000000..4a41f395103bf4 --- /dev/null +++ b/thirdparty/patches/jemalloc_hook.patch @@ -0,0 +1,11 @@ +--- include/jemalloc/jemalloc.sh ++++ include/jemalloc/jemalloc.sh +@@ -12,7 +12,7 @@ extern "C" { + EOF + + for hdr in jemalloc_defs.h jemalloc_rename.h jemalloc_macros.h \ +- jemalloc_protos.h jemalloc_typedefs.h jemalloc_mangle.h ; do ++ jemalloc_protos.h jemalloc_typedefs.h ; do + cat "${objroot}include/jemalloc/${hdr}" \ + | grep -v 'Generated from .* by configure\.' \ + | sed -e 's/ $//g' \ No newline at end of file From a23f3a634940cf5d7751463e0997d008f77e1b55 Mon Sep 17 00:00:00 2001 From: Xinyi Zou Date: Wed, 14 Sep 2022 15:08:40 +0800 Subject: [PATCH 2/2] fix sh format --- thirdparty/build-thirdparty.sh | 4 ++-- thirdparty/download-thirdparty.sh | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/thirdparty/build-thirdparty.sh b/thirdparty/build-thirdparty.sh index 0bddd434b29022..dba6483071ba8a 100755 --- a/thirdparty/build-thirdparty.sh +++ b/thirdparty/build-thirdparty.sh @@ -1264,11 +1264,11 @@ build_jemalloc() { cd "${BUILD_DIR}" cflags='-O3 -fno-omit-frame-pointer -fPIC -g' - CFLAGS="${cflags}" ../configure --prefix=${TP_INSTALL_DIR} --with-jemalloc-prefix=je --enable-prof --disable-cxx --disable-libdl --disable-shared + CFLAGS="${cflags}" ../configure --prefix="${TP_INSTALL_DIR}" --with-jemalloc-prefix=je --enable-prof --disable-cxx --disable-libdl --disable-shared make -j "${PARALLEL}" make install - mv ${TP_INSTALL_DIR}/lib/libjemalloc.a ${TP_INSTALL_DIR}/lib/libjemalloc_doris.a + mv "${TP_INSTALL_DIR}"/lib/libjemalloc.a "${TP_INSTALL_DIR}"/lib/libjemalloc_doris.a } # benchmark diff --git a/thirdparty/download-thirdparty.sh b/thirdparty/download-thirdparty.sh index a8a03bfeb43bc2..a4ca817bd8777f 100755 --- a/thirdparty/download-thirdparty.sh +++ b/thirdparty/download-thirdparty.sh @@ -329,7 +329,7 @@ echo "Finished patching ${LIBRDKAFKA_SOURCE}" if [[ "${JEMALLOC_SOURCE}" = "jemalloc-5.2.1" ]]; then cd "${TP_SOURCE_DIR}/${JEMALLOC_SOURCE}" if [[ ! -f "${PATCHED_MARK}" ]]; then - patch -p0 < "${TP_PATCH_DIR}/jemalloc_hook.patch" + patch -p0 <"${TP_PATCH_DIR}/jemalloc_hook.patch" touch "${PATCHED_MARK}" fi cd -