From 62a76f6b0aa6632473a49c8913d276bea20f7cf2 Mon Sep 17 00:00:00 2001 From: "Mingyu Chen (Rayner)" Date: Sat, 14 Jun 2025 15:13:59 +0800 Subject: [PATCH] [opt](deps) add jindofs in classpath after hadoop libs (#51689) Related PR: #49903 Problem Summary: We should add jindofs jars in CLASSPATH, but after hadoop jars. So that we can use the right hadoop deps instead of deps from jindofs. --- bin/start_be.sh | 8 ++++++++ bin/start_fe.sh | 8 ++++++++ build.sh | 24 +++++++++++++----------- 3 files changed, 29 insertions(+), 11 deletions(-) diff --git a/bin/start_be.sh b/bin/start_be.sh index 06024d4238466b..3cb25ff735f1ac 100755 --- a/bin/start_be.sh +++ b/bin/start_be.sh @@ -236,6 +236,14 @@ if [[ -d "${DORIS_HOME}/lib/hadoop_hdfs/" ]]; then done fi +# add jindofs +# should after jars in lib/hadoop_hdfs/, or it will override the hadoop jars in lib/hadoop_hdfs +if [[ -d "${DORIS_HOME}/lib/java_extensions/jindofs" ]]; then + for f in "${DORIS_HOME}/lib/java_extensions/jindofs"/*.jar; do + DORIS_CLASSPATH="${DORIS_CLASSPATH}:${f}" + done +fi + # add custom_libs to CLASSPATH if [[ -d "${DORIS_HOME}/custom_lib" ]]; then for f in "${DORIS_HOME}/custom_lib"/*.jar; do diff --git a/bin/start_fe.sh b/bin/start_fe.sh index 76ac682ef517b0..56cf5523c3e535 100755 --- a/bin/start_fe.sh +++ b/bin/start_fe.sh @@ -223,6 +223,14 @@ if [[ -d "${DORIS_HOME}/custom_lib" ]]; then done fi +# add jindofs +# should after jars in lib/, or it will override the hadoop jars in lib/ +if [[ -d "${DORIS_HOME}/lib/jindofs" ]]; then + for f in "${DORIS_HOME}/lib/jindofs"/*.jar; do + CLASSPATH="${CLASSPATH}:${f}" + done +fi + # make sure the doris-fe.jar is at first order, so that some classed # with same qualified name can be loaded priority from doris-fe.jar CLASSPATH="${DORIS_FE_JAR}:${CLASSPATH}" diff --git a/build.sh b/build.sh index 6dce884f5ff769..2cf95d8732609a 100755 --- a/build.sh +++ b/build.sh @@ -747,18 +747,19 @@ if [[ "${BUILD_FE}" -eq 1 ]]; then cp -r -p "${DORIS_HOME}/conf/ldap.conf" "${DORIS_OUTPUT}/fe/conf"/ cp -r -p "${DORIS_HOME}/conf/mysql_ssl_default_certificate" "${DORIS_OUTPUT}/fe/"/ rm -rf "${DORIS_OUTPUT}/fe/lib"/* + install -d "${DORIS_OUTPUT}/fe/lib/jindofs" cp -r -p "${DORIS_HOME}/fe/fe-core/target/lib"/* "${DORIS_OUTPUT}/fe/lib"/ cp -r -p "${DORIS_HOME}/fe/fe-core/target/doris-fe.jar" "${DORIS_OUTPUT}/fe/lib"/ #cp -r -p "${DORIS_HOME}/docs/build/help-resource.zip" "${DORIS_OUTPUT}/fe/lib"/ # copy jindofs jars, only support for Linux x64 or arm if [[ "${TARGET_SYSTEM}" == 'Linux' ]] && [[ "${TARGET_ARCH}" == 'x86_64' ]]; then - cp -r -p "${DORIS_THIRDPARTY}"/installed/jindofs_libs/jindo-core-[0-9]*.jar "${DORIS_OUTPUT}/fe/lib"/ - cp -r -p "${DORIS_THIRDPARTY}"/installed/jindofs_libs/jindo-core-linux-ubuntu22-x86_64-[0-9]*.jar "${DORIS_OUTPUT}/fe/lib"/ - cp -r -p "${DORIS_THIRDPARTY}"/installed/jindofs_libs/jindo-sdk-[0-9]*.jar "${DORIS_OUTPUT}/fe/lib"/ + cp -r -p "${DORIS_THIRDPARTY}"/installed/jindofs_libs/jindo-core-[0-9]*.jar "${DORIS_OUTPUT}/fe/lib/jindofs"/ + cp -r -p "${DORIS_THIRDPARTY}"/installed/jindofs_libs/jindo-core-linux-ubuntu22-x86_64-[0-9]*.jar "${DORIS_OUTPUT}/fe/lib/jindofs"/ + cp -r -p "${DORIS_THIRDPARTY}"/installed/jindofs_libs/jindo-sdk-[0-9]*.jar "${DORIS_OUTPUT}/fe/lib/jindofs"/ elif [[ "${TARGET_SYSTEM}" == 'Linux' ]] && [[ "${TARGET_ARCH}" == 'aarch64' ]]; then - cp -r -p "${DORIS_THIRDPARTY}"/installed/jindofs_libs/jindo-core-linux-el7-aarch64-[0-9]*.jar "${DORIS_OUTPUT}/fe/lib"/ - cp -r -p "${DORIS_THIRDPARTY}"/installed/jindofs_libs/jindo-sdk-[0-9]*.jar "${DORIS_OUTPUT}/fe/lib"/ + cp -r -p "${DORIS_THIRDPARTY}"/installed/jindofs_libs/jindo-core-linux-el7-aarch64-[0-9]*.jar "${DORIS_OUTPUT}/fe/lib/jindofs"/ + cp -r -p "${DORIS_THIRDPARTY}"/installed/jindofs_libs/jindo-sdk-[0-9]*.jar "${DORIS_OUTPUT}/fe/lib/jindofs"/ fi cp -r -p "${DORIS_HOME}/minidump" "${DORIS_OUTPUT}/fe"/ @@ -787,7 +788,7 @@ if [[ "${OUTPUT_BE_BINARY}" -eq 1 ]]; then "${DORIS_OUTPUT}/be/www" \ "${DORIS_OUTPUT}/be/tools/FlameGraph" - cp -r -p "${DORIS_HOME}/be/output/bin"/* "${DORIS_OUTPUT}/be/bin"/ + cp -r -p "${DORIS_HOME}/bin"/*_be.sh "${DORIS_OUTPUT}/be/bin"/ cp -r -p "${DORIS_HOME}/be/output/conf"/* "${DORIS_OUTPUT}/be/conf"/ cp -r -p "${DORIS_HOME}/be/output/dict" "${DORIS_OUTPUT}/be/" @@ -892,13 +893,14 @@ EOF done # copy jindofs jars, only support for Linux x64 or arm + install -d "${DORIS_OUTPUT}/be/lib/java_extensions/jindofs"/ if [[ "${TARGET_SYSTEM}" == 'Linux' ]] && [[ "$TARGET_ARCH" == 'x86_64' ]]; then - cp -r -p "${DORIS_THIRDPARTY}"/installed/jindofs_libs/jindo-core-[0-9]*.jar "${DORIS_OUTPUT}/be/lib/java_extensions/preload-extensions"/ - cp -r -p "${DORIS_THIRDPARTY}"/installed/jindofs_libs/jindo-core-linux-ubuntu22-x86_64-[0-9]*.jar "${DORIS_OUTPUT}/be/lib/java_extensions/preload-extensions"/ - cp -r -p "${DORIS_THIRDPARTY}"/installed/jindofs_libs/jindo-sdk-[0-9]*.jar "${DORIS_OUTPUT}/be/lib/java_extensions/preload-extensions"/ + cp -r -p "${DORIS_THIRDPARTY}"/installed/jindofs_libs/jindo-core-[0-9]*.jar "${DORIS_OUTPUT}/be/lib/java_extensions/jindofs"/ + cp -r -p "${DORIS_THIRDPARTY}"/installed/jindofs_libs/jindo-core-linux-ubuntu22-x86_64-[0-9]*.jar "${DORIS_OUTPUT}/be/lib/java_extensions/jindofs"/ + cp -r -p "${DORIS_THIRDPARTY}"/installed/jindofs_libs/jindo-sdk-[0-9]*.jar "${DORIS_OUTPUT}/be/lib/java_extensions/jindofs"/ elif [[ "${TARGET_SYSTEM}" == 'Linux' ]] && [[ "$TARGET_ARCH" == 'aarch64' ]]; then - cp -r -p "${DORIS_THIRDPARTY}"/installed/jindofs_libs/jindo-core-linux-el7-aarch64-[0-9]*.jar "${DORIS_OUTPUT}/be/lib/java_extensions/preload-extensions"/ - cp -r -p "${DORIS_THIRDPARTY}"/installed/jindofs_libs/jindo-sdk-[0-9]*.jar "${DORIS_OUTPUT}/be/lib/java_extensions/preload-extensions"/ + cp -r -p "${DORIS_THIRDPARTY}"/installed/jindofs_libs/jindo-core-linux-el7-aarch64-[0-9]*.jar "${DORIS_OUTPUT}/be/lib/java_extensions/jindofs"/ + cp -r -p "${DORIS_THIRDPARTY}"/installed/jindofs_libs/jindo-sdk-[0-9]*.jar "${DORIS_OUTPUT}/be/lib/java_extensions/jindofs"/ fi cp -r -p "${DORIS_THIRDPARTY}/installed/webroot"/* "${DORIS_OUTPUT}/be/www"/