From 98712bac2acc00a540de52c6102947552710f164 Mon Sep 17 00:00:00 2001 From: morningman Date: Thu, 12 Jun 2025 22:53:19 +0800 Subject: [PATCH] [opt](deps) add jindofs in classpath after hadoop libs --- bin/start_be.sh | 8 ++++++++ bin/start_fe.sh | 8 ++++++++ build.sh | 24 +++++++++++++----------- 3 files changed, 29 insertions(+), 11 deletions(-) diff --git a/bin/start_be.sh b/bin/start_be.sh index 391885ecde789f..db553ecb8d2f3f 100755 --- a/bin/start_be.sh +++ b/bin/start_be.sh @@ -243,6 +243,14 @@ if [[ -d "${DORIS_HOME}/lib/hadoop_hdfs/" ]]; then done fi +# add jindofs +# should after jars in lib/hadoop_hdfs/, or it will override the hadoop jars in lib/hadoop_hdfs +if [[ -d "${DORIS_HOME}/lib/java_extensions/jindofs" ]]; then + for f in "${DORIS_HOME}/lib/java_extensions/jindofs"/*.jar; do + DORIS_CLASSPATH="${DORIS_CLASSPATH}:${f}" + done +fi + # add custom_libs to CLASSPATH # ATTN, custom_libs is deprecated, use plugins/java_extensions if [[ -d "${DORIS_HOME}/custom_lib" ]]; then diff --git a/bin/start_fe.sh b/bin/start_fe.sh index 4a728787a828bb..2fd0920df63bb3 100755 --- a/bin/start_fe.sh +++ b/bin/start_fe.sh @@ -224,6 +224,14 @@ if [[ -d "${DORIS_HOME}/custom_lib" ]]; then done fi +# add jindofs +# should after jars in lib/, or it will override the hadoop jars in lib/ +if [[ -d "${DORIS_HOME}/lib/jindofs" ]]; then + for f in "${DORIS_HOME}/lib/jindofs"/*.jar; do + CLASSPATH="${CLASSPATH}:${f}" + done +fi + # add plugins/java_extensions to CLASSPATH if [[ -d "${DORIS_HOME}/plugins/java_extensions" ]]; then for f in "${DORIS_HOME}/plugins/java_extensions"/*.jar; do diff --git a/build.sh b/build.sh index 0a4bed4135bd89..c24e24d3ce4496 100755 --- a/build.sh +++ b/build.sh @@ -755,18 +755,19 @@ if [[ "${BUILD_FE}" -eq 1 ]]; then cp -r -p "${DORIS_HOME}/conf/ldap.conf" "${DORIS_OUTPUT}/fe/conf"/ cp -r -p "${DORIS_HOME}/conf/mysql_ssl_default_certificate" "${DORIS_OUTPUT}/fe/"/ rm -rf "${DORIS_OUTPUT}/fe/lib"/* + install -d "${DORIS_OUTPUT}/fe/lib/jindofs" cp -r -p "${DORIS_HOME}/fe/fe-core/target/lib"/* "${DORIS_OUTPUT}/fe/lib"/ cp -r -p "${DORIS_HOME}/fe/fe-core/target/doris-fe.jar" "${DORIS_OUTPUT}/fe/lib"/ #cp -r -p "${DORIS_HOME}/docs/build/help-resource.zip" "${DORIS_OUTPUT}/fe/lib"/ # copy jindofs jars, only support for Linux x64 or arm if [[ "${TARGET_SYSTEM}" == 'Linux' ]] && [[ "${TARGET_ARCH}" == 'x86_64' ]]; then - cp -r -p "${DORIS_THIRDPARTY}"/installed/jindofs_libs/jindo-core-[0-9]*.jar "${DORIS_OUTPUT}/fe/lib"/ - cp -r -p "${DORIS_THIRDPARTY}"/installed/jindofs_libs/jindo-core-linux-ubuntu22-x86_64-[0-9]*.jar "${DORIS_OUTPUT}/fe/lib"/ - cp -r -p "${DORIS_THIRDPARTY}"/installed/jindofs_libs/jindo-sdk-[0-9]*.jar "${DORIS_OUTPUT}/fe/lib"/ + cp -r -p "${DORIS_THIRDPARTY}"/installed/jindofs_libs/jindo-core-[0-9]*.jar "${DORIS_OUTPUT}/fe/lib/jindofs"/ + cp -r -p "${DORIS_THIRDPARTY}"/installed/jindofs_libs/jindo-core-linux-ubuntu22-x86_64-[0-9]*.jar "${DORIS_OUTPUT}/fe/lib/jindofs"/ + cp -r -p "${DORIS_THIRDPARTY}"/installed/jindofs_libs/jindo-sdk-[0-9]*.jar "${DORIS_OUTPUT}/fe/lib/jindofs"/ elif [[ "${TARGET_SYSTEM}" == 'Linux' ]] && [[ "${TARGET_ARCH}" == 'aarch64' ]]; then - cp -r -p "${DORIS_THIRDPARTY}"/installed/jindofs_libs/jindo-core-linux-el7-aarch64-[0-9]*.jar "${DORIS_OUTPUT}/fe/lib"/ - cp -r -p "${DORIS_THIRDPARTY}"/installed/jindofs_libs/jindo-sdk-[0-9]*.jar "${DORIS_OUTPUT}/fe/lib"/ + cp -r -p "${DORIS_THIRDPARTY}"/installed/jindofs_libs/jindo-core-linux-el7-aarch64-[0-9]*.jar "${DORIS_OUTPUT}/fe/lib/jindofs"/ + cp -r -p "${DORIS_THIRDPARTY}"/installed/jindofs_libs/jindo-sdk-[0-9]*.jar "${DORIS_OUTPUT}/fe/lib/jindofs"/ fi cp -r -p "${DORIS_HOME}/minidump" "${DORIS_OUTPUT}/fe"/ @@ -799,7 +800,7 @@ if [[ "${OUTPUT_BE_BINARY}" -eq 1 ]]; then "${DORIS_OUTPUT}/be/www" \ "${DORIS_OUTPUT}/be/tools/FlameGraph" - cp -r -p "${DORIS_HOME}/be/output/bin"/* "${DORIS_OUTPUT}/be/bin"/ + cp -r -p "${DORIS_HOME}/bin"/*_be.sh "${DORIS_OUTPUT}/be/bin"/ cp -r -p "${DORIS_HOME}/be/output/conf"/* "${DORIS_OUTPUT}/be/conf"/ cp -r -p "${DORIS_HOME}/be/output/dict" "${DORIS_OUTPUT}/be/" @@ -907,13 +908,14 @@ EOF done # copy jindofs jars, only support for Linux x64 or arm + install -d "${DORIS_OUTPUT}/be/lib/java_extensions/jindofs"/ if [[ "${TARGET_SYSTEM}" == 'Linux' ]] && [[ "$TARGET_ARCH" == 'x86_64' ]]; then - cp -r -p "${DORIS_THIRDPARTY}"/installed/jindofs_libs/jindo-core-[0-9]*.jar "${DORIS_OUTPUT}/be/lib/java_extensions/preload-extensions"/ - cp -r -p "${DORIS_THIRDPARTY}"/installed/jindofs_libs/jindo-core-linux-ubuntu22-x86_64-[0-9]*.jar "${DORIS_OUTPUT}/be/lib/java_extensions/preload-extensions"/ - cp -r -p "${DORIS_THIRDPARTY}"/installed/jindofs_libs/jindo-sdk-[0-9]*.jar "${DORIS_OUTPUT}/be/lib/java_extensions/preload-extensions"/ + cp -r -p "${DORIS_THIRDPARTY}"/installed/jindofs_libs/jindo-core-[0-9]*.jar "${DORIS_OUTPUT}/be/lib/java_extensions/jindofs"/ + cp -r -p "${DORIS_THIRDPARTY}"/installed/jindofs_libs/jindo-core-linux-ubuntu22-x86_64-[0-9]*.jar "${DORIS_OUTPUT}/be/lib/java_extensions/jindofs"/ + cp -r -p "${DORIS_THIRDPARTY}"/installed/jindofs_libs/jindo-sdk-[0-9]*.jar "${DORIS_OUTPUT}/be/lib/java_extensions/jindofs"/ elif [[ "${TARGET_SYSTEM}" == 'Linux' ]] && [[ "$TARGET_ARCH" == 'aarch64' ]]; then - cp -r -p "${DORIS_THIRDPARTY}"/installed/jindofs_libs/jindo-core-linux-el7-aarch64-[0-9]*.jar "${DORIS_OUTPUT}/be/lib/java_extensions/preload-extensions"/ - cp -r -p "${DORIS_THIRDPARTY}"/installed/jindofs_libs/jindo-sdk-[0-9]*.jar "${DORIS_OUTPUT}/be/lib/java_extensions/preload-extensions"/ + cp -r -p "${DORIS_THIRDPARTY}"/installed/jindofs_libs/jindo-core-linux-el7-aarch64-[0-9]*.jar "${DORIS_OUTPUT}/be/lib/java_extensions/jindofs"/ + cp -r -p "${DORIS_THIRDPARTY}"/installed/jindofs_libs/jindo-sdk-[0-9]*.jar "${DORIS_OUTPUT}/be/lib/java_extensions/jindofs"/ fi cp -r -p "${DORIS_THIRDPARTY}/installed/webroot"/* "${DORIS_OUTPUT}/be/www"/