From 4aecbb9c103488dcc67ee978c77dbed138e07947 Mon Sep 17 00:00:00 2001 From: zouxxyy Date: Mon, 8 Sep 2025 12:49:19 +0800 Subject: [PATCH 1/4] v1 --- dev/build_arrow.sh | 22 +++++++++++++++++++--- dev/builddeps-veloxbe.sh | 4 ++-- ep/build-velox/src/build_velox.sh | 11 +++++++++++ 3 files changed, 32 insertions(+), 5 deletions(-) diff --git a/dev/build_arrow.sh b/dev/build_arrow.sh index 4f2bcf2d23f1..a26f04d88840 100755 --- a/dev/build_arrow.sh +++ b/dev/build_arrow.sh @@ -23,6 +23,22 @@ source ${CURRENT_DIR}/build_helper_functions.sh VELOX_ARROW_BUILD_VERSION=15.0.0 ARROW_PREFIX=$CURRENT_DIR/../ep/_ep/arrow_ep BUILD_TYPE=Release +INSTALL_PREFIX=${INSTALL_PREFIX:-"/usr/local"} + +OTHER_ARGUMENTS="" + +for arg in "$@"; do + case $arg in + --install_prefix=*) + INSTALL_PREFIX=("${arg#*=}") + shift # Remove argument name from processing + ;; + *) + OTHER_ARGUMENTS+=("$1") + shift # Remove generic argument from processing + ;; + esac +done function prepare_arrow_build() { mkdir -p ${ARROW_PREFIX}/../ && pushd ${ARROW_PREFIX}/../ && ${SUDO} rm -rf arrow_ep/ @@ -62,14 +78,14 @@ function build_arrow_cpp() { -DARROW_RUNTIME_SIMD_LEVEL=NONE \ -DARROW_WITH_UTF8PROC=OFF \ -DARROW_TESTING=ON \ - -DCMAKE_INSTALL_PREFIX=/usr/local \ + -DCMAKE_INSTALL_PREFIX="${INSTALL_PREFIX}" \ -DCMAKE_BUILD_TYPE=${BUILD_TYPE} \ -DARROW_BUILD_SHARED=OFF \ -DARROW_BUILD_STATIC=ON # Install thrift. cd _build/thrift_ep-prefix/src/thrift_ep-build - ${SUDO} cmake --install ./ --prefix /usr/local/ + ${SUDO} cmake --install ./ --prefix "${INSTALL_PREFIX}"/ popd } @@ -103,7 +119,7 @@ function build_arrow_java() { -Dmaven.test.skip -Drat.skip -Dmaven.gitcommitid.skip -Dcheckstyle.skip -N # Arrow JNI Date Interface CPP libraries - export PKG_CONFIG_PATH=/usr/local/lib64/pkgconfig${PKG_CONFIG_PATH:+:${PKG_CONFIG_PATH}} + export PKG_CONFIG_PATH="${INSTALL_PREFIX}"/lib64/pkgconfig:"${INSTALL_PREFIX}"/lib/pkgconfig${PKG_CONFIG_PATH:+:${PKG_CONFIG_PATH}} mvn generate-resources -Pgenerate-libs-jni-macos-linux -N -Darrow.dataset.jni.dist.dir=$ARROW_INSTALL_DIR \ -DARROW_GANDIVA=OFF -DARROW_JAVA_JNI_ENABLE_GANDIVA=OFF -DARROW_ORC=OFF -DARROW_JAVA_JNI_ENABLE_ORC=OFF \ -Dmaven.test.skip -Drat.skip -Dmaven.gitcommitid.skip -Dcheckstyle.skip -N diff --git a/dev/builddeps-veloxbe.sh b/dev/builddeps-veloxbe.sh index 0eb1a0b2e3ef..cd298ed38297 100755 --- a/dev/builddeps-veloxbe.sh +++ b/dev/builddeps-veloxbe.sh @@ -212,7 +212,7 @@ function build_arrow { get_velox && setup_dependencies fi cd $GLUTEN_DIR/dev - source ./build_arrow.sh + source ./build_arrow.sh --install_prefix=$INSTALL_PREFIX } function build_velox { @@ -222,7 +222,7 @@ function build_velox { ./build_velox.sh --enable_s3=$ENABLE_S3 --enable_gcs=$ENABLE_GCS --build_type=$BUILD_TYPE --enable_hdfs=$ENABLE_HDFS \ --enable_abfs=$ENABLE_ABFS --enable_gpu=$ENABLE_GPU --build_test_utils=$BUILD_TESTS \ --build_tests=$BUILD_VELOX_TESTS --build_benchmarks=$BUILD_VELOX_BENCHMARKS --num_threads=$NUM_THREADS \ - --velox_home=$VELOX_HOME + --velox_home=$VELOX_HOME --install_prefix=$INSTALL_PREFIX } function build_gluten_cpp { diff --git a/ep/build-velox/src/build_velox.sh b/ep/build-velox/src/build_velox.sh index e83b2cdda87b..52bfbf09d060 100755 --- a/ep/build-velox/src/build_velox.sh +++ b/ep/build-velox/src/build_velox.sh @@ -38,6 +38,8 @@ ENABLE_TESTS=OFF BUILD_TEST_UTILS=OFF # Number of threads to use for building. NUM_THREADS="" +# Install prefix for Velox. +INSTALL_PREFIX=${INSTALL_PREFIX:-"/usr/local"} OTHER_ARGUMENTS="" @@ -90,6 +92,10 @@ for arg in "$@"; do NUM_THREADS=("${arg#*=}") shift # Remove argument name from processing ;; + --install_prefix=*) + INSTALL_PREFIX=("${arg#*=}") + shift # Remove argument name from processing + ;; *) OTHER_ARGUMENTS+=("$1") shift # Remove generic argument from processing @@ -138,6 +144,11 @@ function compile { COMPILE_OPTION="$COMPILE_OPTION -DCMAKE_BUILD_TYPE=${BUILD_TYPE}" COMPILE_TYPE=$(if [[ "$BUILD_TYPE" == "debug" ]] || [[ "$BUILD_TYPE" == "Debug" ]]; then echo 'debug'; else echo 'release'; fi) + + if [ $OS == 'Darwin' ]; then + COMPILE_OPTION+=" -DCMAKE_PREFIX_PATH=$INSTALL_PREFIX" + fi + echo "COMPILE_OPTION: "$COMPILE_OPTION NUM_THREADS_OPTS="" From 83e20241dbd34f2862084a0a1a2b0f7d9047185d Mon Sep 17 00:00:00 2001 From: zouxxyy Date: Thu, 11 Sep 2025 12:16:04 +0800 Subject: [PATCH 2/4] fallback --- dev/build_arrow.sh | 15 --------------- dev/builddeps-veloxbe.sh | 8 ++++---- ep/build-velox/src/build_velox.sh | 7 ------- 3 files changed, 4 insertions(+), 26 deletions(-) diff --git a/dev/build_arrow.sh b/dev/build_arrow.sh index a26f04d88840..bb5591bf1eaf 100755 --- a/dev/build_arrow.sh +++ b/dev/build_arrow.sh @@ -25,21 +25,6 @@ ARROW_PREFIX=$CURRENT_DIR/../ep/_ep/arrow_ep BUILD_TYPE=Release INSTALL_PREFIX=${INSTALL_PREFIX:-"/usr/local"} -OTHER_ARGUMENTS="" - -for arg in "$@"; do - case $arg in - --install_prefix=*) - INSTALL_PREFIX=("${arg#*=}") - shift # Remove argument name from processing - ;; - *) - OTHER_ARGUMENTS+=("$1") - shift # Remove generic argument from processing - ;; - esac -done - function prepare_arrow_build() { mkdir -p ${ARROW_PREFIX}/../ && pushd ${ARROW_PREFIX}/../ && ${SUDO} rm -rf arrow_ep/ wget_and_untar https://github.com/apache/arrow/archive/refs/tags/apache-arrow-${VELOX_ARROW_BUILD_VERSION}.tar.gz arrow_ep diff --git a/dev/builddeps-veloxbe.sh b/dev/builddeps-veloxbe.sh index cd298ed38297..34d961e170cd 100755 --- a/dev/builddeps-veloxbe.sh +++ b/dev/builddeps-veloxbe.sh @@ -159,9 +159,9 @@ do done if [[ "$(uname)" == "Darwin" ]]; then - INSTALL_PREFIX=${INSTALL_PREFIX:-${VELOX_HOME}/deps-install} + export INSTALL_PREFIX=${INSTALL_PREFIX:-${VELOX_HOME}/deps-install} else - INSTALL_PREFIX=${INSTALL_PREFIX:-"/usr/local"} + export INSTALL_PREFIX=${INSTALL_PREFIX:-"/usr/local"} fi function concat_velox_param { @@ -212,7 +212,7 @@ function build_arrow { get_velox && setup_dependencies fi cd $GLUTEN_DIR/dev - source ./build_arrow.sh --install_prefix=$INSTALL_PREFIX + source ./build_arrow.sh } function build_velox { @@ -222,7 +222,7 @@ function build_velox { ./build_velox.sh --enable_s3=$ENABLE_S3 --enable_gcs=$ENABLE_GCS --build_type=$BUILD_TYPE --enable_hdfs=$ENABLE_HDFS \ --enable_abfs=$ENABLE_ABFS --enable_gpu=$ENABLE_GPU --build_test_utils=$BUILD_TESTS \ --build_tests=$BUILD_VELOX_TESTS --build_benchmarks=$BUILD_VELOX_BENCHMARKS --num_threads=$NUM_THREADS \ - --velox_home=$VELOX_HOME --install_prefix=$INSTALL_PREFIX + --velox_home=$VELOX_HOME } function build_gluten_cpp { diff --git a/ep/build-velox/src/build_velox.sh b/ep/build-velox/src/build_velox.sh index 52bfbf09d060..6db22845d5dd 100755 --- a/ep/build-velox/src/build_velox.sh +++ b/ep/build-velox/src/build_velox.sh @@ -38,8 +38,6 @@ ENABLE_TESTS=OFF BUILD_TEST_UTILS=OFF # Number of threads to use for building. NUM_THREADS="" -# Install prefix for Velox. -INSTALL_PREFIX=${INSTALL_PREFIX:-"/usr/local"} OTHER_ARGUMENTS="" @@ -144,11 +142,6 @@ function compile { COMPILE_OPTION="$COMPILE_OPTION -DCMAKE_BUILD_TYPE=${BUILD_TYPE}" COMPILE_TYPE=$(if [[ "$BUILD_TYPE" == "debug" ]] || [[ "$BUILD_TYPE" == "Debug" ]]; then echo 'debug'; else echo 'release'; fi) - - if [ $OS == 'Darwin' ]; then - COMPILE_OPTION+=" -DCMAKE_PREFIX_PATH=$INSTALL_PREFIX" - fi - echo "COMPILE_OPTION: "$COMPILE_OPTION NUM_THREADS_OPTS="" From 172562c59fb655447733857d47ad6f6fab41cc56 Mon Sep 17 00:00:00 2001 From: zouxxyy Date: Thu, 11 Sep 2025 12:16:50 +0800 Subject: [PATCH 3/4] fallback --- ep/build-velox/src/build_velox.sh | 4 ---- 1 file changed, 4 deletions(-) diff --git a/ep/build-velox/src/build_velox.sh b/ep/build-velox/src/build_velox.sh index 6db22845d5dd..e83b2cdda87b 100755 --- a/ep/build-velox/src/build_velox.sh +++ b/ep/build-velox/src/build_velox.sh @@ -90,10 +90,6 @@ for arg in "$@"; do NUM_THREADS=("${arg#*=}") shift # Remove argument name from processing ;; - --install_prefix=*) - INSTALL_PREFIX=("${arg#*=}") - shift # Remove argument name from processing - ;; *) OTHER_ARGUMENTS+=("$1") shift # Remove generic argument from processing From 8b1e4603da00ada40a32c2b02f844d9a47535952 Mon Sep 17 00:00:00 2001 From: zouxxyy Date: Thu, 11 Sep 2025 16:01:00 +0800 Subject: [PATCH 4/4] update --- dev/builddeps-veloxbe.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/dev/builddeps-veloxbe.sh b/dev/builddeps-veloxbe.sh index 34d961e170cd..2df2c94e35da 100755 --- a/dev/builddeps-veloxbe.sh +++ b/dev/builddeps-veloxbe.sh @@ -160,8 +160,6 @@ done if [[ "$(uname)" == "Darwin" ]]; then export INSTALL_PREFIX=${INSTALL_PREFIX:-${VELOX_HOME}/deps-install} -else - export INSTALL_PREFIX=${INSTALL_PREFIX:-"/usr/local"} fi function concat_velox_param {