From f1917a86c6be553b8733dce344bcb24864263833 Mon Sep 17 00:00:00 2001 From: Hiroyuki Sato Date: Mon, 19 May 2025 16:09:27 +0900 Subject: [PATCH 1/3] GH-46496: [CI][Dev] Fix shellcheck SC2086 errors in ci/scripts directory --- .pre-commit-config.yaml | 18 ++++++++++++++++++ ci/scripts/csharp_build.sh | 2 +- ci/scripts/csharp_pack.sh | 2 +- ci/scripts/install_chromedriver.sh | 2 +- ci/scripts/install_cmake.sh | 22 +++++++++++----------- ci/scripts/install_emscripten.sh | 10 +++++----- ci/scripts/install_iwyu.sh | 18 +++++++++--------- ci/scripts/install_ninja.sh | 4 ++-- ci/scripts/install_numpy.sh | 2 +- ci/scripts/install_pandas.sh | 4 ++-- ci/scripts/install_python.sh | 8 ++++---- ci/scripts/install_vcpkg.sh | 6 +++--- ci/scripts/java_full_build.sh | 8 ++++---- ci/scripts/matlab_build.sh | 8 ++++---- ci/scripts/msys2_system_clean.sh | 12 ++++++------ ci/scripts/python_sdist_build.sh | 2 +- ci/scripts/release_test.sh | 2 +- ci/scripts/ruby_test.sh | 2 +- ci/scripts/rust_build.sh | 4 ++-- 19 files changed, 77 insertions(+), 59 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index fe178e06936..3ef23a08ece 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -196,13 +196,31 @@ repos: ?^ci/scripts/c_glib_build\.sh$| ?^ci/scripts/c_glib_test\.sh$| ?^ci/scripts/conan_setup\.sh$| + ?^ci/scripts/csharp_build\.sh$| + ?^ci/scripts/csharp_pack\.sh$| ?^ci/scripts/download_tz_database\.sh$| ?^ci/scripts/install_azurite\.sh$| ?^ci/scripts/install_ccache\.sh$| ?^ci/scripts/install_ceph\.sh$| + ?^ci/scripts/install_chromedriver\.sh$| + ?^ci/scripts/install_cmake\.sh$| + ?^ci/scripts/install_emscripten\.sh$| + ?^ci/scripts/install_iwyu\.sh$| + ?^ci/scripts/install_ninja\.sh$| + ?^ci/scripts/install_numpy\.sh$| + ?^ci/scripts/install_pandas\.sh$| + ?^ci/scripts/install_python\.sh$| ?^ci/scripts/install_spark\.sh$| + ?^ci/scripts/install_vcpkg\.sh$| ?^ci/scripts/integration_dask\.sh$| + ?^ci/scripts/java_full_build\.sh$| + ?^ci/scripts/matlab_build\.sh$| + ?^ci/scripts/msys2_system_clean\.sh$| ?^ci/scripts/msys2_system_upgrade\.sh$| + ?^ci/scripts/python_sdist_build\.sh$| + ?^ci/scripts/release_test\.sh$| + ?^ci/scripts/ruby_test\.sh$| + ?^ci/scripts/rust_build\.sh$| ?^ci/scripts/util_free_space\.sh$| ?^cpp/build-support/build-lz4-lib\.sh$| ?^cpp/build-support/build-zstd-lib\.sh$| diff --git a/ci/scripts/csharp_build.sh b/ci/scripts/csharp_build.sh index 5a397679487..f90233f16e7 100755 --- a/ci/scripts/csharp_build.sh +++ b/ci/scripts/csharp_build.sh @@ -21,6 +21,6 @@ set -ex source_dir=${1}/csharp -pushd ${source_dir} +pushd "${source_dir}" dotnet build popd diff --git a/ci/scripts/csharp_pack.sh b/ci/scripts/csharp_pack.sh index e9dfc664ec5..9849df682e7 100755 --- a/ci/scripts/csharp_pack.sh +++ b/ci/scripts/csharp_pack.sh @@ -21,6 +21,6 @@ set -eux source_dir=${1}/csharp -pushd ${source_dir} +pushd "${source_dir}" dotnet pack -c Release popd diff --git a/ci/scripts/install_chromedriver.sh b/ci/scripts/install_chromedriver.sh index 9097a20bfc5..9167ae70e8d 100755 --- a/ci/scripts/install_chromedriver.sh +++ b/ci/scripts/install_chromedriver.sh @@ -23,7 +23,7 @@ set -e chrome_version=$1 -if [ $chrome_version = "latest" ]; then +if [ "$chrome_version" = "latest" ]; then latest_release_path=LATEST_RELEASE_STABLE else latest_release_path=LATEST_RELEASE_${chrome_version} diff --git a/ci/scripts/install_cmake.sh b/ci/scripts/install_cmake.sh index d01a7a744dc..a6916b255e2 100755 --- a/ci/scripts/install_cmake.sh +++ b/ci/scripts/install_cmake.sh @@ -30,7 +30,7 @@ archs=([x86_64]=x86_64 [aarch64]=aarch64) arch=$(uname -m) -if [ -z ${archs[$arch]} ]; then +if [ -z "${archs[$arch]}" ]; then echo "Unsupported architecture: ${arch}" exit 0 fi @@ -56,25 +56,25 @@ case ${platform} in ;; esac -mkdir -p ${prefix} +mkdir -p "${prefix}" url="https://github.com/Kitware/CMake/releases/download/v${version}/cmake-${version}-${platform}-" case ${platform} in macos) url+="universal.tar.gz" - curl -L ${url} | tar -xzf - --directory ${prefix} --strip-components=1 - ln -s CMake.app/Contents/bin ${prefix}/bin + curl -L "${url}" | tar -xzf - --directory "${prefix}" --strip-components=1 + ln -s CMake.app/Contents/bin "${prefix}/bin" ;; windows) url+="${arch}.zip" - archive_name=$(basename ${url}) - curl -L -o ${archive_name} ${url} - unzip ${archive_name} - base_name=$(basename ${archive_name} .zip) - mv ${base_name}/* ${prefix} - rm -rf ${base_name} ${archive_name} + archive_name=$(basename "${url}") + curl -L -o "${archive_name}" "${url}" + unzip "${archive_name}" + base_name=$(basename "${archive_name}" .zip) + mv "${base_name}"/* "${prefix}" + rm -rf "${base_name}" "${archive_name}" ;; *) url+="${arch}.tar.gz" - curl -L ${url} | tar -xzf - --directory ${prefix} --strip-components=1 + curl -L "${url}" | tar -xzf - --directory "${prefix}" --strip-components=1 ;; esac diff --git a/ci/scripts/install_emscripten.sh b/ci/scripts/install_emscripten.sh index 4bad7238a6c..6cf3f023ba3 100755 --- a/ci/scripts/install_emscripten.sh +++ b/ci/scripts/install_emscripten.sh @@ -24,13 +24,13 @@ set -e target_path=$1 pyodide_path=$2 -emscripten_version=$(${pyodide_path}/python -c "import sys;print(*sys._emscripten_info.emscripten_version,sep='.')") +emscripten_version=$("${pyodide_path}/python" -c "import sys;print(*sys._emscripten_info.emscripten_version,sep='.')") -cd ${target_path} +cd "${target_path}" if [ ! -d emsdk ]; then git clone https://github.com/emscripten-core/emsdk.git fi cd emsdk -./emsdk install ${emscripten_version} -./emsdk activate ${emscripten_version} -echo "Installed emsdk to: ${target_path}" \ No newline at end of file +./emsdk install "${emscripten_version}" +./emsdk activate "${emscripten_version}" +echo "Installed emsdk to: ${target_path}" diff --git a/ci/scripts/install_iwyu.sh b/ci/scripts/install_iwyu.sh index 3cd2cbc95fe..03e6b92f299 100755 --- a/ci/scripts/install_iwyu.sh +++ b/ci/scripts/install_iwyu.sh @@ -23,26 +23,26 @@ install_prefix=${2:-/usr/local} clang_tools_version=${3:-8} iwyu_branch_name="clang_${clang_tools_version}" -if [ ${clang_tools_version} -lt 10 ]; then +if [ "${clang_tools_version}" -lt 10 ]; then iwyu_branch_name="${iwyu_branch_name}.0" fi -git clone --single-branch --branch ${iwyu_branch_name} \ - https://github.com/include-what-you-use/include-what-you-use.git ${source_dir} +git clone --single-branch --branch "${iwyu_branch_name}" \ + https://github.com/include-what-you-use/include-what-you-use.git "${source_dir}" -mkdir -p ${source_dir}/build -pushd ${source_dir}/build +mkdir -p "${source_dir}/build" +pushd "${source_dir}/build" # Build IWYU for current Clang export CC=clang-${clang_tools_version} export CXX=clang++-${clang_tools_version} -cmake -DCMAKE_PREFIX_PATH=/usr/lib/llvm-${clang_tools_version} \ - -DCMAKE_INSTALL_PREFIX=${install_prefix} \ - ${source_dir} +cmake -DCMAKE_PREFIX_PATH="/usr/lib/llvm-${clang_tools_version}" \ + -DCMAKE_INSTALL_PREFIX="${install_prefix}" \ + "${source_dir}" make -j4 make install popd -rm -rf ${source_dir} +rm -rf "${source_dir}" diff --git a/ci/scripts/install_ninja.sh b/ci/scripts/install_ninja.sh index 0440d563fb1..5cfa0cbea2d 100755 --- a/ci/scripts/install_ninja.sh +++ b/ci/scripts/install_ninja.sh @@ -30,11 +30,11 @@ prefix=$2 url="https://github.com/ninja-build/ninja/archive/v${version}.tar.gz" mkdir /tmp/ninja -wget -q ${url} -O - | tar -xzf - --directory /tmp/ninja --strip-components=1 +wget -q "${url}" -O - | tar -xzf - --directory /tmp/ninja --strip-components=1 pushd /tmp/ninja ./configure.py --bootstrap -mv ninja ${prefix}/bin +mv ninja "${prefix}/bin" popd rm -rf /tmp/ninja diff --git a/ci/scripts/install_numpy.sh b/ci/scripts/install_numpy.sh index f04fe81b669..8addc7c023e 100755 --- a/ci/scripts/install_numpy.sh +++ b/ci/scripts/install_numpy.sh @@ -29,5 +29,5 @@ numpy=${1:-"latest"} if [ "${numpy}" = "latest" ]; then pip install numpy else - pip install numpy==${numpy} + pip install numpy=="${numpy}" fi diff --git a/ci/scripts/install_pandas.sh b/ci/scripts/install_pandas.sh index 6a506a86514..03009b3824f 100755 --- a/ci/scripts/install_pandas.sh +++ b/ci/scripts/install_pandas.sh @@ -32,7 +32,7 @@ if [ "${numpy}" = "nightly" ]; then elif [ "${numpy}" = "latest" ]; then pip install numpy else - pip install numpy==${numpy} + pip install numpy=="${numpy}" fi if [ "${pandas}" = "upstream_devel" ]; then @@ -42,5 +42,5 @@ elif [ "${pandas}" = "nightly" ]; then elif [ "${pandas}" = "latest" ]; then pip install pandas else - pip install pandas==${pandas} + pip install pandas=="${pandas}" fi diff --git a/ci/scripts/install_python.sh b/ci/scripts/install_python.sh index a84d136c0c2..1a1b31c5f58 100755 --- a/ci/scripts/install_python.sh +++ b/ci/scripts/install_python.sh @@ -44,7 +44,7 @@ platform=${platforms[$1]} version=$2 full_version=${versions[$2]} -if [ $platform = "macOS" ]; then +if [ "$platform" = "macOS" ]; then echo "Downloading Python installer..." if [ "$(uname -m)" = "x86_64" ] && [ "$version" = "3.9" ]; @@ -74,12 +74,12 @@ if [ $platform = "macOS" ]; then EOF - installer -pkg $fname -applyChoiceChangesXML ./choicechanges.plist -target / + installer -pkg "$fname" -applyChoiceChangesXML ./choicechanges.plist -target / rm ./choicechanges.plist else - installer -pkg $fname -target / + installer -pkg "$fname" -target / fi - rm $fname + rm "$fname" python="/Library/Frameworks/Python.framework/Versions/${version}/bin/python${version}" if [[ $2 == "3.13t" ]]; then diff --git a/ci/scripts/install_vcpkg.sh b/ci/scripts/install_vcpkg.sh index 4183a606512..35980043285 100755 --- a/ci/scripts/install_vcpkg.sh +++ b/ci/scripts/install_vcpkg.sh @@ -36,9 +36,9 @@ if [ -z "${vcpkg_version}" ]; then fi # reduce the fetched data using a shallow clone -git clone --shallow-since=2021-04-01 https://github.com/microsoft/vcpkg ${vcpkg_destination} +git clone --shallow-since=2021-04-01 https://github.com/microsoft/vcpkg "${vcpkg_destination}" -pushd ${vcpkg_destination} +pushd "${vcpkg_destination}" git checkout "${vcpkg_version}" @@ -49,7 +49,7 @@ else fi if [ -f "${vcpkg_ports_patch}" ]; then - git apply --verbose --ignore-whitespace ${vcpkg_ports_patch} + git apply --verbose --ignore-whitespace "${vcpkg_ports_patch}" echo "Patch successfully applied to the VCPKG port files!" fi diff --git a/ci/scripts/java_full_build.sh b/ci/scripts/java_full_build.sh index 4beade50b45..19d85305b2e 100755 --- a/ci/scripts/java_full_build.sh +++ b/ci/scripts/java_full_build.sh @@ -24,7 +24,7 @@ dist_dir=${2} export ARROW_TEST_DATA=${arrow_dir}/testing/data -pushd ${arrow_dir}/java +pushd "${arrow_dir}/java" # Ensure that there is no old jar # inside the maven repository @@ -52,8 +52,8 @@ mvn clean \ -Papache-release \ -Parrow-c-data \ -Parrow-jni \ - -Darrow.cpp.build.dir=$dist_dir \ - -Darrow.c.jni.dist.dir=$dist_dir \ + -Darrow.cpp.build.dir="$dist_dir" \ + -Darrow.c.jni.dist.dir="$dist_dir" \ --no-transfer-progress # copy all jar, zip and pom files to the distribution folder @@ -66,6 +66,6 @@ find ~/.m2/repository/org/apache/arrow \ -name "*.zip" \ ")" \ -exec echo {} ";" \ - -exec cp {} $dist_dir ";" + -exec cp {} "$dist_dir" ";" popd diff --git a/ci/scripts/matlab_build.sh b/ci/scripts/matlab_build.sh index d3f86adbb8a..0548cd08726 100755 --- a/ci/scripts/matlab_build.sh +++ b/ci/scripts/matlab_build.sh @@ -26,9 +26,9 @@ build_dir=${base_dir}/matlab/build install_dir=${base_dir}/matlab/install cmake \ - -S ${source_dir} \ - -B ${build_dir} \ + -S "${source_dir}" \ + -B "${build_dir}" \ -G Ninja \ - -D CMAKE_INSTALL_PREFIX=${install_dir} \ + -D CMAKE_INSTALL_PREFIX="${install_dir}" \ -D MATLAB_ADD_INSTALL_DIR_TO_SEARCH_PATH=OFF -cmake --build ${build_dir} --config Release --target install +cmake --build "${build_dir}" --config Release --target install diff --git a/ci/scripts/msys2_system_clean.sh b/ci/scripts/msys2_system_clean.sh index c4de8a6b265..35b28e6a2ce 100755 --- a/ci/scripts/msys2_system_clean.sh +++ b/ci/scripts/msys2_system_clean.sh @@ -30,11 +30,11 @@ case "${MINGW_PACKAGE_PREFIX}" in --nosave \ --recursive \ --remove \ - ${MINGW_PACKAGE_PREFIX}-clang-tools-extra \ - ${MINGW_PACKAGE_PREFIX}-gcc-ada \ - ${MINGW_PACKAGE_PREFIX}-gcc-fortran \ - ${MINGW_PACKAGE_PREFIX}-gcc-libgfortran \ - ${MINGW_PACKAGE_PREFIX}-gcc-objc \ - ${MINGW_PACKAGE_PREFIX}-libgccjit + "${MINGW_PACKAGE_PREFIX}-clang-tools-extra" \ + "${MINGW_PACKAGE_PREFIX}-gcc-ada" \ + "${MINGW_PACKAGE_PREFIX}-gcc-fortran" \ + "${MINGW_PACKAGE_PREFIX}-gcc-libgfortran" \ + "${MINGW_PACKAGE_PREFIX}-gcc-objc" \ + "${MINGW_PACKAGE_PREFIX}-libgccjit" ;; esac diff --git a/ci/scripts/python_sdist_build.sh b/ci/scripts/python_sdist_build.sh index f9e9359b6f6..dfb99518431 100755 --- a/ci/scripts/python_sdist_build.sh +++ b/ci/scripts/python_sdist_build.sh @@ -21,7 +21,7 @@ set -eux source_dir=${1}/python -pushd ${source_dir} +pushd "${source_dir}" export SETUPTOOLS_SCM_PRETEND_VERSION=${PYARROW_VERSION:-} ${PYTHON:-python} setup.py sdist popd diff --git a/ci/scripts/release_test.sh b/ci/scripts/release_test.sh index 583d9618c65..af6d7dbb3a5 100755 --- a/ci/scripts/release_test.sh +++ b/ci/scripts/release_test.sh @@ -21,7 +21,7 @@ set -eux arrow_dir=${1} -pushd ${arrow_dir} +pushd "${arrow_dir}" dev/release/run-test.rb -vv diff --git a/ci/scripts/ruby_test.sh b/ci/scripts/ruby_test.sh index 507fa7858e8..da4c6454116 100755 --- a/ci/scripts/ruby_test.sh +++ b/ci/scripts/ruby_test.sh @@ -32,4 +32,4 @@ if [ -z "${ARROW_DEBUG_MEMORY_POOL}" ]; then export ARROW_DEBUG_MEMORY_POOL=trap fi -rake -f ${source_dir}/Rakefile BUILD_DIR=${build_dir} USE_BUNDLER=yes +rake -f "${source_dir}/Rakefile" BUILD_DIR="${build_dir}" USE_BUNDLER=yes diff --git a/ci/scripts/rust_build.sh b/ci/scripts/rust_build.sh index 5fc21d454b0..866b4e333e1 100755 --- a/ci/scripts/rust_build.sh +++ b/ci/scripts/rust_build.sh @@ -52,10 +52,10 @@ set -x # show activated toolchain rustup show -pushd ${source_dir} +pushd "${source_dir}" # build only the integration testing binaries -cargo build -p arrow-integration-testing --target-dir ${build_dir} +cargo build -p arrow-integration-testing --target-dir "${build_dir}" # Save disk space by removing large temporary build products rm -rf target/debug/deps From 463137b86ec2cad852c533376f39a7fe80e67bf5 Mon Sep 17 00:00:00 2001 From: Hiroyuki Sato Date: Mon, 19 May 2025 17:09:37 +0900 Subject: [PATCH 2/3] Revert java_full_build.sh change --- .pre-commit-config.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3ef23a08ece..bb359226273 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -213,7 +213,6 @@ repos: ?^ci/scripts/install_spark\.sh$| ?^ci/scripts/install_vcpkg\.sh$| ?^ci/scripts/integration_dask\.sh$| - ?^ci/scripts/java_full_build\.sh$| ?^ci/scripts/matlab_build\.sh$| ?^ci/scripts/msys2_system_clean\.sh$| ?^ci/scripts/msys2_system_upgrade\.sh$| From 709f074f43dcdbb60229c6b1a4a5d25bcb720a6c Mon Sep 17 00:00:00 2001 From: Hiroyuki Sato Date: Mon, 19 May 2025 17:42:19 +0900 Subject: [PATCH 3/3] Revert java_full_build.sh change --- ci/scripts/java_full_build.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ci/scripts/java_full_build.sh b/ci/scripts/java_full_build.sh index 19d85305b2e..4beade50b45 100755 --- a/ci/scripts/java_full_build.sh +++ b/ci/scripts/java_full_build.sh @@ -24,7 +24,7 @@ dist_dir=${2} export ARROW_TEST_DATA=${arrow_dir}/testing/data -pushd "${arrow_dir}/java" +pushd ${arrow_dir}/java # Ensure that there is no old jar # inside the maven repository @@ -52,8 +52,8 @@ mvn clean \ -Papache-release \ -Parrow-c-data \ -Parrow-jni \ - -Darrow.cpp.build.dir="$dist_dir" \ - -Darrow.c.jni.dist.dir="$dist_dir" \ + -Darrow.cpp.build.dir=$dist_dir \ + -Darrow.c.jni.dist.dir=$dist_dir \ --no-transfer-progress # copy all jar, zip and pom files to the distribution folder @@ -66,6 +66,6 @@ find ~/.m2/repository/org/apache/arrow \ -name "*.zip" \ ")" \ -exec echo {} ";" \ - -exec cp {} "$dist_dir" ";" + -exec cp {} $dist_dir ";" popd