From 18a72f18cc5b01fdbf1af8c82bd8f5c8c85e4abc Mon Sep 17 00:00:00 2001 From: Will Jones Date: Thu, 30 Jun 2022 11:54:09 -0700 Subject: [PATCH 1/4] feat: Enable brotli on autobrew --- dev/tasks/homebrew-formulae/autobrew/apache-arrow.rb | 2 ++ r/tools/autobrew | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/dev/tasks/homebrew-formulae/autobrew/apache-arrow.rb b/dev/tasks/homebrew-formulae/autobrew/apache-arrow.rb index 45c04463b6d..aa90246f760 100644 --- a/dev/tasks/homebrew-formulae/autobrew/apache-arrow.rb +++ b/dev/tasks/homebrew-formulae/autobrew/apache-arrow.rb @@ -31,6 +31,7 @@ class ApacheArrow < Formula # NOTE: if you add something here, be sure to add to PKG_LIBS in r/tools/autobrew depends_on "boost" => :build + depends_on "brotli" depends_on "cmake" => :build depends_on "aws-sdk-cpp" depends_on "lz4" @@ -57,6 +58,7 @@ def install -DARROW_S3=ON -DARROW_USE_GLOG=OFF -DARROW_VERBOSE_THIRDPARTY_BUILD=ON + -DARROW_WITH_BROTLI=ON -DARROW_WITH_LZ4=ON -DARROW_WITH_SNAPPY=ON -DARROW_WITH_ZLIB=ON diff --git a/r/tools/autobrew b/r/tools/autobrew index 8ba06a64c27..7d55fc352f9 100644 --- a/r/tools/autobrew +++ b/r/tools/autobrew @@ -48,7 +48,7 @@ fi # Hardcode this for my custom autobrew build rm -f $BREWDIR/lib/*.dylib AWS_LIBS="-laws-cpp-sdk-config -laws-cpp-sdk-transfer -laws-cpp-sdk-identity-management -laws-cpp-sdk-cognito-identity -laws-cpp-sdk-sts -laws-cpp-sdk-s3 -laws-cpp-sdk-core -laws-c-event-stream -laws-checksums -laws-c-common -laws-crt-cpp -laws-c-io -laws-c-s3 -laws-c-auth -laws-c-http -laws-c-cal -laws-c-compression -laws-c-mqtt -lpthread -lcurl" -PKG_LIBS="-lparquet -larrow_dataset -larrow -larrow_bundled_dependencies -lthrift -llz4 -lsnappy -lzstd $AWS_LIBS" +PKG_LIBS="-lparquet -larrow_dataset -larrow -larrow_bundled_dependencies -lthrift -lbrotlienc -brotlidec -brotlicommon -llz4 -lsnappy -lzstd $AWS_LIBS" PKG_DIRS="-L$BREWDIR/lib" # Prevent CRAN builder from linking against old libs in /usr/local/lib From 2901223a072bdf99799275ac36df5380c79a5769 Mon Sep 17 00:00:00 2001 From: Will Jones Date: Thu, 30 Jun 2022 13:17:52 -0700 Subject: [PATCH 2/4] fix: Need l --- r/tools/autobrew | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/r/tools/autobrew b/r/tools/autobrew index 7d55fc352f9..37a94472bd2 100644 --- a/r/tools/autobrew +++ b/r/tools/autobrew @@ -48,7 +48,7 @@ fi # Hardcode this for my custom autobrew build rm -f $BREWDIR/lib/*.dylib AWS_LIBS="-laws-cpp-sdk-config -laws-cpp-sdk-transfer -laws-cpp-sdk-identity-management -laws-cpp-sdk-cognito-identity -laws-cpp-sdk-sts -laws-cpp-sdk-s3 -laws-cpp-sdk-core -laws-c-event-stream -laws-checksums -laws-c-common -laws-crt-cpp -laws-c-io -laws-c-s3 -laws-c-auth -laws-c-http -laws-c-cal -laws-c-compression -laws-c-mqtt -lpthread -lcurl" -PKG_LIBS="-lparquet -larrow_dataset -larrow -larrow_bundled_dependencies -lthrift -lbrotlienc -brotlidec -brotlicommon -llz4 -lsnappy -lzstd $AWS_LIBS" +PKG_LIBS="-lparquet -larrow_dataset -larrow -larrow_bundled_dependencies -lthrift -lbrotlienc -lbrotlidec -lbrotlicommon -llz4 -lsnappy -lzstd $AWS_LIBS" PKG_DIRS="-L$BREWDIR/lib" # Prevent CRAN builder from linking against old libs in /usr/local/lib From fb96cb53839de9a6a07233c7cc9ab25e7b0a51e3 Mon Sep 17 00:00:00 2001 From: Will Jones Date: Thu, 30 Jun 2022 14:27:14 -0700 Subject: [PATCH 3/4] feat: Add bzip2 to MacOS and Windows R packages --- ci/scripts/PKGBUILD | 2 ++ ci/scripts/r_windows_build.sh | 6 +++--- dev/tasks/homebrew-formulae/autobrew/apache-arrow.rb | 1 + r/configure.win | 2 +- r/tests/testthat/test-compressed.R | 2 ++ 5 files changed, 9 insertions(+), 4 deletions(-) diff --git a/ci/scripts/PKGBUILD b/ci/scripts/PKGBUILD index ea17fba17ed..428447d2638 100644 --- a/ci/scripts/PKGBUILD +++ b/ci/scripts/PKGBUILD @@ -25,6 +25,7 @@ arch=("any") url="https://arrow.apache.org/" license=("Apache-2.0") depends=("${MINGW_PACKAGE_PREFIX}-aws-sdk-cpp" + "${MINGW_PACKAGE_PREFIX}-bzip2" "${MINGW_PACKAGE_PREFIX}-curl" # for google-cloud-cpp bundled build "${MINGW_PACKAGE_PREFIX}-libutf8proc" "${MINGW_PACKAGE_PREFIX}-re2" @@ -123,6 +124,7 @@ build() { -DARROW_WITH_ZLIB=ON \ -DARROW_WITH_ZSTD=ON \ -DARROW_WITH_BROTLI=ON \ + -DARROW_WITH_BZ2=ON \ -DARROW_ZSTD_USE_SHARED=OFF \ -DARROW_CXXFLAGS="${CPPFLAGS}" \ -DCMAKE_BUILD_TYPE="release" \ diff --git a/ci/scripts/r_windows_build.sh b/ci/scripts/r_windows_build.sh index 3334eab8663..dd7a5826348 100755 --- a/ci/scripts/r_windows_build.sh +++ b/ci/scripts/r_windows_build.sh @@ -87,7 +87,7 @@ if [ -d mingw64/lib/ ]; then # These may be from https://dl.bintray.com/rtools/backports/ cp $MSYS_LIB_DIR/mingw64/lib/lib{thrift,snappy}.a $DST_DIR/${RWINLIB_LIB_DIR}/x64 # These are from https://dl.bintray.com/rtools/mingw{32,64}/ - cp $MSYS_LIB_DIR/mingw64/lib/lib{zstd,lz4,brotli*,crypto,curl,ss*,utf8proc,re2,aws*}.a $DST_DIR/lib/x64 + cp $MSYS_LIB_DIR/mingw64/lib/lib{zstd,lz4,brotli*,bzip2,crypto,curl,ss*,utf8proc,re2,aws*}.a $DST_DIR/lib/x64 fi # Same for the 32-bit versions @@ -97,7 +97,7 @@ if [ -d mingw32/lib/ ]; then mkdir -p $DST_DIR/lib/i386 mv mingw32/lib/*.a $DST_DIR/${RWINLIB_LIB_DIR}/i386 cp $MSYS_LIB_DIR/mingw32/lib/lib{thrift,snappy}.a $DST_DIR/${RWINLIB_LIB_DIR}/i386 - cp $MSYS_LIB_DIR/mingw32/lib/lib{zstd,lz4,brotli*,crypto,curl,ss*,utf8proc,re2,aws*}.a $DST_DIR/lib/i386 + cp $MSYS_LIB_DIR/mingw32/lib/lib{zstd,lz4,brotli*,bzip2,crypto,curl,ss*,utf8proc,re2,aws*}.a $DST_DIR/lib/i386 fi # Do the same also for ucrt64 @@ -105,7 +105,7 @@ if [ -d ucrt64/lib/ ]; then ls $MSYS_LIB_DIR/ucrt64/lib/ mkdir -p $DST_DIR/lib/x64-ucrt mv ucrt64/lib/*.a $DST_DIR/lib/x64-ucrt - cp $MSYS_LIB_DIR/ucrt64/lib/lib{thrift,snappy,zstd,lz4,brotli*,crypto,curl,ss*,utf8proc,re2,aws*}.a $DST_DIR/lib/x64-ucrt + cp $MSYS_LIB_DIR/ucrt64/lib/lib{thrift,snappy,zstd,lz4,brotli*,bzip2,crypto,curl,ss*,utf8proc,re2,aws*}.a $DST_DIR/lib/x64-ucrt fi # Create build artifact diff --git a/dev/tasks/homebrew-formulae/autobrew/apache-arrow.rb b/dev/tasks/homebrew-formulae/autobrew/apache-arrow.rb index aa90246f760..dde994ab433 100644 --- a/dev/tasks/homebrew-formulae/autobrew/apache-arrow.rb +++ b/dev/tasks/homebrew-formulae/autobrew/apache-arrow.rb @@ -59,6 +59,7 @@ def install -DARROW_USE_GLOG=OFF -DARROW_VERBOSE_THIRDPARTY_BUILD=ON -DARROW_WITH_BROTLI=ON + -DARROW_WITH_BZ2=ON -DARROW_WITH_LZ4=ON -DARROW_WITH_SNAPPY=ON -DARROW_WITH_ZLIB=ON diff --git a/r/configure.win b/r/configure.win index dfd2c87ab4f..62184031c29 100755 --- a/r/configure.win +++ b/r/configure.win @@ -64,7 +64,7 @@ function configure_release() { PKG_LIBS="-L${RWINLIB}/lib"'$(subst gcc,,$(COMPILED_BY))$(R_ARCH) ' PKG_LIBS="$PKG_LIBS -L${RWINLIB}/lib"'$(R_ARCH)$(CRT) ' PKG_LIBS="$PKG_LIBS -lparquet -larrow_dataset -larrow -larrow_bundled_dependencies \ - -lutf8proc -lthrift -lsnappy -lz -lzstd -llz4 ${BROTLI_LIBS} -lole32 \ + -lutf8proc -lthrift -lsnappy -lz -lzstd -llz4 -lbzip2 ${BROTLI_LIBS} -lole32 \ ${MIMALLOC_LIBS} ${OPENSSL_LIBS}" # S3, GCS, and re2 support only for Rtools40 (i.e. R >= 4.0) diff --git a/r/tests/testthat/test-compressed.R b/r/tests/testthat/test-compressed.R index 7b3c38206ec..485e16769f7 100644 --- a/r/tests/testthat/test-compressed.R +++ b/r/tests/testthat/test-compressed.R @@ -28,6 +28,8 @@ test_that("Compression codecs are included in the Windows build", { expect_true(codec_is_available("lz4")) expect_true(codec_is_available("zstd")) expect_true(codec_is_available("brotli")) + expect_true(codec_is_available("bz2")) + expect_true(codec_is_available("snappy")) }) test_that("Codec attributes", { From e95c99cdaf0557b8f6e1bfa4d5bbfb55ed235125 Mon Sep 17 00:00:00 2001 From: Will Jones Date: Thu, 30 Jun 2022 14:57:24 -0700 Subject: [PATCH 4/4] fix: Address Windows library names --- ci/scripts/r_windows_build.sh | 6 +++--- r/configure.win | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ci/scripts/r_windows_build.sh b/ci/scripts/r_windows_build.sh index dd7a5826348..c361af1d267 100755 --- a/ci/scripts/r_windows_build.sh +++ b/ci/scripts/r_windows_build.sh @@ -87,7 +87,7 @@ if [ -d mingw64/lib/ ]; then # These may be from https://dl.bintray.com/rtools/backports/ cp $MSYS_LIB_DIR/mingw64/lib/lib{thrift,snappy}.a $DST_DIR/${RWINLIB_LIB_DIR}/x64 # These are from https://dl.bintray.com/rtools/mingw{32,64}/ - cp $MSYS_LIB_DIR/mingw64/lib/lib{zstd,lz4,brotli*,bzip2,crypto,curl,ss*,utf8proc,re2,aws*}.a $DST_DIR/lib/x64 + cp $MSYS_LIB_DIR/mingw64/lib/lib{zstd,lz4,brotli*,bz2,crypto,curl,ss*,utf8proc,re2,aws*}.a $DST_DIR/lib/x64 fi # Same for the 32-bit versions @@ -97,7 +97,7 @@ if [ -d mingw32/lib/ ]; then mkdir -p $DST_DIR/lib/i386 mv mingw32/lib/*.a $DST_DIR/${RWINLIB_LIB_DIR}/i386 cp $MSYS_LIB_DIR/mingw32/lib/lib{thrift,snappy}.a $DST_DIR/${RWINLIB_LIB_DIR}/i386 - cp $MSYS_LIB_DIR/mingw32/lib/lib{zstd,lz4,brotli*,bzip2,crypto,curl,ss*,utf8proc,re2,aws*}.a $DST_DIR/lib/i386 + cp $MSYS_LIB_DIR/mingw32/lib/lib{zstd,lz4,brotli*,bz2,crypto,curl,ss*,utf8proc,re2,aws*}.a $DST_DIR/lib/i386 fi # Do the same also for ucrt64 @@ -105,7 +105,7 @@ if [ -d ucrt64/lib/ ]; then ls $MSYS_LIB_DIR/ucrt64/lib/ mkdir -p $DST_DIR/lib/x64-ucrt mv ucrt64/lib/*.a $DST_DIR/lib/x64-ucrt - cp $MSYS_LIB_DIR/ucrt64/lib/lib{thrift,snappy,zstd,lz4,brotli*,bzip2,crypto,curl,ss*,utf8proc,re2,aws*}.a $DST_DIR/lib/x64-ucrt + cp $MSYS_LIB_DIR/ucrt64/lib/lib{thrift,snappy,zstd,lz4,brotli*,bz2,crypto,curl,ss*,utf8proc,re2,aws*}.a $DST_DIR/lib/x64-ucrt fi # Create build artifact diff --git a/r/configure.win b/r/configure.win index 62184031c29..7aa7e47fc1d 100755 --- a/r/configure.win +++ b/r/configure.win @@ -64,7 +64,7 @@ function configure_release() { PKG_LIBS="-L${RWINLIB}/lib"'$(subst gcc,,$(COMPILED_BY))$(R_ARCH) ' PKG_LIBS="$PKG_LIBS -L${RWINLIB}/lib"'$(R_ARCH)$(CRT) ' PKG_LIBS="$PKG_LIBS -lparquet -larrow_dataset -larrow -larrow_bundled_dependencies \ - -lutf8proc -lthrift -lsnappy -lz -lzstd -llz4 -lbzip2 ${BROTLI_LIBS} -lole32 \ + -lutf8proc -lthrift -lsnappy -lz -lzstd -llz4 -lbz2 ${BROTLI_LIBS} -lole32 \ ${MIMALLOC_LIBS} ${OPENSSL_LIBS}" # S3, GCS, and re2 support only for Rtools40 (i.e. R >= 4.0)