From 020c1ccde45e375c078f14a332f68047d05fa5c7 Mon Sep 17 00:00:00 2001 From: Neal Richardson Date: Mon, 10 Aug 2020 11:03:03 -0700 Subject: [PATCH 01/13] Turn on ARROW_S3 in autobrew and rwinlib --- ci/scripts/PKGBUILD | 4 +++- ci/scripts/r_windows_build.sh | 10 ++++++++-- dev/tasks/homebrew-formulae/autobrew/apache-arrow.rb | 3 ++- r/configure.win | 5 +++-- r/tools/autobrew | 2 +- 5 files changed, 17 insertions(+), 7 deletions(-) diff --git a/ci/scripts/PKGBUILD b/ci/scripts/PKGBUILD index df5f5860445..a585d06ee9a 100644 --- a/ci/scripts/PKGBUILD +++ b/ci/scripts/PKGBUILD @@ -24,7 +24,8 @@ pkgdesc="Apache Arrow is a cross-language development platform for in-memory dat arch=("any") url="https://arrow.apache.org/" license=("Apache-2.0") -depends=("${MINGW_PACKAGE_PREFIX}-thrift" +depends=("${MINGW_PACKAGE_PREFIX}-aws-sdk-cpp" + "${MINGW_PACKAGE_PREFIX}-thrift" "${MINGW_PACKAGE_PREFIX}-snappy" "${MINGW_PACKAGE_PREFIX}-zlib" "${MINGW_PACKAGE_PREFIX}-lz4" @@ -94,6 +95,7 @@ build() { -DARROW_MIMALLOC=ON \ -DARROW_PACKAGE_PREFIX="${MINGW_PREFIX}" \ -DARROW_PARQUET=ON \ + -DARROW_S3=ON \ -DARROW_SNAPPY_USE_SHARED=OFF \ -DARROW_USE_GLOG=OFF \ -DARROW_WITH_LZ4=ON \ diff --git a/ci/scripts/r_windows_build.sh b/ci/scripts/r_windows_build.sh index ed9e211f9ac..c736b6ad8b3 100755 --- a/ci/scripts/r_windows_build.sh +++ b/ci/scripts/r_windows_build.sh @@ -34,6 +34,12 @@ if [ "$RTOOLS_VERSION" = "35" ]; then # lib-4.9.3 is for libraries compiled with gcc 4.9 (Rtools 3.5) RWINLIB_LIB_DIR="lib-4.9.3" else + # Uncomment L38-41 if you're testing a new rtools dependency that hasn't yet sync'd to CRAN + curl https://raw.githubusercontent.com/r-windows/rtools-packages/master/pacman.conf > /etc/pacman.conf + curl -OSsl "http://repo.msys2.org/msys/x86_64/msys2-keyring-r21.b39fb11-1-any.pkg.tar.xz" + pacman -U --noconfirm msys2-keyring-r21.b39fb11-1-any.pkg.tar.xz && rm msys2-keyring-r21.b39fb11-1-any.pkg.tar.xz + pacman --noconfirm -Scc + pacman --noconfirm -Syy RWINLIB_LIB_DIR="lib" fi @@ -84,8 +90,8 @@ cp $MSYS_LIB_DIR/mingw64/lib/lib{thrift,snappy}.a $DST_DIR/${RWINLIB_LIB_DIR}/x6 cp $MSYS_LIB_DIR/mingw32/lib/lib{thrift,snappy}.a $DST_DIR/${RWINLIB_LIB_DIR}/i386 # These are from https://dl.bintray.com/rtools/mingw{32,64}/ -cp $MSYS_LIB_DIR/mingw64/lib/lib{zstd,lz4,crypto}.a $DST_DIR/lib/x64 -cp $MSYS_LIB_DIR/mingw32/lib/lib{zstd,lz4,crypto}.a $DST_DIR/lib/i386 +cp $MSYS_LIB_DIR/mingw64/lib/lib{zstd,lz4,crypto,aws*}.a $DST_DIR/lib/x64 +cp $MSYS_LIB_DIR/mingw32/lib/lib{zstd,lz4,crypto,aws*}.a $DST_DIR/lib/i386 # Create build artifact zip -r ${DST_DIR}.zip $DST_DIR diff --git a/dev/tasks/homebrew-formulae/autobrew/apache-arrow.rb b/dev/tasks/homebrew-formulae/autobrew/apache-arrow.rb index 725847acd9c..3f8ece2974f 100644 --- a/dev/tasks/homebrew-formulae/autobrew/apache-arrow.rb +++ b/dev/tasks/homebrew-formulae/autobrew/apache-arrow.rb @@ -30,6 +30,7 @@ class ApacheArrow < Formula end # NOTE: if you add something here, be sure to add to PKG_LIBS in r/tools/autobrew + depends_on "aws-sdk-cpp" depends_on "boost" => :build depends_on "cmake" => :build depends_on "lz4" @@ -50,7 +51,7 @@ def install -DARROW_JEMALLOC=ON -DARROW_USE_GLOG=OFF -DARROW_PYTHON=OFF - -DARROW_S3=OFF + -DARROW_S3=ON -DARROW_WITH_LZ4=ON -DARROW_WITH_ZLIB=ON -DARROW_WITH_SNAPPY=ON diff --git a/r/configure.win b/r/configure.win index 5caf8e24663..48fd3b33704 100644 --- a/r/configure.win +++ b/r/configure.win @@ -44,11 +44,12 @@ else fi OPENSSL_LIBS="-lcrypto -lcrypt32" MIMALLOC_LIBS="-lbcrypt -lpsapi" +AWS_LIBS="-laws-c-common -laws-c-event-stream -laws-checksums -laws-cpp-sdk-config -laws-cpp-sdk-core -laws-cpp-sdk-s3" # NOTE: If you make changes to the libraries below, you should also change # ci/scripts/r_windows_build.sh and ci/scripts/PKGBUILD -PKG_CFLAGS="-I${RWINLIB}/include -DARROW_STATIC -DPARQUET_STATIC -DARROW_DS_STATIC -DARROW_R_WITH_ARROW" -PKG_LIBS="-L${RWINLIB}/lib"'$(subst gcc,,$(COMPILED_BY))$(R_ARCH) '"-L${RWINLIB}/lib"'$(R_ARCH) '"-lparquet -larrow_dataset -larrow -larrow_bundled_dependencies -lthrift -lsnappy -lz -lzstd -llz4 ${MIMALLOC_LIBS} ${OPENSSL_LIBS} -lws2_32" +PKG_CFLAGS="-I${RWINLIB}/include -DARROW_STATIC -DPARQUET_STATIC -DARROW_DS_STATIC -DARROW_R_WITH_ARROW -DARROW_R_WITH_S3" +PKG_LIBS="-L${RWINLIB}/lib"'$(subst gcc,,$(COMPILED_BY))$(R_ARCH) '"-L${RWINLIB}/lib"'$(R_ARCH) '"-lparquet -larrow_dataset -larrow -larrow_bundled_dependencies -lthrift -lsnappy -lz -lzstd -llz4 ${MIMALLOC_LIBS} ${OPENSSL_LIBS} -lws2_32 ${AWS_LIBS}" # Set any user-defined CXXFLAGS if [ "$ARROW_R_CXXFLAGS" ]; then diff --git a/r/tools/autobrew b/r/tools/autobrew index 7dcfffb22d1..1967bacbe6f 100644 --- a/r/tools/autobrew +++ b/r/tools/autobrew @@ -47,7 +47,7 @@ fi # Hardcode this for my custom autobrew build rm -f $BREWDIR/lib/*.dylib -PKG_LIBS="-L$BREWDIR/lib -lparquet -larrow_dataset -larrow -lthrift -llz4 -lsnappy" +PKG_LIBS="-L$BREWDIR/lib -lparquet -larrow_dataset -larrow -lthrift -llz4 -lsnappy -laws-c-common -laws-c-event-stream -laws-checksums -laws-cpp-sdk-config -laws-cpp-sdk-core -laws-cpp-sdk-s3" # Prevent CRAN builder from linking against old libs in /usr/local/lib for FILE in $BREWDIR/Cellar/*/*/lib/*.a; do From e2a3f1c97c9cc7a7320e871f3848e6d46c822a77 Mon Sep 17 00:00:00 2001 From: Neal Richardson Date: Mon, 10 Aug 2020 13:15:25 -0700 Subject: [PATCH 02/13] Update -libs --- r/configure.win | 4 ++-- r/tools/autobrew | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/r/configure.win b/r/configure.win index 48fd3b33704..a70c2a8927b 100644 --- a/r/configure.win +++ b/r/configure.win @@ -44,12 +44,12 @@ else fi OPENSSL_LIBS="-lcrypto -lcrypt32" MIMALLOC_LIBS="-lbcrypt -lpsapi" -AWS_LIBS="-laws-c-common -laws-c-event-stream -laws-checksums -laws-cpp-sdk-config -laws-cpp-sdk-core -laws-cpp-sdk-s3" +AWS_LIBS="-laws-cpp-sdk-config -laws-cpp-sdk-transfer -laws-cpp-sdk-s3 -laws-cpp-sdk-core -laws-c-event-stream -laws-checksums -laws-c-common -lUserenv -lversion -lws2_32 -lBcrypt -lWininet -lwinhttp" # NOTE: If you make changes to the libraries below, you should also change # ci/scripts/r_windows_build.sh and ci/scripts/PKGBUILD PKG_CFLAGS="-I${RWINLIB}/include -DARROW_STATIC -DPARQUET_STATIC -DARROW_DS_STATIC -DARROW_R_WITH_ARROW -DARROW_R_WITH_S3" -PKG_LIBS="-L${RWINLIB}/lib"'$(subst gcc,,$(COMPILED_BY))$(R_ARCH) '"-L${RWINLIB}/lib"'$(R_ARCH) '"-lparquet -larrow_dataset -larrow -larrow_bundled_dependencies -lthrift -lsnappy -lz -lzstd -llz4 ${MIMALLOC_LIBS} ${OPENSSL_LIBS} -lws2_32 ${AWS_LIBS}" +PKG_LIBS="-L${RWINLIB}/lib"'$(subst gcc,,$(COMPILED_BY))$(R_ARCH) '"-L${RWINLIB}/lib"'$(R_ARCH) '"-lparquet -larrow_dataset -larrow -larrow_bundled_dependencies -lthrift -lsnappy -lz -lzstd -llz4 ${MIMALLOC_LIBS} ${OPENSSL_LIBS} ${AWS_LIBS}" # Set any user-defined CXXFLAGS if [ "$ARROW_R_CXXFLAGS" ]; then diff --git a/r/tools/autobrew b/r/tools/autobrew index 1967bacbe6f..70ffdf9bd18 100644 --- a/r/tools/autobrew +++ b/r/tools/autobrew @@ -47,7 +47,7 @@ fi # Hardcode this for my custom autobrew build rm -f $BREWDIR/lib/*.dylib -PKG_LIBS="-L$BREWDIR/lib -lparquet -larrow_dataset -larrow -lthrift -llz4 -lsnappy -laws-c-common -laws-c-event-stream -laws-checksums -laws-cpp-sdk-config -laws-cpp-sdk-core -laws-cpp-sdk-s3" +PKG_LIBS="-L$BREWDIR/lib -lparquet -larrow_dataset -larrow -lthrift -llz4 -lsnappy -laws-cpp-sdk-config -laws-cpp-sdk-transfer -laws-cpp-sdk-s3 -laws-cpp-sdk-core -laws-c-event-stream -laws-checksums -laws-c-common" # Prevent CRAN builder from linking against old libs in /usr/local/lib for FILE in $BREWDIR/Cellar/*/*/lib/*.a; do From 31fcc398a660d6e4ef2d05e61027efd4daa49b73 Mon Sep 17 00:00:00 2001 From: Neal Richardson Date: Mon, 10 Aug 2020 14:26:39 -0700 Subject: [PATCH 03/13] Set -DAWSSDK_HOME in autobrew formula --- dev/tasks/homebrew-formulae/autobrew/apache-arrow.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/dev/tasks/homebrew-formulae/autobrew/apache-arrow.rb b/dev/tasks/homebrew-formulae/autobrew/apache-arrow.rb index 3f8ece2974f..0e92142b873 100644 --- a/dev/tasks/homebrew-formulae/autobrew/apache-arrow.rb +++ b/dev/tasks/homebrew-formulae/autobrew/apache-arrow.rb @@ -60,6 +60,7 @@ def install -DARROW_VERBOSE_THIRDPARTY_BUILD=ON -DCMAKE_UNITY_BUILD=ON -DPARQUET_BUILD_EXECUTABLES=ON + -DAWSSDK_HOME=#{Formula["aws-sdk-cpp"].prefix} -DLZ4_HOME=#{Formula["lz4"].prefix} -DTHRIFT_HOME=#{Formula["thrift"].prefix} ] From 26151252091fc287c8ee550cfd2222a5b7af65de Mon Sep 17 00:00:00 2001 From: Neal Richardson Date: Tue, 11 Aug 2020 07:19:42 -0700 Subject: [PATCH 04/13] Reorder autobrew dependencies to pass brew audit --- dev/tasks/homebrew-formulae/autobrew/apache-arrow.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/tasks/homebrew-formulae/autobrew/apache-arrow.rb b/dev/tasks/homebrew-formulae/autobrew/apache-arrow.rb index 0e92142b873..c65ce5e6aed 100644 --- a/dev/tasks/homebrew-formulae/autobrew/apache-arrow.rb +++ b/dev/tasks/homebrew-formulae/autobrew/apache-arrow.rb @@ -30,9 +30,9 @@ class ApacheArrow < Formula end # NOTE: if you add something here, be sure to add to PKG_LIBS in r/tools/autobrew - depends_on "aws-sdk-cpp" depends_on "boost" => :build depends_on "cmake" => :build + depends_on "aws-sdk-cpp" depends_on "lz4" depends_on "snappy" depends_on "thrift" From 2ef8d4d8c5bab622910d46af1ad4ce4fe38c5916 Mon Sep 17 00:00:00 2001 From: Neal Richardson Date: Thu, 13 Aug 2020 14:03:08 -0700 Subject: [PATCH 05/13] Only enable S3 for Rtools40, not Rtools35 --- ci/scripts/PKGBUILD | 5 ++++- r/configure.win | 11 +++++++++-- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/ci/scripts/PKGBUILD b/ci/scripts/PKGBUILD index a585d06ee9a..63625399cef 100644 --- a/ci/scripts/PKGBUILD +++ b/ci/scripts/PKGBUILD @@ -75,6 +75,9 @@ build() { export PATH="/C/Rtools${MINGW_PREFIX/mingw/mingw_}/bin:$PATH" export CPPFLAGS="${CPPFLAGS} -I${MINGW_PREFIX}/include" export LIBS="-L${MINGW_PREFIX}/libs" + export ARROW_S3=OFF + else + export ARROW_S3=ON fi MSYS2_ARG_CONV_EXCL="-DCMAKE_INSTALL_PREFIX=" \ @@ -95,7 +98,7 @@ build() { -DARROW_MIMALLOC=ON \ -DARROW_PACKAGE_PREFIX="${MINGW_PREFIX}" \ -DARROW_PARQUET=ON \ - -DARROW_S3=ON \ + -DARROW_S3="${ARROW_S3}" \ -DARROW_SNAPPY_USE_SHARED=OFF \ -DARROW_USE_GLOG=OFF \ -DARROW_WITH_LZ4=ON \ diff --git a/r/configure.win b/r/configure.win index a70c2a8927b..e23be505cb9 100644 --- a/r/configure.win +++ b/r/configure.win @@ -48,8 +48,15 @@ AWS_LIBS="-laws-cpp-sdk-config -laws-cpp-sdk-transfer -laws-cpp-sdk-s3 -laws-cpp # NOTE: If you make changes to the libraries below, you should also change # ci/scripts/r_windows_build.sh and ci/scripts/PKGBUILD -PKG_CFLAGS="-I${RWINLIB}/include -DARROW_STATIC -DPARQUET_STATIC -DARROW_DS_STATIC -DARROW_R_WITH_ARROW -DARROW_R_WITH_S3" -PKG_LIBS="-L${RWINLIB}/lib"'$(subst gcc,,$(COMPILED_BY))$(R_ARCH) '"-L${RWINLIB}/lib"'$(R_ARCH) '"-lparquet -larrow_dataset -larrow -larrow_bundled_dependencies -lthrift -lsnappy -lz -lzstd -llz4 ${MIMALLOC_LIBS} ${OPENSSL_LIBS} ${AWS_LIBS}" +PKG_CFLAGS="-I${RWINLIB}/include -DARROW_STATIC -DPARQUET_STATIC -DARROW_DS_STATIC -DARROW_R_WITH_ARROW" +PKG_LIBS="-L${RWINLIB}/lib"'$(subst gcc,,$(COMPILED_BY))$(R_ARCH) '"-L${RWINLIB}/lib"'$(R_ARCH) '"-lparquet -larrow_dataset -larrow -larrow_bundled_dependencies -lthrift -lsnappy -lz -lzstd -llz4 ${MIMALLOC_LIBS} ${OPENSSL_LIBS}" + +# S3 support only for Rtools40 (i.e. R >= 4.0) +"${R_HOME}/bin${R_ARCH_BIN}/Rscript.exe" -e 'R.version$major >= 4' | grep TRUE >/dev/null 2>&1 +if [ $? -eq 0 ]; then + PKG_CFLAGS="${PKG_CFLAGS} -DARROW_R_WITH_S3" + PKG_LIBS="${PKG_LIBS} ${AWS_LIBS}" +fi # Set any user-defined CXXFLAGS if [ "$ARROW_R_CXXFLAGS" ]; then From 8a0b7b5a2e1edf4479890f74ba8e11e0b999285e Mon Sep 17 00:00:00 2001 From: Neal Richardson Date: Thu, 13 Aug 2020 14:04:35 -0700 Subject: [PATCH 06/13] Move -lws2_32 --- r/configure.win | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/r/configure.win b/r/configure.win index e23be505cb9..5d6be5fbd00 100644 --- a/r/configure.win +++ b/r/configure.win @@ -44,12 +44,12 @@ else fi OPENSSL_LIBS="-lcrypto -lcrypt32" MIMALLOC_LIBS="-lbcrypt -lpsapi" -AWS_LIBS="-laws-cpp-sdk-config -laws-cpp-sdk-transfer -laws-cpp-sdk-s3 -laws-cpp-sdk-core -laws-c-event-stream -laws-checksums -laws-c-common -lUserenv -lversion -lws2_32 -lBcrypt -lWininet -lwinhttp" +AWS_LIBS="-laws-cpp-sdk-config -laws-cpp-sdk-transfer -laws-cpp-sdk-s3 -laws-cpp-sdk-core -laws-c-event-stream -laws-checksums -laws-c-common -lUserenv -lversion -lBcrypt -lWininet -lwinhttp" # NOTE: If you make changes to the libraries below, you should also change # ci/scripts/r_windows_build.sh and ci/scripts/PKGBUILD PKG_CFLAGS="-I${RWINLIB}/include -DARROW_STATIC -DPARQUET_STATIC -DARROW_DS_STATIC -DARROW_R_WITH_ARROW" -PKG_LIBS="-L${RWINLIB}/lib"'$(subst gcc,,$(COMPILED_BY))$(R_ARCH) '"-L${RWINLIB}/lib"'$(R_ARCH) '"-lparquet -larrow_dataset -larrow -larrow_bundled_dependencies -lthrift -lsnappy -lz -lzstd -llz4 ${MIMALLOC_LIBS} ${OPENSSL_LIBS}" +PKG_LIBS="-L${RWINLIB}/lib"'$(subst gcc,,$(COMPILED_BY))$(R_ARCH) '"-L${RWINLIB}/lib"'$(R_ARCH) '"-lparquet -larrow_dataset -larrow -larrow_bundled_dependencies -lthrift -lsnappy -lz -lzstd -llz4 ${MIMALLOC_LIBS} ${OPENSSL_LIBS} -lws2_32" # S3 support only for Rtools40 (i.e. R >= 4.0) "${R_HOME}/bin${R_ARCH_BIN}/Rscript.exe" -e 'R.version$major >= 4' | grep TRUE >/dev/null 2>&1 From ea711c05ea2aa4519890b5b7420003626c3d126b Mon Sep 17 00:00:00 2001 From: Neal Richardson Date: Thu, 13 Aug 2020 14:38:03 -0700 Subject: [PATCH 07/13] Order matters? --- r/configure.win | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/r/configure.win b/r/configure.win index 5d6be5fbd00..094308a888b 100644 --- a/r/configure.win +++ b/r/configure.win @@ -44,18 +44,21 @@ else fi OPENSSL_LIBS="-lcrypto -lcrypt32" MIMALLOC_LIBS="-lbcrypt -lpsapi" -AWS_LIBS="-laws-cpp-sdk-config -laws-cpp-sdk-transfer -laws-cpp-sdk-s3 -laws-cpp-sdk-core -laws-c-event-stream -laws-checksums -laws-c-common -lUserenv -lversion -lBcrypt -lWininet -lwinhttp" +AWS_LIBS="-laws-cpp-sdk-config -laws-cpp-sdk-transfer -laws-cpp-sdk-s3 -laws-cpp-sdk-core -laws-c-event-stream -laws-checksums -laws-c-common -lUserenv -lversion -lws2_32 -lBcrypt -lWininet -lwinhttp" # NOTE: If you make changes to the libraries below, you should also change # ci/scripts/r_windows_build.sh and ci/scripts/PKGBUILD PKG_CFLAGS="-I${RWINLIB}/include -DARROW_STATIC -DPARQUET_STATIC -DARROW_DS_STATIC -DARROW_R_WITH_ARROW" -PKG_LIBS="-L${RWINLIB}/lib"'$(subst gcc,,$(COMPILED_BY))$(R_ARCH) '"-L${RWINLIB}/lib"'$(R_ARCH) '"-lparquet -larrow_dataset -larrow -larrow_bundled_dependencies -lthrift -lsnappy -lz -lzstd -llz4 ${MIMALLOC_LIBS} ${OPENSSL_LIBS} -lws2_32" +PKG_LIBS="-L${RWINLIB}/lib"'$(subst gcc,,$(COMPILED_BY))$(R_ARCH) '"-L${RWINLIB}/lib"'$(R_ARCH) '"-lparquet -larrow_dataset -larrow -larrow_bundled_dependencies -lthrift -lsnappy -lz -lzstd -llz4 ${MIMALLOC_LIBS} ${OPENSSL_LIBS}" # S3 support only for Rtools40 (i.e. R >= 4.0) "${R_HOME}/bin${R_ARCH_BIN}/Rscript.exe" -e 'R.version$major >= 4' | grep TRUE >/dev/null 2>&1 if [ $? -eq 0 ]; then PKG_CFLAGS="${PKG_CFLAGS} -DARROW_R_WITH_S3" PKG_LIBS="${PKG_LIBS} ${AWS_LIBS}" +else + # It seems that order matters + PKG_LIBS="${PKG_LIBS} -lws2_32" fi # Set any user-defined CXXFLAGS From 2d821d8c9bf7bb5ab7527b0afdc3db933ec34edb Mon Sep 17 00:00:00 2001 From: Neal Richardson Date: Fri, 14 Aug 2020 11:07:00 -0700 Subject: [PATCH 08/13] Tweaks to get autobrew working (On My Machine at least) --- dev/tasks/homebrew-formulae/autobrew/apache-arrow.rb | 2 +- r/configure | 5 +++-- r/tools/autobrew | 6 ++++-- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/dev/tasks/homebrew-formulae/autobrew/apache-arrow.rb b/dev/tasks/homebrew-formulae/autobrew/apache-arrow.rb index c65ce5e6aed..1a8223c7c88 100644 --- a/dev/tasks/homebrew-formulae/autobrew/apache-arrow.rb +++ b/dev/tasks/homebrew-formulae/autobrew/apache-arrow.rb @@ -58,7 +58,7 @@ def install -DARROW_WITH_UTF8PROC=OFF -DARROW_BUILD_UTILITIES=ON -DARROW_VERBOSE_THIRDPARTY_BUILD=ON - -DCMAKE_UNITY_BUILD=ON + -DCMAKE_UNITY_BUILD=OFF -DPARQUET_BUILD_EXECUTABLES=ON -DAWSSDK_HOME=#{Formula["aws-sdk-cpp"].prefix} -DLZ4_HOME=#{Formula["lz4"].prefix} diff --git a/r/configure b/r/configure index ecaeff77d6c..4ad48c4d6c4 100755 --- a/r/configure +++ b/r/configure @@ -98,6 +98,8 @@ else if [ "$FORCE_AUTOBREW" != "true" ] && [ "`command -v brew`" ] && [ "`brew ls --versions ${PKG_BREW_NAME}`" != "" ]; then echo "*** Using Homebrew ${PKG_BREW_NAME}" BREWDIR=`brew --prefix` + PKG_LIBS="-L$BREWDIR/opt/$PKG_BREW_NAME/lib $PKG_LIBS -larrow_bundled_dependencies" + PKG_CFLAGS="-I$BREWDIR/opt/$PKG_BREW_NAME/include" else echo "*** Downloading ${PKG_BREW_NAME}" if [ -f "autobrew" ]; then @@ -112,9 +114,8 @@ else if [ $? -ne 0 ]; then echo "Failed to retrieve binary for ${PKG_BREW_NAME}" fi + # autobrew sets `PKG_LIBS` and `PKG_CFLAGS` fi - PKG_CFLAGS="-I$BREWDIR/opt/$PKG_BREW_NAME/include" - PKG_LIBS="-L$BREWDIR/opt/$PKG_BREW_NAME/lib $PKG_LIBS -larrow_bundled_dependencies" elif [ "$UNAME" = "Linux" ]; then # Set some default values/backwards compatibility if [ "${LIBARROW_DOWNLOAD}" = "" ] && [ "${NOT_CRAN}" != "" ]; then diff --git a/r/tools/autobrew b/r/tools/autobrew index 70ffdf9bd18..458919c0b78 100644 --- a/r/tools/autobrew +++ b/r/tools/autobrew @@ -40,14 +40,14 @@ if [ -f "$LOCAL_FORMULA" ]; then $BREW deps -n "$LOCAL_FORMULA" 2>/dev/null BREW_DEPS=$($BREW deps -n "$LOCAL_FORMULA" 2>/dev/null) $BREW install --force-bottle $BREW_DEPS 2>&1 | perl -pe 's/Warning/Note/gi' - $BREW install --build-from-source --HEAD "$LOCAL_FORMULA" 2>&1 | perl -pe 's/Warning/Note/gi' + $BREW install -v --build-from-source --HEAD "$LOCAL_FORMULA" 2>&1 | perl -pe 's/Warning/Note/gi' else $BREW install --force-bottle $BREW_DEPS $PKG_BREW_NAME 2>&1 | perl -pe 's/Warning/Note/gi' fi # Hardcode this for my custom autobrew build rm -f $BREWDIR/lib/*.dylib -PKG_LIBS="-L$BREWDIR/lib -lparquet -larrow_dataset -larrow -lthrift -llz4 -lsnappy -laws-cpp-sdk-config -laws-cpp-sdk-transfer -laws-cpp-sdk-s3 -laws-cpp-sdk-core -laws-c-event-stream -laws-checksums -laws-c-common" +PKG_LIBS="-L$BREWDIR/lib -lparquet -larrow_dataset -larrow -larrow_bundled_dependencies -lthrift -llz4 -lsnappy -laws-cpp-sdk-config -laws-cpp-sdk-transfer -laws-cpp-sdk-s3 -laws-cpp-sdk-core -laws-c-event-stream -laws-checksums -laws-c-common -lpthread -lcurl" # Prevent CRAN builder from linking against old libs in /usr/local/lib for FILE in $BREWDIR/Cellar/*/*/lib/*.a; do @@ -58,5 +58,7 @@ for FILE in $BREWDIR/Cellar/*/*/lib/*.a; do PKG_LIBS=`echo $PKG_LIBS | sed "s/-l$LIBNAME/-lbrew$LIBNAME/g"` done +PKG_CFLAGS="-I$BREWDIR/opt/$PKG_BREW_NAME/include -DARROW_R_WITH_S3" + unset HOMEBREW_NO_ANALYTICS unset HOMEBREW_NO_AUTO_UPDATE From 0a04d735448b136b872d6782d478aa46022d904d Mon Sep 17 00:00:00 2001 From: Neal Richardson Date: Fri, 14 Aug 2020 11:37:13 -0700 Subject: [PATCH 09/13] Always print install logs (so we can confirm -DARROW_R_WITH_S3) --- dev/tasks/homebrew-formulae/travis.osx.r.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/dev/tasks/homebrew-formulae/travis.osx.r.yml b/dev/tasks/homebrew-formulae/travis.osx.r.yml index 9442caeb298..45559e34288 100644 --- a/dev/tasks/homebrew-formulae/travis.osx.r.yml +++ b/dev/tasks/homebrew-formulae/travis.osx.r.yml @@ -49,6 +49,5 @@ script: - Rscript -e 'install.packages("rcmdcheck")' # Note that this is not --as-cran. CRAN doesn't do macOS checks --as-cran - travis_wait Rscript -e "rcmdcheck::rcmdcheck(build_args = '--no-build-vignettes', args = c('--no-manual', '--ignore-vignettes', '--run-donttest'), error_on = 'warning', check_dir = 'check')" -after_failure: -# If there's a build failure, it's probably in this log +# If there's a build failure, it's probably in this log. Let's print it regardless though - cat arrow.Rcheck/00install.out From 06f31806a13af954359d87c6cbbd1738a6968ab2 Mon Sep 17 00:00:00 2001 From: Neal Richardson Date: Mon, 17 Aug 2020 13:35:14 -0700 Subject: [PATCH 10/13] Fix path to install log --- dev/tasks/homebrew-formulae/travis.osx.r.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/tasks/homebrew-formulae/travis.osx.r.yml b/dev/tasks/homebrew-formulae/travis.osx.r.yml index 45559e34288..020010111b1 100644 --- a/dev/tasks/homebrew-formulae/travis.osx.r.yml +++ b/dev/tasks/homebrew-formulae/travis.osx.r.yml @@ -50,4 +50,4 @@ script: # Note that this is not --as-cran. CRAN doesn't do macOS checks --as-cran - travis_wait Rscript -e "rcmdcheck::rcmdcheck(build_args = '--no-build-vignettes', args = c('--no-manual', '--ignore-vignettes', '--run-donttest'), error_on = 'warning', check_dir = 'check')" # If there's a build failure, it's probably in this log. Let's print it regardless though -- cat arrow.Rcheck/00install.out +- cat check/arrow.Rcheck/00install.out From d3fafa03d553a232c682a116984ffbb66f84d8c2 Mon Sep 17 00:00:00 2001 From: Neal Richardson Date: Tue, 18 Aug 2020 07:27:38 -0700 Subject: [PATCH 11/13] Own the (aws) libs --- r/configure.win | 2 +- r/tools/autobrew | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/r/configure.win b/r/configure.win index 094308a888b..886205fdafa 100644 --- a/r/configure.win +++ b/r/configure.win @@ -44,7 +44,7 @@ else fi OPENSSL_LIBS="-lcrypto -lcrypt32" MIMALLOC_LIBS="-lbcrypt -lpsapi" -AWS_LIBS="-laws-cpp-sdk-config -laws-cpp-sdk-transfer -laws-cpp-sdk-s3 -laws-cpp-sdk-core -laws-c-event-stream -laws-checksums -laws-c-common -lUserenv -lversion -lws2_32 -lBcrypt -lWininet -lwinhttp" +AWS_LIBS="-laws-cpp-sdk-config -laws-cpp-sdk-transfer -laws-cpp-sdk-identity-management -laws-cpp-sdk-cognito-identity -laws-cpp-sdk-sts -laws-cpp-sdk-s3 -laws-cpp-sdk-core -laws-c-event-stream -laws-checksums -laws-c-common -lUserenv -lversion -lws2_32 -lBcrypt -lWininet -lwinhttp" # NOTE: If you make changes to the libraries below, you should also change # ci/scripts/r_windows_build.sh and ci/scripts/PKGBUILD diff --git a/r/tools/autobrew b/r/tools/autobrew index 458919c0b78..bb02bf88aee 100644 --- a/r/tools/autobrew +++ b/r/tools/autobrew @@ -47,7 +47,8 @@ fi # Hardcode this for my custom autobrew build rm -f $BREWDIR/lib/*.dylib -PKG_LIBS="-L$BREWDIR/lib -lparquet -larrow_dataset -larrow -larrow_bundled_dependencies -lthrift -llz4 -lsnappy -laws-cpp-sdk-config -laws-cpp-sdk-transfer -laws-cpp-sdk-s3 -laws-cpp-sdk-core -laws-c-event-stream -laws-checksums -laws-c-common -lpthread -lcurl" +AWS_LIBS="-laws-cpp-sdk-config -laws-cpp-sdk-transfer -laws-cpp-sdk-identity-management -laws-cpp-sdk-cognito-identity -laws-cpp-sdk-sts -laws-cpp-sdk-s3 -laws-cpp-sdk-core -laws-c-event-stream -laws-checksums -laws-c-common -lpthread -lcurl" +PKG_LIBS="-L$BREWDIR/lib -lparquet -larrow_dataset -larrow -larrow_bundled_dependencies -lthrift -llz4 -lsnappy $AWS_LIBS" # Prevent CRAN builder from linking against old libs in /usr/local/lib for FILE in $BREWDIR/Cellar/*/*/lib/*.a; do From 363ee27bd0e6812d75084d920c49d52af342e8e8 Mon Sep 17 00:00:00 2001 From: Neal Richardson Date: Thu, 20 Aug 2020 07:51:02 -0700 Subject: [PATCH 12/13] Switch back to official pacman repo --- ci/scripts/r_windows_build.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ci/scripts/r_windows_build.sh b/ci/scripts/r_windows_build.sh index c736b6ad8b3..b084975fb12 100755 --- a/ci/scripts/r_windows_build.sh +++ b/ci/scripts/r_windows_build.sh @@ -35,10 +35,10 @@ if [ "$RTOOLS_VERSION" = "35" ]; then RWINLIB_LIB_DIR="lib-4.9.3" else # Uncomment L38-41 if you're testing a new rtools dependency that hasn't yet sync'd to CRAN - curl https://raw.githubusercontent.com/r-windows/rtools-packages/master/pacman.conf > /etc/pacman.conf - curl -OSsl "http://repo.msys2.org/msys/x86_64/msys2-keyring-r21.b39fb11-1-any.pkg.tar.xz" - pacman -U --noconfirm msys2-keyring-r21.b39fb11-1-any.pkg.tar.xz && rm msys2-keyring-r21.b39fb11-1-any.pkg.tar.xz - pacman --noconfirm -Scc + # curl https://raw.githubusercontent.com/r-windows/rtools-packages/master/pacman.conf > /etc/pacman.conf + # curl -OSsl "http://repo.msys2.org/msys/x86_64/msys2-keyring-r21.b39fb11-1-any.pkg.tar.xz" + # pacman -U --noconfirm msys2-keyring-r21.b39fb11-1-any.pkg.tar.xz && rm msys2-keyring-r21.b39fb11-1-any.pkg.tar.xz + # pacman --noconfirm -Scc pacman --noconfirm -Syy RWINLIB_LIB_DIR="lib" From 97517a901118d110076b3565ddd984962ce9532f Mon Sep 17 00:00:00 2001 From: Neal Richardson Date: Thu, 20 Aug 2020 08:01:00 -0700 Subject: [PATCH 13/13] Try pruning formula --- dev/tasks/homebrew-formulae/autobrew/apache-arrow.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/dev/tasks/homebrew-formulae/autobrew/apache-arrow.rb b/dev/tasks/homebrew-formulae/autobrew/apache-arrow.rb index 1a8223c7c88..5018df000d7 100644 --- a/dev/tasks/homebrew-formulae/autobrew/apache-arrow.rb +++ b/dev/tasks/homebrew-formulae/autobrew/apache-arrow.rb @@ -60,7 +60,6 @@ def install -DARROW_VERBOSE_THIRDPARTY_BUILD=ON -DCMAKE_UNITY_BUILD=OFF -DPARQUET_BUILD_EXECUTABLES=ON - -DAWSSDK_HOME=#{Formula["aws-sdk-cpp"].prefix} -DLZ4_HOME=#{Formula["lz4"].prefix} -DTHRIFT_HOME=#{Formula["thrift"].prefix} ]