diff --git a/ci/scripts/PKGBUILD b/ci/scripts/PKGBUILD index df5f5860445..63625399cef 100644 --- a/ci/scripts/PKGBUILD +++ b/ci/scripts/PKGBUILD @@ -24,7 +24,8 @@ pkgdesc="Apache Arrow is a cross-language development platform for in-memory dat arch=("any") url="https://arrow.apache.org/" license=("Apache-2.0") -depends=("${MINGW_PACKAGE_PREFIX}-thrift" +depends=("${MINGW_PACKAGE_PREFIX}-aws-sdk-cpp" + "${MINGW_PACKAGE_PREFIX}-thrift" "${MINGW_PACKAGE_PREFIX}-snappy" "${MINGW_PACKAGE_PREFIX}-zlib" "${MINGW_PACKAGE_PREFIX}-lz4" @@ -74,6 +75,9 @@ build() { export PATH="/C/Rtools${MINGW_PREFIX/mingw/mingw_}/bin:$PATH" export CPPFLAGS="${CPPFLAGS} -I${MINGW_PREFIX}/include" export LIBS="-L${MINGW_PREFIX}/libs" + export ARROW_S3=OFF + else + export ARROW_S3=ON fi MSYS2_ARG_CONV_EXCL="-DCMAKE_INSTALL_PREFIX=" \ @@ -94,6 +98,7 @@ build() { -DARROW_MIMALLOC=ON \ -DARROW_PACKAGE_PREFIX="${MINGW_PREFIX}" \ -DARROW_PARQUET=ON \ + -DARROW_S3="${ARROW_S3}" \ -DARROW_SNAPPY_USE_SHARED=OFF \ -DARROW_USE_GLOG=OFF \ -DARROW_WITH_LZ4=ON \ diff --git a/ci/scripts/r_windows_build.sh b/ci/scripts/r_windows_build.sh index ed9e211f9ac..b084975fb12 100755 --- a/ci/scripts/r_windows_build.sh +++ b/ci/scripts/r_windows_build.sh @@ -34,6 +34,12 @@ if [ "$RTOOLS_VERSION" = "35" ]; then # lib-4.9.3 is for libraries compiled with gcc 4.9 (Rtools 3.5) RWINLIB_LIB_DIR="lib-4.9.3" else + # Uncomment L38-41 if you're testing a new rtools dependency that hasn't yet sync'd to CRAN + # curl https://raw.githubusercontent.com/r-windows/rtools-packages/master/pacman.conf > /etc/pacman.conf + # curl -OSsl "http://repo.msys2.org/msys/x86_64/msys2-keyring-r21.b39fb11-1-any.pkg.tar.xz" + # pacman -U --noconfirm msys2-keyring-r21.b39fb11-1-any.pkg.tar.xz && rm msys2-keyring-r21.b39fb11-1-any.pkg.tar.xz + # pacman --noconfirm -Scc + pacman --noconfirm -Syy RWINLIB_LIB_DIR="lib" fi @@ -84,8 +90,8 @@ cp $MSYS_LIB_DIR/mingw64/lib/lib{thrift,snappy}.a $DST_DIR/${RWINLIB_LIB_DIR}/x6 cp $MSYS_LIB_DIR/mingw32/lib/lib{thrift,snappy}.a $DST_DIR/${RWINLIB_LIB_DIR}/i386 # These are from https://dl.bintray.com/rtools/mingw{32,64}/ -cp $MSYS_LIB_DIR/mingw64/lib/lib{zstd,lz4,crypto}.a $DST_DIR/lib/x64 -cp $MSYS_LIB_DIR/mingw32/lib/lib{zstd,lz4,crypto}.a $DST_DIR/lib/i386 +cp $MSYS_LIB_DIR/mingw64/lib/lib{zstd,lz4,crypto,aws*}.a $DST_DIR/lib/x64 +cp $MSYS_LIB_DIR/mingw32/lib/lib{zstd,lz4,crypto,aws*}.a $DST_DIR/lib/i386 # Create build artifact zip -r ${DST_DIR}.zip $DST_DIR diff --git a/dev/tasks/homebrew-formulae/autobrew/apache-arrow.rb b/dev/tasks/homebrew-formulae/autobrew/apache-arrow.rb index 725847acd9c..5018df000d7 100644 --- a/dev/tasks/homebrew-formulae/autobrew/apache-arrow.rb +++ b/dev/tasks/homebrew-formulae/autobrew/apache-arrow.rb @@ -32,6 +32,7 @@ class ApacheArrow < Formula # NOTE: if you add something here, be sure to add to PKG_LIBS in r/tools/autobrew depends_on "boost" => :build depends_on "cmake" => :build + depends_on "aws-sdk-cpp" depends_on "lz4" depends_on "snappy" depends_on "thrift" @@ -50,14 +51,14 @@ def install -DARROW_JEMALLOC=ON -DARROW_USE_GLOG=OFF -DARROW_PYTHON=OFF - -DARROW_S3=OFF + -DARROW_S3=ON -DARROW_WITH_LZ4=ON -DARROW_WITH_ZLIB=ON -DARROW_WITH_SNAPPY=ON -DARROW_WITH_UTF8PROC=OFF -DARROW_BUILD_UTILITIES=ON -DARROW_VERBOSE_THIRDPARTY_BUILD=ON - -DCMAKE_UNITY_BUILD=ON + -DCMAKE_UNITY_BUILD=OFF -DPARQUET_BUILD_EXECUTABLES=ON -DLZ4_HOME=#{Formula["lz4"].prefix} -DTHRIFT_HOME=#{Formula["thrift"].prefix} diff --git a/dev/tasks/homebrew-formulae/travis.osx.r.yml b/dev/tasks/homebrew-formulae/travis.osx.r.yml index 9442caeb298..020010111b1 100644 --- a/dev/tasks/homebrew-formulae/travis.osx.r.yml +++ b/dev/tasks/homebrew-formulae/travis.osx.r.yml @@ -49,6 +49,5 @@ script: - Rscript -e 'install.packages("rcmdcheck")' # Note that this is not --as-cran. CRAN doesn't do macOS checks --as-cran - travis_wait Rscript -e "rcmdcheck::rcmdcheck(build_args = '--no-build-vignettes', args = c('--no-manual', '--ignore-vignettes', '--run-donttest'), error_on = 'warning', check_dir = 'check')" -after_failure: -# If there's a build failure, it's probably in this log -- cat arrow.Rcheck/00install.out +# If there's a build failure, it's probably in this log. Let's print it regardless though +- cat check/arrow.Rcheck/00install.out diff --git a/r/configure b/r/configure index ecaeff77d6c..4ad48c4d6c4 100755 --- a/r/configure +++ b/r/configure @@ -98,6 +98,8 @@ else if [ "$FORCE_AUTOBREW" != "true" ] && [ "`command -v brew`" ] && [ "`brew ls --versions ${PKG_BREW_NAME}`" != "" ]; then echo "*** Using Homebrew ${PKG_BREW_NAME}" BREWDIR=`brew --prefix` + PKG_LIBS="-L$BREWDIR/opt/$PKG_BREW_NAME/lib $PKG_LIBS -larrow_bundled_dependencies" + PKG_CFLAGS="-I$BREWDIR/opt/$PKG_BREW_NAME/include" else echo "*** Downloading ${PKG_BREW_NAME}" if [ -f "autobrew" ]; then @@ -112,9 +114,8 @@ else if [ $? -ne 0 ]; then echo "Failed to retrieve binary for ${PKG_BREW_NAME}" fi + # autobrew sets `PKG_LIBS` and `PKG_CFLAGS` fi - PKG_CFLAGS="-I$BREWDIR/opt/$PKG_BREW_NAME/include" - PKG_LIBS="-L$BREWDIR/opt/$PKG_BREW_NAME/lib $PKG_LIBS -larrow_bundled_dependencies" elif [ "$UNAME" = "Linux" ]; then # Set some default values/backwards compatibility if [ "${LIBARROW_DOWNLOAD}" = "" ] && [ "${NOT_CRAN}" != "" ]; then diff --git a/r/configure.win b/r/configure.win index 5caf8e24663..886205fdafa 100644 --- a/r/configure.win +++ b/r/configure.win @@ -44,11 +44,22 @@ else fi OPENSSL_LIBS="-lcrypto -lcrypt32" MIMALLOC_LIBS="-lbcrypt -lpsapi" +AWS_LIBS="-laws-cpp-sdk-config -laws-cpp-sdk-transfer -laws-cpp-sdk-identity-management -laws-cpp-sdk-cognito-identity -laws-cpp-sdk-sts -laws-cpp-sdk-s3 -laws-cpp-sdk-core -laws-c-event-stream -laws-checksums -laws-c-common -lUserenv -lversion -lws2_32 -lBcrypt -lWininet -lwinhttp" # NOTE: If you make changes to the libraries below, you should also change # ci/scripts/r_windows_build.sh and ci/scripts/PKGBUILD PKG_CFLAGS="-I${RWINLIB}/include -DARROW_STATIC -DPARQUET_STATIC -DARROW_DS_STATIC -DARROW_R_WITH_ARROW" -PKG_LIBS="-L${RWINLIB}/lib"'$(subst gcc,,$(COMPILED_BY))$(R_ARCH) '"-L${RWINLIB}/lib"'$(R_ARCH) '"-lparquet -larrow_dataset -larrow -larrow_bundled_dependencies -lthrift -lsnappy -lz -lzstd -llz4 ${MIMALLOC_LIBS} ${OPENSSL_LIBS} -lws2_32" +PKG_LIBS="-L${RWINLIB}/lib"'$(subst gcc,,$(COMPILED_BY))$(R_ARCH) '"-L${RWINLIB}/lib"'$(R_ARCH) '"-lparquet -larrow_dataset -larrow -larrow_bundled_dependencies -lthrift -lsnappy -lz -lzstd -llz4 ${MIMALLOC_LIBS} ${OPENSSL_LIBS}" + +# S3 support only for Rtools40 (i.e. R >= 4.0) +"${R_HOME}/bin${R_ARCH_BIN}/Rscript.exe" -e 'R.version$major >= 4' | grep TRUE >/dev/null 2>&1 +if [ $? -eq 0 ]; then + PKG_CFLAGS="${PKG_CFLAGS} -DARROW_R_WITH_S3" + PKG_LIBS="${PKG_LIBS} ${AWS_LIBS}" +else + # It seems that order matters + PKG_LIBS="${PKG_LIBS} -lws2_32" +fi # Set any user-defined CXXFLAGS if [ "$ARROW_R_CXXFLAGS" ]; then diff --git a/r/tools/autobrew b/r/tools/autobrew index 7dcfffb22d1..bb02bf88aee 100644 --- a/r/tools/autobrew +++ b/r/tools/autobrew @@ -40,14 +40,15 @@ if [ -f "$LOCAL_FORMULA" ]; then $BREW deps -n "$LOCAL_FORMULA" 2>/dev/null BREW_DEPS=$($BREW deps -n "$LOCAL_FORMULA" 2>/dev/null) $BREW install --force-bottle $BREW_DEPS 2>&1 | perl -pe 's/Warning/Note/gi' - $BREW install --build-from-source --HEAD "$LOCAL_FORMULA" 2>&1 | perl -pe 's/Warning/Note/gi' + $BREW install -v --build-from-source --HEAD "$LOCAL_FORMULA" 2>&1 | perl -pe 's/Warning/Note/gi' else $BREW install --force-bottle $BREW_DEPS $PKG_BREW_NAME 2>&1 | perl -pe 's/Warning/Note/gi' fi # Hardcode this for my custom autobrew build rm -f $BREWDIR/lib/*.dylib -PKG_LIBS="-L$BREWDIR/lib -lparquet -larrow_dataset -larrow -lthrift -llz4 -lsnappy" +AWS_LIBS="-laws-cpp-sdk-config -laws-cpp-sdk-transfer -laws-cpp-sdk-identity-management -laws-cpp-sdk-cognito-identity -laws-cpp-sdk-sts -laws-cpp-sdk-s3 -laws-cpp-sdk-core -laws-c-event-stream -laws-checksums -laws-c-common -lpthread -lcurl" +PKG_LIBS="-L$BREWDIR/lib -lparquet -larrow_dataset -larrow -larrow_bundled_dependencies -lthrift -llz4 -lsnappy $AWS_LIBS" # Prevent CRAN builder from linking against old libs in /usr/local/lib for FILE in $BREWDIR/Cellar/*/*/lib/*.a; do @@ -58,5 +59,7 @@ for FILE in $BREWDIR/Cellar/*/*/lib/*.a; do PKG_LIBS=`echo $PKG_LIBS | sed "s/-l$LIBNAME/-lbrew$LIBNAME/g"` done +PKG_CFLAGS="-I$BREWDIR/opt/$PKG_BREW_NAME/include -DARROW_R_WITH_S3" + unset HOMEBREW_NO_ANALYTICS unset HOMEBREW_NO_AUTO_UPDATE