From ad77b8fdacb942cee7556e20b09fc8dcf82c5e1a Mon Sep 17 00:00:00 2001 From: Matt Corley Date: Tue, 18 Aug 2020 14:56:17 -0700 Subject: [PATCH 1/6] ARROW-9266: [Python/Packaging] enable C++ S3FS in macOS wheels --- dev/tasks/python-wheels/osx-build.sh | 3 +++ dev/tasks/python-wheels/travis.osx.yml | 1 + 2 files changed, 4 insertions(+) diff --git a/dev/tasks/python-wheels/osx-build.sh b/dev/tasks/python-wheels/osx-build.sh index 91292905303..06d662df7e5 100755 --- a/dev/tasks/python-wheels/osx-build.sh +++ b/dev/tasks/python-wheels/osx-build.sh @@ -74,6 +74,7 @@ function build_wheel { -DARROW_PROTOBUF_USE_SHARED=OFF \ -DARROW_PYTHON=ON \ -DARROW_RPATH_ORIGIN=ON \ + -DARROW_S3=ON \ -DARROW_VERBOSE_THIRDPARTY_BUILD=ON \ -DARROW_WITH_BROTLI=ON \ -DARROW_WITH_BZ2=ON \ @@ -108,6 +109,7 @@ function build_wheel { export PYARROW_WITH_PLASMA=1 export PYARROW_WITH_PARQUET=1 export PYARROW_WITH_ORC=0 + export PYARROW_WITH_S3=1 export PYARROW_WITH_JEMALLOC=1 export PYARROW_WITH_PLASMA=1 export PYARROW_WITH_GANDIVA=0 @@ -157,6 +159,7 @@ import pyarrow.parquet import pyarrow.plasma import pyarrow.fs import pyarrow._hdfs +import pyarrow._s3fs import pyarrow.dataset import pyarrow.flight " diff --git a/dev/tasks/python-wheels/travis.osx.yml b/dev/tasks/python-wheels/travis.osx.yml index f2a62b9443a..c438e610f01 100644 --- a/dev/tasks/python-wheels/travis.osx.yml +++ b/dev/tasks/python-wheels/travis.osx.yml @@ -21,6 +21,7 @@ language: generic addons: homebrew: packages: + - aws-sdk-cpp - automake - boost - brotli From 5dd1f2221732850eaef66439faa9afec51c2a79b Mon Sep 17 00:00:00 2001 From: Matt Corley Date: Mon, 24 Aug 2020 11:58:42 -0700 Subject: [PATCH 2/6] set AWSSDK_SOURCE=SYSTEM for osx wheel build --- dev/tasks/python-wheels/osx-build.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/dev/tasks/python-wheels/osx-build.sh b/dev/tasks/python-wheels/osx-build.sh index 06d662df7e5..02de8176e9b 100755 --- a/dev/tasks/python-wheels/osx-build.sh +++ b/dev/tasks/python-wheels/osx-build.sh @@ -82,6 +82,7 @@ function build_wheel { -DARROW_WITH_SNAPPY=ON \ -DARROW_WITH_ZLIB=ON \ -DARROW_WITH_ZSTD=ON \ + -DAWSSDK_SOURCE=SYSTEM \ -DBOOST_SOURCE=SYSTEM \ -DCMAKE_BUILD_TYPE=Release \ -DCMAKE_INSTALL_PREFIX=$ARROW_HOME \ From e2359f53f4f053ea9181963d3859d7d43c5dee04 Mon Sep 17 00:00:00 2001 From: Matt Corley Date: Mon, 24 Aug 2020 12:00:44 -0700 Subject: [PATCH 3/6] alphabetize homebrew packages --- dev/tasks/python-wheels/travis.osx.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dev/tasks/python-wheels/travis.osx.yml b/dev/tasks/python-wheels/travis.osx.yml index c438e610f01..26fb089c671 100644 --- a/dev/tasks/python-wheels/travis.osx.yml +++ b/dev/tasks/python-wheels/travis.osx.yml @@ -21,14 +21,16 @@ language: generic addons: homebrew: packages: - - aws-sdk-cpp - automake + - aws-sdk-cpp + - bison - boost - brotli - c-ares - ccache - cmake - flatbuffers + - flex - git - glog - grpc @@ -44,8 +46,6 @@ addons: - thrift - wget - zstd - - bison - - flex update: true # don't build twice From 2722ca57357f9f14cdae7f7eb7ae33760e35b039 Mon Sep 17 00:00:00 2001 From: Matt Corley Date: Mon, 24 Aug 2020 15:02:07 -0700 Subject: [PATCH 4/6] enable build of AWS C++ SDK --- cpp/cmake_modules/ThirdpartyToolchain.cmake | 2 -- 1 file changed, 2 deletions(-) diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index 6e508049707..95a23042939 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -2612,8 +2612,6 @@ endif() # AWS SDK for C++ macro(build_awssdk) - message( - FATAL_ERROR "FIXME: Building AWS C++ SDK from source will link with wrong libcrypto") message("Building AWS C++ SDK from source") set(AWSSDK_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/awssdk_ep-install") From 0b3365dbee909715c7fa7dbdaeebac50357f72d2 Mon Sep 17 00:00:00 2001 From: Matt Corley Date: Mon, 24 Aug 2020 15:46:55 -0700 Subject: [PATCH 5/6] Revert "set AWSSDK_SOURCE=SYSTEM for osx wheel build" This reverts commit 5dd1f2221732850eaef66439faa9afec51c2a79b. --- dev/tasks/python-wheels/osx-build.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/dev/tasks/python-wheels/osx-build.sh b/dev/tasks/python-wheels/osx-build.sh index 02de8176e9b..06d662df7e5 100755 --- a/dev/tasks/python-wheels/osx-build.sh +++ b/dev/tasks/python-wheels/osx-build.sh @@ -82,7 +82,6 @@ function build_wheel { -DARROW_WITH_SNAPPY=ON \ -DARROW_WITH_ZLIB=ON \ -DARROW_WITH_ZSTD=ON \ - -DAWSSDK_SOURCE=SYSTEM \ -DBOOST_SOURCE=SYSTEM \ -DCMAKE_BUILD_TYPE=Release \ -DCMAKE_INSTALL_PREFIX=$ARROW_HOME \ From e7948bed71cc81ee2f25f8a01ebf3c383e1fac9c Mon Sep 17 00:00:00 2001 From: Matt Corley Date: Mon, 24 Aug 2020 19:02:04 -0700 Subject: [PATCH 6/6] remove aws-sdk-cpp homebrew dependency --- dev/tasks/python-wheels/travis.osx.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/dev/tasks/python-wheels/travis.osx.yml b/dev/tasks/python-wheels/travis.osx.yml index 26fb089c671..e3584ca7ad5 100644 --- a/dev/tasks/python-wheels/travis.osx.yml +++ b/dev/tasks/python-wheels/travis.osx.yml @@ -22,7 +22,6 @@ addons: homebrew: packages: - automake - - aws-sdk-cpp - bison - boost - brotli