From bd480b72095f04c950845bde20bf36ad1fd28be4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Tue, 6 Oct 2020 00:09:16 +0200 Subject: [PATCH 1/4] enable s3 in the macos wheels --- dev/tasks/python-wheels/osx-build.sh | 5 +++ dev/tasks/python-wheels/travis.osx.yml | 5 ++- dev/tasks/tasks.yml | 58 ++++++++++++++++++++++++-- 3 files changed, 62 insertions(+), 6 deletions(-) diff --git a/dev/tasks/python-wheels/osx-build.sh b/dev/tasks/python-wheels/osx-build.sh index a0bf7b81f5e..d8fb1eb4581 100755 --- a/dev/tasks/python-wheels/osx-build.sh +++ b/dev/tasks/python-wheels/osx-build.sh @@ -67,6 +67,7 @@ function build_wheel { -DARROW_GANDIVA=OFF \ -DARROW_GRPC_USE_SHARED=OFF \ -DARROW_HDFS=ON \ + -DARROW_S3=${ARROW_S3} \ -DARROW_JEMALLOC=ON \ -DARROW_OPENSSL_USE_SHARED=OFF \ -DARROW_ORC=OFF \ @@ -103,6 +104,7 @@ function build_wheel { export PYARROW_WITH_DATASET=1 export PYARROW_WITH_FLIGHT=1 export PYARROW_WITH_HDFS=1 + export PYARROW_WITH_S3=${ARROW_S3} export PYARROW_WITH_PLASMA=1 export PYARROW_WITH_PARQUET=1 export PYARROW_WITH_ORC=0 @@ -158,4 +160,7 @@ import pyarrow._hdfs import pyarrow.dataset import pyarrow.flight " + if [ "$ARROW_S3" = "ON" ]; then + python -c "import pyarrow._s3fs" + fi } diff --git a/dev/tasks/python-wheels/travis.osx.yml b/dev/tasks/python-wheels/travis.osx.yml index 172333fd690..a5b06e8c6a4 100644 --- a/dev/tasks/python-wheels/travis.osx.yml +++ b/dev/tasks/python-wheels/travis.osx.yml @@ -45,9 +45,10 @@ env: - TRAVIS_TAG={{ task.tag }} - PYARROW_VERSION={{ arrow.no_rc_version }} - PYARROW_BUILD_VERBOSE=1 - - MB_PYTHON_VERSION={{ python_version }} - - MACOSX_DEPLOYMENT_TARGET="10.9" - HOMEBREW_NO_AUTO_UPDATE=1 + - MB_PYTHON_VERSION={{ python_version }} + - MACOSX_DEPLOYMENT_TARGET={{ macos_deployment_target }} + - ARROW_S3={{ arrow_s3 }} before_install: - git clone https://github.com/matthew-brett/multibuild diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml index 76ce517cdb0..d5642316042 100644 --- a/dev/tasks/tasks.yml +++ b/dev/tasks/tasks.yml @@ -444,38 +444,88 @@ tasks: ############################## Wheel OSX #################################### - wheel-osx-cp35m: + wheel-osx-mavericks-cp35m: ci: travis template: python-wheels/travis.osx.yml params: python_version: 3.5 + macos_deployment_target: 10.9 + arrow_s3: "OFF" artifacts: - pyarrow-{no_rc_version}-cp35-cp35m-macosx_10_9_intel.whl - wheel-osx-cp36m: + wheel-osx-mavericks-cp36m: ci: travis template: python-wheels/travis.osx.yml params: python_version: 3.6 + macos_deployment_target: 10.9 + arrow_s3: "OFF" artifacts: - pyarrow-{no_rc_version}-cp36-cp36m-macosx_10_9_intel.whl - wheel-osx-cp37m: + wheel-osx-mavericks-cp37m: ci: travis template: python-wheels/travis.osx.yml params: python_version: 3.7 + macos_deployment_target: 10.9 + arrow_s3: "OFF" artifacts: - pyarrow-{no_rc_version}-cp37-cp37m-macosx_10_9_intel.whl - wheel-osx-cp38: + wheel-osx-mavericks-cp38: ci: travis template: python-wheels/travis.osx.yml params: python_version: 3.8 + macos_deployment_target: 10.9 + arrow_s3: "ON" artifacts: - pyarrow-{no_rc_version}-cp38-cp38-macosx_10_9_x86_64.whl + # enable S3 support from macOS 10.13 so we don't need to bundle curl, crypt and ssl + + wheel-osx-high-sierra-cp35m: + ci: travis + template: python-wheels/travis.osx.yml + params: + python_version: 3.5 + macos_deployment_target: 10.13 + arrow_s3: "ON" + artifacts: + - pyarrow-{no_rc_version}-cp35-cp35m-macosx_10_13_intel.whl + + wheel-osx-high-sierra-cp36m: + ci: travis + template: python-wheels/travis.osx.yml + params: + python_version: 3.6 + macos_deployment_target: 10.13 + arrow_s3: "ON" + artifacts: + - pyarrow-{no_rc_version}-cp36-cp36m-macosx_10_13_intel.whl + + wheel-osx-high-sierra-cp37m: + ci: travis + template: python-wheels/travis.osx.yml + params: + python_version: 3.7 + macos_deployment_target: 10.13 + arrow_s3: "ON" + artifacts: + - pyarrow-{no_rc_version}-cp37-cp37m-macosx_10_13_intel.whl + + wheel-osx-high-sierra-cp38: + ci: travis + template: python-wheels/travis.osx.yml + params: + python_version: 3.8 + macos_deployment_target: 10.13 + arrow_s3: "ON" + artifacts: + - pyarrow-{no_rc_version}-cp38-cp38-macosx_10_13_x86_64.whl + ############################## Wheel Windows ################################ wheel-win-cp35m: From 97b5b3248a2becd7bf86d89cb8bd71de2bf43e30 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Tue, 6 Oct 2020 12:01:09 +0200 Subject: [PATCH 2/4] sort --- dev/tasks/python-wheels/osx-build.sh | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/dev/tasks/python-wheels/osx-build.sh b/dev/tasks/python-wheels/osx-build.sh index d8fb1eb4581..e029932ecf6 100755 --- a/dev/tasks/python-wheels/osx-build.sh +++ b/dev/tasks/python-wheels/osx-build.sh @@ -67,7 +67,6 @@ function build_wheel { -DARROW_GANDIVA=OFF \ -DARROW_GRPC_USE_SHARED=OFF \ -DARROW_HDFS=ON \ - -DARROW_S3=${ARROW_S3} \ -DARROW_JEMALLOC=ON \ -DARROW_OPENSSL_USE_SHARED=OFF \ -DARROW_ORC=OFF \ @@ -76,6 +75,7 @@ function build_wheel { -DARROW_PROTOBUF_USE_SHARED=OFF \ -DARROW_PYTHON=ON \ -DARROW_RPATH_ORIGIN=ON \ + -DARROW_S3=${ARROW_S3} \ -DARROW_VERBOSE_THIRDPARTY_BUILD=ON \ -DARROW_WITH_BROTLI=ON \ -DARROW_WITH_BZ2=ON \ @@ -101,19 +101,19 @@ function build_wheel { unset ARROW_HOME unset PARQUET_HOME + export PYARROW_BUILD_TYPE='release' + export PYARROW_BUNDLE_ARROW_CPP=1 + export PYARROW_INSTALL_TESTS=1 export PYARROW_WITH_DATASET=1 export PYARROW_WITH_FLIGHT=1 + export PYARROW_WITH_GANDIVA=0 export PYARROW_WITH_HDFS=1 - export PYARROW_WITH_S3=${ARROW_S3} - export PYARROW_WITH_PLASMA=1 - export PYARROW_WITH_PARQUET=1 - export PYARROW_WITH_ORC=0 export PYARROW_WITH_JEMALLOC=1 + export PYARROW_WITH_ORC=0 + export PYARROW_WITH_PARQUET=1 export PYARROW_WITH_PLASMA=1 - export PYARROW_WITH_GANDIVA=0 - export PYARROW_BUNDLE_ARROW_CPP=1 - export PYARROW_BUILD_TYPE='release' - export PYARROW_INSTALL_TESTS=1 + export PYARROW_WITH_PLASMA=1 + export PYARROW_WITH_S3=${ARROW_S3} export SETUPTOOLS_SCM_PRETEND_VERSION=$PYARROW_VERSION pushd python python setup.py build_ext bdist_wheel From c40e6e4ada9a8ce7cc2a594eedf5d085a5b17fc8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Wed, 7 Oct 2020 15:05:54 +0200 Subject: [PATCH 3/4] try not updating brew --- dev/tasks/python-wheels/travis.osx.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/tasks/python-wheels/travis.osx.yml b/dev/tasks/python-wheels/travis.osx.yml index a5b06e8c6a4..99e8591d013 100644 --- a/dev/tasks/python-wheels/travis.osx.yml +++ b/dev/tasks/python-wheels/travis.osx.yml @@ -34,7 +34,7 @@ addons: - python@3.8 - thrift - wget - update: true + update: false # don't build twice if: tag IS blank From 7bcd83e9096bc4ff4078b5b908e8ca295b0c3192 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Wed, 7 Oct 2020 16:34:01 +0200 Subject: [PATCH 4/4] turn off s3 on mavericks py38 --- dev/tasks/tasks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml index d5642316042..e4d3d0a6078 100644 --- a/dev/tasks/tasks.yml +++ b/dev/tasks/tasks.yml @@ -480,7 +480,7 @@ tasks: params: python_version: 3.8 macos_deployment_target: 10.9 - arrow_s3: "ON" + arrow_s3: "OFF" artifacts: - pyarrow-{no_rc_version}-cp38-cp38-macosx_10_9_x86_64.whl