diff --git a/cpp/Brewfile b/cpp/Brewfile index f0f58a52105..f338e91efbb 100644 --- a/cpp/Brewfile +++ b/cpp/Brewfile @@ -29,7 +29,7 @@ brew "llvm@7" brew "lz4" brew "ninja" brew "numpy" -brew "openssl" +brew "openssl@1.1" brew "protobuf" brew "python" brew "rapidjson" diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index fcd82e313d9..0076fc62353 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -767,11 +767,18 @@ if(PARQUET_REQUIRE_ENCRYPTION AND NOT ARROW_PARQUET) endif() set(ARROW_OPENSSL_REQUIRED_VERSION "1.0.2") if(BREW_BIN AND NOT OPENSSL_ROOT_DIR) - execute_process(COMMAND ${BREW_BIN} --prefix "openssl" - OUTPUT_VARIABLE OPENSSL_BREW_PREFIX + execute_process(COMMAND ${BREW_BIN} --prefix "openssl@1.1" + OUTPUT_VARIABLE OPENSSL11_BREW_PREFIX OUTPUT_STRIP_TRAILING_WHITESPACE) - if(OPENSSL_BREW_PREFIX) - set(OPENSSL_ROOT_DIR ${OPENSSL_BREW_PREFIX}) + if(OPENSSL11_BREW_PREFIX) + set(OPENSSL_ROOT_DIR ${OPENSSL11_BREW_PREFIX}) + else() + execute_process(COMMAND ${BREW_BIN} --prefix "openssl" + OUTPUT_VARIABLE OPENSSL_BREW_PREFIX + OUTPUT_STRIP_TRAILING_WHITESPACE) + if(OPENSSL_BREW_PREFIX) + set(OPENSSL_ROOT_DIR ${OPENSSL_BREW_PREFIX}) + endif() endif() endif() @@ -1122,10 +1129,15 @@ macro(build_protobuf) "--prefix=${PROTOBUF_PREFIX}" "CFLAGS=${EP_C_FLAGS}" "CXXFLAGS=${EP_CXX_FLAGS}") + set(PROTOBUF_BUILD_COMMAND ${MAKE} ${MAKE_BUILD_ARGS}) + if(CMAKE_OSX_SYSROOT) + list(APPEND PROTOBUF_CONFIGURE_ARGS "SDKROOT=${CMAKE_OSX_SYSROOT}") + list(APPEND PROTOBUF_BUILD_COMMAND "SDKROOT=${CMAKE_OSX_SYSROOT}") + endif() externalproject_add(protobuf_ep CONFIGURE_COMMAND "./configure" ${PROTOBUF_CONFIGURE_ARGS} - BUILD_COMMAND ${MAKE} ${MAKE_BUILD_ARGS} + BUILD_COMMAND ${PROTOBUF_BUILD_COMMAND} BUILD_IN_SOURCE 1 URL ${PROTOBUF_SOURCE_URL} BUILD_BYPRODUCTS "${PROTOBUF_STATIC_LIB}" "${PROTOBUF_COMPILER}" diff --git a/dev/tasks/python-wheels/osx-build.sh b/dev/tasks/python-wheels/osx-build.sh index 82ee9162f99..6da7867779f 100755 --- a/dev/tasks/python-wheels/osx-build.sh +++ b/dev/tasks/python-wheels/osx-build.sh @@ -28,7 +28,7 @@ function build_wheel { # Include brew installed versions of flex and bison. # We need them to build Thrift. The ones that come with Xcode are too old. - export PATH="/usr/local/opt/flex/bin:/usr/local/opt/bison/bin:$PATH" + export PATH="$(brew --prefix flex)/bin:$(brew --prefix bison)/bin:$PATH" echo `pwd` echo CFLAGS=${CFLAGS} @@ -119,37 +119,39 @@ function build_wheel { pushd cpp mkdir build pushd build - cmake -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_INSTALL_PREFIX=$ARROW_HOME \ - -DARROW_VERBOSE_THIRDPARTY_BUILD=ON \ - -DARROW_BUILD_TESTS=OFF \ + cmake -DARROW_BOOST_USE_SHARED=ON \ -DARROW_BUILD_SHARED=ON \ - -DARROW_BOOST_USE_SHARED=ON \ + -DARROW_BUILD_TESTS=OFF \ + -DARROW_DEPENDENCY_SOURCE=BUNDLED \ + -DARROW_FLIGHT=ON \ + -DARROW_GANDIVA=${BUILD_ARROW_GANDIVA} \ -DARROW_JEMALLOC=ON \ + -DARROW_ORC=OFF \ + -DARROW_PARQUET=ON \ -DARROW_PLASMA=ON \ - -DARROW_RPATH_ORIGIN=ON \ + -DARROW_PROTOBUF_USE_SHARED=OFF \ -DARROW_PYTHON=ON \ + -DARROW_RPATH_ORIGIN=ON \ + -DARROW_VERBOSE_THIRDPARTY_BUILD=ON \ + -DARROW_WITH_BROTLI=ON \ -DARROW_WITH_BZ2=ON \ - -DARROW_WITH_ZLIB=ON \ - -DARROW_WITH_ZSTD=ON \ -DARROW_WITH_LZ4=ON \ -DARROW_WITH_SNAPPY=ON \ - -DARROW_WITH_BROTLI=ON \ - -DARROW_PARQUET=ON \ - -DARROW_GANDIVA=${BUILD_ARROW_GANDIVA} \ - -DARROW_ORC=OFF \ + -DARROW_WITH_ZLIB=ON \ + -DARROW_WITH_ZSTD=ON \ + -DBOOST_SOURCE=SYSTEM \ -DBOOST_ROOT="$arrow_boost_dist" \ -DBoost_NAMESPACE=arrow_boost \ - -DARROW_FLIGHT=ON \ - -DgRPC_SOURCE=SYSTEM \ - -Dc-ares_SOURCE=BUNDLED \ - -Dzlib_SOURCE=BUNDLED \ - -DARROW_PROTOBUF_USE_SHARED=OFF \ - -DOPENSSL_USE_STATIC_LIBS=ON \ - -DOPENSSL_ROOT_DIR=$(brew --prefix openssl@1.1) \ + -DBoost_NO_BOOST_CMAKE=ON \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX=$ARROW_HOME \ + -DLLVM_SOURCE=SYSTEM \ -DMAKE=make \ + -DOPENSSL_USE_STATIC_LIBS=ON \ + -DProtobuf_SOURCE=SYSTEM \ + -DgRPC_SOURCE=SYSTEM \ .. - make -j5 + make -j$(sysctl -n hw.logicalcpu) make install popd popd @@ -171,7 +173,10 @@ function build_wheel { export PYARROW_BUNDLE_ARROW_CPP=1 export PYARROW_BUILD_TYPE='release' export PYARROW_BOOST_NAMESPACE='arrow_boost' - export PYARROW_CMAKE_OPTIONS="-DBOOST_ROOT=$arrow_boost_dist" + PYARROW_CMAKE_OPTIONS="" + PYARROW_CMAKE_OPTIONS="${PYARROW_CMAKE_OPTIONS} -DBOOST_ROOT=$arrow_boost_dist" + PYARROW_CMAKE_OPTIONS="${PYARROW_CMAKE_OPTIONS} -DBoost_NO_BOOST_CMAKE=ON" + export PYARROW_CMAKE_OPTIONS export SETUPTOOLS_SCM_PRETEND_VERSION=$PYARROW_VERSION pushd python python setup.py build_ext bdist_wheel diff --git a/dev/tasks/python-wheels/travis.osx.yml b/dev/tasks/python-wheels/travis.osx.yml index cdf39bc58b5..6b161b7fc09 100644 --- a/dev/tasks/python-wheels/travis.osx.yml +++ b/dev/tasks/python-wheels/travis.osx.yml @@ -15,7 +15,7 @@ # limitations under the License. os: osx -osx_image: xcode9.3 +osx_image: xcode11.2 language: generic # don't build twice @@ -38,15 +38,12 @@ before_install: - git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }} - git -C arrow checkout FETCH_HEAD - # ARROW-3976 Old versions of git can cause failures when Homebrew prints a - # donation solicitation. Attempt to update git - - brew upgrade git python + - travis_wait 30 brew bundle --file=arrow/cpp/Brewfile + # Install tools to build bundled Thrift + - brew install + bison + flex - # Also remove artifacts that depend on Boost - - brew uninstall boost cgal postgis sfcgal - - brew update - - brew upgrade cmake - - travis_wait 30 brew install bison flex grpc openssl@1.1 llvm@7 zlib gperftools # remove shared grpc libraries installed by brew to make sure # we are linked against the static ones. - rm -f /usr/local/opt/grpc/lib/*.dylib @@ -63,19 +60,17 @@ install: # the following functions are defined in osx-build.sh - build_wheel arrow - # test the built wheels, remove llvm and grpc dependencies to ensure + # test the built wheels, move Homebrew directory to ensure # things are properly statically-linked - - brew uninstall --ignore-dependencies llvm@7 grpc c-ares openssl openssl@1.1 zlib gperftools + - sudo mv $(brew --cellar){,.bak} # install the built wheel and test dependencies - install_wheel arrow # run unit tests before removing the system libraries - run_unit_tests arrow - # remove libz to ensure that it is properly bundled - - sudo find /usr -name libz.* -delete # run the import tests - run_import_tests - - brew install openssl@1.1 + - sudo mv $(brew --cellar){.bak,} # before_install activates a virtualenv but we need the system python3 - deactivate # crossbow dependencies for deployment