ARROW-7247: [CI][Python] Fix wheel build error on macOS

cpp/Brewfile

@@ -29,7 +29,7 @@ brew "llvm@7"
 brew "lz4"
 brew "ninja"
 brew "numpy"
-brew "openssl"
+brew "openssl@1.1"
 brew "protobuf"
 brew "python"
 brew "rapidjson"

cpp/cmake_modules/ThirdpartyToolchain.cmake

@@ -767,11 +767,18 @@ if(PARQUET_REQUIRE_ENCRYPTION AND NOT ARROW_PARQUET)
 endif()
 set(ARROW_OPENSSL_REQUIRED_VERSION "1.0.2")
 if(BREW_BIN AND NOT OPENSSL_ROOT_DIR)
-  execute_process(COMMAND ${BREW_BIN} --prefix "openssl"
-                  OUTPUT_VARIABLE OPENSSL_BREW_PREFIX
+  execute_process(COMMAND ${BREW_BIN} --prefix "openssl@1.1"
+                  OUTPUT_VARIABLE OPENSSL11_BREW_PREFIX
                   OUTPUT_STRIP_TRAILING_WHITESPACE)
-  if(OPENSSL_BREW_PREFIX)
-    set(OPENSSL_ROOT_DIR ${OPENSSL_BREW_PREFIX})
+  if(OPENSSL11_BREW_PREFIX)
+    set(OPENSSL_ROOT_DIR ${OPENSSL11_BREW_PREFIX})
+  else()
+    execute_process(COMMAND ${BREW_BIN} --prefix "openssl"
+                    OUTPUT_VARIABLE OPENSSL_BREW_PREFIX
+                    OUTPUT_STRIP_TRAILING_WHITESPACE)
+    if(OPENSSL_BREW_PREFIX)
+      set(OPENSSL_ROOT_DIR ${OPENSSL_BREW_PREFIX})
+    endif()
   endif()
 endif()
 
@@ -1122,10 +1129,15 @@ macro(build_protobuf)
       "--prefix=${PROTOBUF_PREFIX}"
       "CFLAGS=${EP_C_FLAGS}"
       "CXXFLAGS=${EP_CXX_FLAGS}")
+  set(PROTOBUF_BUILD_COMMAND ${MAKE} ${MAKE_BUILD_ARGS})
+  if(CMAKE_OSX_SYSROOT)
+    list(APPEND PROTOBUF_CONFIGURE_ARGS "SDKROOT=${CMAKE_OSX_SYSROOT}")
+    list(APPEND PROTOBUF_BUILD_COMMAND "SDKROOT=${CMAKE_OSX_SYSROOT}")
+  endif()
 
   externalproject_add(protobuf_ep
                       CONFIGURE_COMMAND "./configure" ${PROTOBUF_CONFIGURE_ARGS}
-                      BUILD_COMMAND ${MAKE} ${MAKE_BUILD_ARGS}
+                      BUILD_COMMAND ${PROTOBUF_BUILD_COMMAND}
                       BUILD_IN_SOURCE 1
                       URL ${PROTOBUF_SOURCE_URL}
                       BUILD_BYPRODUCTS "${PROTOBUF_STATIC_LIB}" "${PROTOBUF_COMPILER}"

dev/tasks/python-wheels/osx-build.sh

@@ -28,7 +28,7 @@ function build_wheel {
 
     # Include brew installed versions of flex and bison.
     # We need them to build Thrift. The ones that come with Xcode are too old.
-    export PATH="/usr/local/opt/flex/bin:/usr/local/opt/bison/bin:$PATH"
+    export PATH="$(brew --prefix flex)/bin:$(brew --prefix bison)/bin:$PATH"
 
     echo `pwd`
     echo CFLAGS=${CFLAGS}
@@ -119,37 +119,39 @@ function build_wheel {
     pushd cpp
     mkdir build
     pushd build
-    cmake -DCMAKE_BUILD_TYPE=Release \
-          -DCMAKE_INSTALL_PREFIX=$ARROW_HOME \
-          -DARROW_VERBOSE_THIRDPARTY_BUILD=ON \
-          -DARROW_BUILD_TESTS=OFF \
+    cmake -DARROW_BOOST_USE_SHARED=ON \
           -DARROW_BUILD_SHARED=ON \
-          -DARROW_BOOST_USE_SHARED=ON \
+          -DARROW_BUILD_TESTS=OFF \
+          -DARROW_DEPENDENCY_SOURCE=BUNDLED \
+          -DARROW_FLIGHT=ON \
+          -DARROW_GANDIVA=${BUILD_ARROW_GANDIVA} \
           -DARROW_JEMALLOC=ON \
+          -DARROW_ORC=OFF \
+          -DARROW_PARQUET=ON \
           -DARROW_PLASMA=ON \
-          -DARROW_RPATH_ORIGIN=ON \
+          -DARROW_PROTOBUF_USE_SHARED=OFF \
           -DARROW_PYTHON=ON \
+          -DARROW_RPATH_ORIGIN=ON \
+          -DARROW_VERBOSE_THIRDPARTY_BUILD=ON \
+          -DARROW_WITH_BROTLI=ON \
           -DARROW_WITH_BZ2=ON \
-          -DARROW_WITH_ZLIB=ON \
-          -DARROW_WITH_ZSTD=ON \
           -DARROW_WITH_LZ4=ON \
           -DARROW_WITH_SNAPPY=ON \
-          -DARROW_WITH_BROTLI=ON \
-          -DARROW_PARQUET=ON \
-          -DARROW_GANDIVA=${BUILD_ARROW_GANDIVA} \
-          -DARROW_ORC=OFF \
+          -DARROW_WITH_ZLIB=ON \
+          -DARROW_WITH_ZSTD=ON \
+          -DBOOST_SOURCE=SYSTEM \
           -DBOOST_ROOT="$arrow_boost_dist" \
           -DBoost_NAMESPACE=arrow_boost \
-          -DARROW_FLIGHT=ON \
-          -DgRPC_SOURCE=SYSTEM \
-          -Dc-ares_SOURCE=BUNDLED \
-          -Dzlib_SOURCE=BUNDLED \
-          -DARROW_PROTOBUF_USE_SHARED=OFF \
-          -DOPENSSL_USE_STATIC_LIBS=ON  \
-          -DOPENSSL_ROOT_DIR=$(brew --prefix openssl@1.1) \
+          -DBoost_NO_BOOST_CMAKE=ON \
+          -DCMAKE_BUILD_TYPE=Release \
+          -DCMAKE_INSTALL_PREFIX=$ARROW_HOME \
+          -DLLVM_SOURCE=SYSTEM \
           -DMAKE=make \
+          -DOPENSSL_USE_STATIC_LIBS=ON \
+          -DProtobuf_SOURCE=SYSTEM \
+          -DgRPC_SOURCE=SYSTEM \
           ..
-    make -j5
+    make -j$(sysctl -n hw.logicalcpu)
     make install
     popd
     popd
@@ -171,7 +173,10 @@ function build_wheel {
     export PYARROW_BUNDLE_ARROW_CPP=1
     export PYARROW_BUILD_TYPE='release'
     export PYARROW_BOOST_NAMESPACE='arrow_boost'
-    export PYARROW_CMAKE_OPTIONS="-DBOOST_ROOT=$arrow_boost_dist"
+    PYARROW_CMAKE_OPTIONS=""
+    PYARROW_CMAKE_OPTIONS="${PYARROW_CMAKE_OPTIONS} -DBOOST_ROOT=$arrow_boost_dist"
+    PYARROW_CMAKE_OPTIONS="${PYARROW_CMAKE_OPTIONS} -DBoost_NO_BOOST_CMAKE=ON"
+    export PYARROW_CMAKE_OPTIONS
     export SETUPTOOLS_SCM_PRETEND_VERSION=$PYARROW_VERSION
     pushd python
     python setup.py build_ext bdist_wheel

dev/tasks/python-wheels/travis.osx.yml

@@ -15,7 +15,7 @@
 # limitations under the License.
 
 os: osx
-osx_image: xcode9.3
+osx_image: xcode11.2
 language: generic
 
 # don't build twice
@@ -38,15 +38,12 @@ before_install:
   - git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }}
   - git -C arrow checkout FETCH_HEAD
 
-  # ARROW-3976 Old versions of git can cause failures when Homebrew prints a
-  # donation solicitation. Attempt to update git
-  - brew upgrade git python
+  - travis_wait 30 brew bundle --file=arrow/cpp/Brewfile
+  # Install tools to build bundled Thrift
+  - brew install
+      bison
+      flex
 
-  # Also remove artifacts that depend on Boost
-  - brew uninstall boost cgal postgis sfcgal
-  - brew update
-  - brew upgrade cmake
-  - travis_wait 30 brew install bison flex grpc openssl@1.1 llvm@7 zlib gperftools
   # remove shared grpc libraries installed by brew to make sure
   # we are linked against the static ones.
   - rm -f /usr/local/opt/grpc/lib/*.dylib
@@ -63,19 +60,17 @@ install:
   # the following functions are defined in osx-build.sh
   - build_wheel arrow
 
-  # test the built wheels, remove llvm and grpc dependencies to ensure
+  # test the built wheels, move Homebrew directory to ensure
   # things are properly statically-linked
-  - brew uninstall --ignore-dependencies llvm@7 grpc c-ares openssl openssl@1.1 zlib gperftools
+  - sudo mv $(brew --cellar){,.bak}
   # install the built wheel and test dependencies
   - install_wheel arrow
   # run unit tests before removing the system libraries
   - run_unit_tests arrow
-  # remove libz to ensure that it is properly bundled
-  - sudo find /usr -name libz.* -delete
   # run the import tests
   - run_import_tests
 
-  - brew install openssl@1.1
+  - sudo mv $(brew --cellar){.bak,}
   # before_install activates a virtualenv but we need the system python3
   - deactivate
   # crossbow dependencies for deployment