From 157d3c1e8bc6317fefcf4fd04f99c1c2d8aaf97c Mon Sep 17 00:00:00 2001 From: Renat Valiullin Date: Sun, 10 Feb 2019 15:55:04 -0800 Subject: [PATCH 1/8] Improve FindBrotli.cmake --- cpp/cmake_modules/FindBrotli.cmake | 81 +++++++++++------------------- 1 file changed, 29 insertions(+), 52 deletions(-) diff --git a/cpp/cmake_modules/FindBrotli.cmake b/cpp/cmake_modules/FindBrotli.cmake index ea971f09ed8..c10e663288b 100644 --- a/cpp/cmake_modules/FindBrotli.cmake +++ b/cpp/cmake_modules/FindBrotli.cmake @@ -26,8 +26,7 @@ # # This module defines # BROTLI_INCLUDE_DIR, directory containing headers -# BROTLI_LIBS, directory containing brotli libraries -# BROTLI_STATIC_LIB, path to libbrotli.a +# BROTLI_STATIC_LIB, path to libbrotli's static library # BROTLI_SHARED_LIB, path to libbrotli's shared library # BROTLI_FOUND, whether brotli has been found @@ -43,66 +42,46 @@ find_path( BROTLI_INCLUDE_DIR NAMES brotli/decode.h NO_DEFAULT_PATH PATH_SUFFIXES "include" ) -find_library( BROTLI_LIBRARY_ENC NAMES libbrotlienc.a libbrotlienc-static.a brotlienc - PATHS ${_brotli_roots} - NO_DEFAULT_PATH - PATH_SUFFIXES "lib/${CMAKE_LIBRARY_ARCHITECTURE}" "lib" ) +set(BROTLI_LIB_NAME brotli) -find_library( BROTLI_LIBRARY_DEC NAMES libbrotlidec.a libbrotlidec-static.a brotlidec - PATHS ${_brotli_roots} - NO_DEFAULT_PATH - PATH_SUFFIXES "lib/${CMAKE_LIBRARY_ARCHITECTURE}" "lib" ) +if (DEFINED BROTLI_MSVC_STATIC_LIB_SUFFIX) + set(BROTLI_STATIC_LIB_SUFFIX "${BROTLI_MSVC_STATIC_LIB_SUFFIX}") +endif() +if (NOT DEFINED BROTLI_STATIC_LIB_SUFFIX) + if (MSVC) + set(BROTLI_STATIC_LIB_SUFFIX _static) + else() + set(BROTLI_STATIC_LIB_SUFFIX "") + endif() +endif() -find_library( BROTLI_LIBRARY_COMMON NAMES libbrotlicommon.a libbrotlicommon-static.a brotlicommon - PATHS ${_brotli_roots} - NO_DEFAULT_PATH - PATH_SUFFIXES "lib/${CMAKE_LIBRARY_ARCHITECTURE}" "lib" ) +foreach (BROTLI_LIB enc dec common) + string(TOUPPER "LIBRARY_${BROTLI_LIB}" LIBRARY) -set(BROTLI_LIBRARIES ${BROTLI_LIBRARY_ENC} ${BROTLI_LIBRARY_DEC} - ${BROTLI_LIBRARY_COMMON}) + find_library(BROTLI_SHARED_${LIBRARY} + NAMES ${CMAKE_SHARED_LIBRARY_PREFIX}${BROTLI_LIB_NAME}${BROTLI_LIB}${BROTLI_SHARED_LIB_SUFFIX}${CMAKE_SHARED_LIBRARY_SUFFIX} + PATHS ${_brotli_roots} NO_DEFAULT_PATH + PATH_SUFFIXES "lib/${CMAKE_LIBRARY_ARCHITECTURE}" "lib") + list(APPEND BROTLI_SHARED_LIB ${BROTLI_SHARED_${LIBRARY}}) -if (BROTLI_INCLUDE_DIR AND (PARQUET_MINIMAL_DEPENDENCY OR BROTLI_LIBRARIES)) - set(BROTLI_FOUND TRUE) - get_filename_component( BROTLI_LIBS ${BROTLI_LIBRARY_ENC} PATH ) - set(BROTLI_LIB_NAME brotli) - if (BROTLI_MSVC_STATIC_LIB_SUFFIX) - set(BROTLI_STATIC_LIB_SUFFIX "${BROTLI_MSVC_STATIC_LIB_SUFFIX}") - endif() - if (NOT BROTLI_STATIC_LIB_SUFFIX) - if (EXISTS "${BROTLI_LIBS}/${CMAKE_STATIC_LIBRARY_PREFIX}${BROTLI_LIB_NAME}enc-static${CMAKE_STATIC_LIBRARY_SUFFIX}") - set(BROTLI_STATIC_LIB_SUFFIX -static) - else() - if (MSVC) - set(BROTLI_STATIC_LIB_SUFFIX _static) - else() - set(BROTLI_STATIC_LIB_SUFFIX "") - endif() - endif() - endif() - set(BROTLI_STATIC_LIB - ${BROTLI_LIBS}/${CMAKE_STATIC_LIBRARY_PREFIX}${BROTLI_LIB_NAME}enc${BROTLI_STATIC_LIB_SUFFIX}${CMAKE_STATIC_LIBRARY_SUFFIX} - ${BROTLI_LIBS}/${CMAKE_STATIC_LIBRARY_PREFIX}${BROTLI_LIB_NAME}dec${BROTLI_STATIC_LIB_SUFFIX}${CMAKE_STATIC_LIBRARY_SUFFIX} - ${BROTLI_LIBS}/${CMAKE_STATIC_LIBRARY_PREFIX}${BROTLI_LIB_NAME}common${BROTLI_STATIC_LIB_SUFFIX}${CMAKE_STATIC_LIBRARY_SUFFIX}) - set(BROTLI_STATIC_LIBRARY_ENC ${BROTLI_LIBS}/${CMAKE_STATIC_LIBRARY_PREFIX}${BROTLI_LIB_NAME}enc${BROTLI_STATIC_LIB_SUFFIX}${CMAKE_STATIC_LIBRARY_SUFFIX}) - set(BROTLI_STATIC_LIBRARY_DEC ${BROTLI_LIBS}/${CMAKE_STATIC_LIBRARY_PREFIX}${BROTLI_LIB_NAME}dec${BROTLI_STATIC_LIB_SUFFIX}${CMAKE_STATIC_LIBRARY_SUFFIX}) - set(BROTLI_STATIC_LIBRARY_COMMON ${BROTLI_LIBS}/${CMAKE_STATIC_LIBRARY_PREFIX}${BROTLI_LIB_NAME}common${BROTLI_STATIC_LIB_SUFFIX}${CMAKE_STATIC_LIBRARY_SUFFIX}) - set(BROTLI_SHARED_LIB - ${BROTLI_LIBS}/${CMAKE_SHARED_LIBRARY_PREFIX}${BROTLI_LIB_NAME}enc${CMAKE_SHARED_LIBRARY_SUFFIX} - ${BROTLI_LIBS}/${CMAKE_SHARED_LIBRARY_PREFIX}${BROTLI_LIB_NAME}dec${CMAKE_SHARED_LIBRARY_SUFFIX} - ${BROTLI_LIBS}/${CMAKE_SHARED_LIBRARY_PREFIX}${BROTLI_LIB_NAME}common${CMAKE_SHARED_LIBRARY_SUFFIX}) -else () - set(BROTLI_FOUND FALSE) -endif () + find_library(BROTLI_STATIC_${LIBRARY} + NAMES ${CMAKE_STATIC_LIBRARY_PREFIX}${BROTLI_LIB_NAME}${BROTLI_LIB}${BROTLI_STATIC_LIB_SUFFIX}${CMAKE_STATIC_LIBRARY_SUFFIX} + PATHS ${_brotli_roots} NO_DEFAULT_PATH + PATH_SUFFIXES "lib/${CMAKE_LIBRARY_ARCHITECTURE}" "lib") + list(APPEND BROTLI_STATIC_LIB ${BROTLI_STATIC_${LIBRARY}}) +endforeach() -if (BROTLI_FOUND) +if (BROTLI_INCLUDE_DIR AND (PARQUET_MINIMAL_DEPENDENCY OR BROTLI_STATIC_LIB OR BROTLI_SHARED_LIB)) + set(BROTLI_FOUND TRUE) if (NOT Brotli_FIND_QUIETLY) if (PARQUET_MINIMAL_DEPENDENCY) message(STATUS "Found the Brotli headers: ${BROTLI_INCLUDE_DIR}") else () - message(STATUS "Found the Brotli library: ${BROTLI_LIBRARIES}") + message(STATUS "Found the Brotli library: ${BROTLI_STATIC_LIB} ${BROTLI_SHARED_LIB}") endif () endif () else () + set(BROTLI_FOUND FALSE) if (NOT Brotli_FIND_QUIETLY) set(BROTLI_ERR_MSG "Could not find the Brotli library. Looked in ") if ( _brotli_roots ) @@ -120,8 +99,6 @@ endif () mark_as_advanced( BROTLI_INCLUDE_DIR - BROTLI_LIBS - BROTLI_LIBRARIES BROTLI_STATIC_LIB BROTLI_SHARED_LIB ) From 67aedac2bff254030844d32a575715e16829b1b2 Mon Sep 17 00:00:00 2001 From: Renat Valiullin Date: Sun, 10 Feb 2019 16:29:53 -0800 Subject: [PATCH 2/8] fix build when we have only shared brotli's libs --- cpp/cmake_modules/FindBrotli.cmake | 7 ++++++- cpp/cmake_modules/ThirdpartyToolchain.cmake | 23 ++++++++++++++------- 2 files changed, 22 insertions(+), 8 deletions(-) diff --git a/cpp/cmake_modules/FindBrotli.cmake b/cpp/cmake_modules/FindBrotli.cmake index c10e663288b..9b95f3b7bf6 100644 --- a/cpp/cmake_modules/FindBrotli.cmake +++ b/cpp/cmake_modules/FindBrotli.cmake @@ -77,7 +77,12 @@ if (BROTLI_INCLUDE_DIR AND (PARQUET_MINIMAL_DEPENDENCY OR BROTLI_STATIC_LIB OR B if (PARQUET_MINIMAL_DEPENDENCY) message(STATUS "Found the Brotli headers: ${BROTLI_INCLUDE_DIR}") else () - message(STATUS "Found the Brotli library: ${BROTLI_STATIC_LIB} ${BROTLI_SHARED_LIB}") + if (BROTLI_STATIC_LIB) + message(STATUS "Found the Brotli static library: ${BROTLI_STATIC_LIB}") + endif() + if (BROTLI_SHARED_LIB) + message(STATUS "Found the Brotli shared library: ${BROTLI_SHARED_LIB}") + endif() endif () endif () else () diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index 5cfc95db352..396aa1a3aa6 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -1063,7 +1063,7 @@ if (ARROW_WITH_BROTLI) # ---------------------------------------------------------------------- # Brotli - if("${BROTLI_HOME}" STREQUAL "") + if ("${BROTLI_HOME}" STREQUAL "") set(BROTLI_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/brotli_ep/src/brotli_ep-install") set(BROTLI_HOME "${BROTLI_PREFIX}") set(BROTLI_INCLUDE_DIR "${BROTLI_PREFIX}/include") @@ -1104,12 +1104,21 @@ if (ARROW_WITH_BROTLI) endif() include_directories(SYSTEM ${BROTLI_INCLUDE_DIR}) - ADD_THIRDPARTY_LIB(brotli_enc - STATIC_LIB ${BROTLI_STATIC_LIBRARY_ENC}) - ADD_THIRDPARTY_LIB(brotli_dec - STATIC_LIB ${BROTLI_STATIC_LIBRARY_DEC}) - ADD_THIRDPARTY_LIB(brotli_common - STATIC_LIB ${BROTLI_STATIC_LIBRARY_COMMON}) + if (BROTLI_STATIC_LIB OR BROTLI_VENDORED) + ADD_THIRDPARTY_LIB(brotli_enc + STATIC_LIB ${BROTLI_STATIC_LIBRARY_ENC} + ADD_THIRDPARTY_LIB(brotli_dec + STATIC_LIB ${BROTLI_STATIC_LIBRARY_DEC} + ADD_THIRDPARTY_LIB(brotli_common + STATIC_LIB ${BROTLI_STATIC_LIBRARY_COMMON} + else() + ADD_THIRDPARTY_LIB(brotli_enc + SHARED_LIB ${BROTLI_SHARED_LIBRARY_ENC}) + ADD_THIRDPARTY_LIB(brotli_dec + SHARED_LIB ${BROTLI_SHARED_LIBRARY_DEC}) + ADD_THIRDPARTY_LIB(brotli_common + SHARED_LIB ${BROTLI_SHARED_LIBRARY_COMMON}) + endif() if (BROTLI_VENDORED) add_dependencies(brotli_enc_static brotli_ep) From f021908ddda4bd1e4035b5c11256e2d3d73bd25e Mon Sep 17 00:00:00 2001 From: Renat Valiullin Date: Sun, 10 Feb 2019 16:39:14 -0800 Subject: [PATCH 3/8] fix of build fix.. --- cpp/cmake_modules/ThirdpartyToolchain.cmake | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index 396aa1a3aa6..90b38738041 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -1106,11 +1106,11 @@ if (ARROW_WITH_BROTLI) include_directories(SYSTEM ${BROTLI_INCLUDE_DIR}) if (BROTLI_STATIC_LIB OR BROTLI_VENDORED) ADD_THIRDPARTY_LIB(brotli_enc - STATIC_LIB ${BROTLI_STATIC_LIBRARY_ENC} + STATIC_LIB ${BROTLI_STATIC_LIBRARY_ENC}) ADD_THIRDPARTY_LIB(brotli_dec - STATIC_LIB ${BROTLI_STATIC_LIBRARY_DEC} + STATIC_LIB ${BROTLI_STATIC_LIBRARY_DEC}) ADD_THIRDPARTY_LIB(brotli_common - STATIC_LIB ${BROTLI_STATIC_LIBRARY_COMMON} + STATIC_LIB ${BROTLI_STATIC_LIBRARY_COMMON}) else() ADD_THIRDPARTY_LIB(brotli_enc SHARED_LIB ${BROTLI_SHARED_LIBRARY_ENC}) From 404e1f8144554bec9e262beb83072cdece3b799f Mon Sep 17 00:00:00 2001 From: Renat Valiullin Date: Sun, 10 Feb 2019 16:53:57 -0800 Subject: [PATCH 4/8] fix default BROTLI_STATIC_LIB_SUFFIX --- cpp/cmake_modules/FindBrotli.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/cmake_modules/FindBrotli.cmake b/cpp/cmake_modules/FindBrotli.cmake index 9b95f3b7bf6..506c98b2787 100644 --- a/cpp/cmake_modules/FindBrotli.cmake +++ b/cpp/cmake_modules/FindBrotli.cmake @@ -51,7 +51,7 @@ if (NOT DEFINED BROTLI_STATIC_LIB_SUFFIX) if (MSVC) set(BROTLI_STATIC_LIB_SUFFIX _static) else() - set(BROTLI_STATIC_LIB_SUFFIX "") + set(BROTLI_STATIC_LIB_SUFFIX -static) endif() endif() From a151401b67079a6da401cd2c8d2ebc1d04fa4f47 Mon Sep 17 00:00:00 2001 From: Renat Valiullin Date: Sun, 10 Feb 2019 17:24:21 -0800 Subject: [PATCH 5/8] try to fix manylinux1 --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index 254bfa06ebf..dc8c380c94d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -224,6 +224,8 @@ matrix: - $TRAVIS_BUILD_DIR/ci/travis_script_python.sh 3.6 - name: "[manylinux1] Python" language: cpp + env: + - BROTLI_STATIC_LIB_SUFFIX="" before_script: - if [ $ARROW_CI_PYTHON_AFFECTED == "1" ]; then docker pull quay.io/xhochy/arrow_manylinux1_x86_64_base:llvm-7-manylinux1; fi script: From 03d75d70af380915f73761781236c8468b83ef09 Mon Sep 17 00:00:00 2001 From: Renat Valiullin Date: Sun, 10 Feb 2019 17:42:38 -0800 Subject: [PATCH 6/8] one more attempt to fix manylinux1 --- .travis.yml | 2 -- python/manylinux1/build_arrow.sh | 1 + 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index dc8c380c94d..254bfa06ebf 100644 --- a/.travis.yml +++ b/.travis.yml @@ -224,8 +224,6 @@ matrix: - $TRAVIS_BUILD_DIR/ci/travis_script_python.sh 3.6 - name: "[manylinux1] Python" language: cpp - env: - - BROTLI_STATIC_LIB_SUFFIX="" before_script: - if [ $ARROW_CI_PYTHON_AFFECTED == "1" ]; then docker pull quay.io/xhochy/arrow_manylinux1_x86_64_base:llvm-7-manylinux1; fi script: diff --git a/python/manylinux1/build_arrow.sh b/python/manylinux1/build_arrow.sh index 75537a64e73..54485f8265e 100755 --- a/python/manylinux1/build_arrow.sh +++ b/python/manylinux1/build_arrow.sh @@ -79,6 +79,7 @@ for PYTHON_TUPLE in ${PYTHON_VERSIONS}; do -DARROW_BOOST_USE_SHARED=ON \ -DARROW_GANDIVA_PC_CXX_FLAGS="-isystem;/opt/rh/devtoolset-2/root/usr/include/c++/4.8.2;-isystem;/opt/rh/devtoolset-2/root/usr/include/c++/4.8.2/x86_64-CentOS-linux/" \ -DARROW_JEMALLOC=ON \ + -BROTLI_STATIC_LIB_SUFFIX= \ -DARROW_RPATH_ORIGIN=ON \ -DARROW_PYTHON=ON \ -DARROW_PARQUET=ON \ From 9194ca42bea02784672bc619b7377dec630c350e Mon Sep 17 00:00:00 2001 From: Renat Valiullin Date: Mon, 11 Feb 2019 13:25:33 -0800 Subject: [PATCH 7/8] add lib64 to BROTLI_PATH_SUFFIXES --- cpp/cmake_modules/FindBrotli.cmake | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/cpp/cmake_modules/FindBrotli.cmake b/cpp/cmake_modules/FindBrotli.cmake index 506c98b2787..ff6fcbacbff 100644 --- a/cpp/cmake_modules/FindBrotli.cmake +++ b/cpp/cmake_modules/FindBrotli.cmake @@ -55,19 +55,21 @@ if (NOT DEFINED BROTLI_STATIC_LIB_SUFFIX) endif() endif() +list(APPEND BROTLI_PATH_SUFFIXES "lib/${CMAKE_LIBRARY_ARCHITECTURE}" "lib" "lib64") + foreach (BROTLI_LIB enc dec common) string(TOUPPER "LIBRARY_${BROTLI_LIB}" LIBRARY) find_library(BROTLI_SHARED_${LIBRARY} NAMES ${CMAKE_SHARED_LIBRARY_PREFIX}${BROTLI_LIB_NAME}${BROTLI_LIB}${BROTLI_SHARED_LIB_SUFFIX}${CMAKE_SHARED_LIBRARY_SUFFIX} PATHS ${_brotli_roots} NO_DEFAULT_PATH - PATH_SUFFIXES "lib/${CMAKE_LIBRARY_ARCHITECTURE}" "lib") + PATH_SUFFIXES ${BROTLI_PATH_SUFFIXES}) list(APPEND BROTLI_SHARED_LIB ${BROTLI_SHARED_${LIBRARY}}) find_library(BROTLI_STATIC_${LIBRARY} NAMES ${CMAKE_STATIC_LIBRARY_PREFIX}${BROTLI_LIB_NAME}${BROTLI_LIB}${BROTLI_STATIC_LIB_SUFFIX}${CMAKE_STATIC_LIBRARY_SUFFIX} PATHS ${_brotli_roots} NO_DEFAULT_PATH - PATH_SUFFIXES "lib/${CMAKE_LIBRARY_ARCHITECTURE}" "lib") + PATH_SUFFIXES ${BROTLI_PATH_SUFFIXES}) list(APPEND BROTLI_STATIC_LIB ${BROTLI_STATIC_${LIBRARY}}) endforeach() From ea519085ead840ec23a87e171d2cd0a211bb656c Mon Sep 17 00:00:00 2001 From: Renat Valiullin Date: Tue, 12 Feb 2019 13:52:35 -0800 Subject: [PATCH 8/8] fix BROTLI_STATIC_LIB_SUFFIX definition --- cpp/cmake_modules/FindBrotli.cmake | 2 +- python/manylinux1/build_arrow.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cpp/cmake_modules/FindBrotli.cmake b/cpp/cmake_modules/FindBrotli.cmake index ff6fcbacbff..17129474584 100644 --- a/cpp/cmake_modules/FindBrotli.cmake +++ b/cpp/cmake_modules/FindBrotli.cmake @@ -30,7 +30,7 @@ # BROTLI_SHARED_LIB, path to libbrotli's shared library # BROTLI_FOUND, whether brotli has been found -if( NOT "${BROTLI_HOME}" STREQUAL "") +if (NOT "${BROTLI_HOME}" STREQUAL "") file( TO_CMAKE_PATH "${BROTLI_HOME}" _native_path ) list( APPEND _brotli_roots ${_native_path} ) elseif ( Brotli_HOME ) diff --git a/python/manylinux1/build_arrow.sh b/python/manylinux1/build_arrow.sh index 54485f8265e..3de75a757d6 100755 --- a/python/manylinux1/build_arrow.sh +++ b/python/manylinux1/build_arrow.sh @@ -79,7 +79,7 @@ for PYTHON_TUPLE in ${PYTHON_VERSIONS}; do -DARROW_BOOST_USE_SHARED=ON \ -DARROW_GANDIVA_PC_CXX_FLAGS="-isystem;/opt/rh/devtoolset-2/root/usr/include/c++/4.8.2;-isystem;/opt/rh/devtoolset-2/root/usr/include/c++/4.8.2/x86_64-CentOS-linux/" \ -DARROW_JEMALLOC=ON \ - -BROTLI_STATIC_LIB_SUFFIX= \ + -DBROTLI_STATIC_LIB_SUFFIX= \ -DARROW_RPATH_ORIGIN=ON \ -DARROW_PYTHON=ON \ -DARROW_PARQUET=ON \