From 64cfa34aa6c840f44327b097655a0b6f6be59d61 Mon Sep 17 00:00:00 2001 From: Wes McKinney Date: Thu, 16 Jan 2020 18:46:34 -0600 Subject: [PATCH 01/16] Work on creating libarrow_dependencies.a build libarrow_bundled_dependencies.a --- LICENSE.txt | 28 +++++++ cpp/CMakeLists.txt | 4 - cpp/cmake_modules/BuildUtils.cmake | 90 +++++++++++++++++++++ cpp/cmake_modules/ThirdpartyToolchain.cmake | 46 +++++++++++ cpp/src/arrow/CMakeLists.txt | 12 +++ 5 files changed, 176 insertions(+), 4 deletions(-) diff --git a/LICENSE.txt b/LICENSE.txt index 13faeb2ee8e..673b146c807 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -2179,3 +2179,31 @@ IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +-------------------------------------------------------------------------------- + +The file cpp/cmake_modules/BuildUtils.cmake contains code from + +https://gist.github.com/cristianadam/ef920342939a89fae3e8a85ca9459b49 + +which is made available under the MIT license + +Copyright (c) 2019 Cristian Adam + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 32a149492a0..da916ba3a68 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -642,10 +642,6 @@ endif() # # TODO: Also rework how these libs work -set(ARROW_LINK_LIBS) -set(ARROW_SHARED_INSTALL_INTERFACE_LIBS) -set(ARROW_STATIC_INSTALL_INTERFACE_LIBS) - # Libraries to link statically with libarrow.so set(ARROW_LINK_LIBS) set(ARROW_STATIC_LINK_LIBS) diff --git a/cpp/cmake_modules/BuildUtils.cmake b/cpp/cmake_modules/BuildUtils.cmake index 7adc5624bd4..adc3a393461 100644 --- a/cpp/cmake_modules/BuildUtils.cmake +++ b/cpp/cmake_modules/BuildUtils.cmake @@ -128,6 +128,87 @@ function(REUSE_PRECOMPILED_HEADER_LIB TARGET_NAME LIB_NAME) endif() endfunction() +# Based on MIT-licensed +# https://gist.github.com/cristianadam/ef920342939a89fae3e8a85ca9459b49 +function(create_merged_static_lib output_target) + set(options) + set(one_value_args NAME ROOT) + set(multi_value_args TO_MERGE) + cmake_parse_arguments(ARG + "${options}" + "${one_value_args}" + "${multi_value_args}" + ${ARGN}) + if(ARG_UNPARSED_ARGUMENTS) + message(SEND_ERROR "Error: unrecognized arguments: ${ARG_UNPARSED_ARGUMENTS}") + endif() + + set( + output_lib_path + ${BUILD_OUTPUT_ROOT_DIRECTORY}${CMAKE_STATIC_LIBRARY_PREFIX}${ARG_NAME}${CMAKE_STATIC_LIBRARY_SUFFIX} + ) + + set(all_library_paths $) + foreach(lib ${ARG_TO_MERGE}) + list(APPEND all_library_paths $) + endforeach() + + if(APPLE) + set(BUNDLE_COMMAND + "libtool" + "-no_warning_for_no_symbols" + "-static" + "-o" + ${output_lib_path} + ${all_library_paths}) + elseif(CMAKE_CXX_COMPILER_ID MATCHES "^(Clang|GNU)$") + set(ar_script_path ${CMAKE_BINARY_DIR}/${ARG_NAME}.ar) + + file(WRITE ${ar_script_path}.in "CREATE ${output_lib_path}\n") + file(APPEND ${ar_script_path}.in "ADDLIB $\n") + + foreach(lib ${ARG_TO_MERGE}) + file(APPEND ${ar_script_path}.in "ADDLIB $\n") + endforeach() + + file(APPEND ${ar_script_path}.in "SAVE\nEND\n") + file(GENERATE OUTPUT ${ar_script_path} INPUT ${ar_script_path}.in) + set(ar_tool ${CMAKE_AR}) + + if(CMAKE_INTERPROCEDURAL_OPTIMIZATION) + set(ar_tool ${CMAKE_CXX_COMPILER_AR}) + endif() + + set(BUNDLE_COMMAND ${ar_tool} -M < ${ar_script_path}) + + elseif(MSVC) + if(NOT CMAKE_LIBTOOL) + find_program(lib_tool lib HINTS "${CMAKE_CXX_COMPILER}/..") + if("${lib_tool}" STREQUAL "lib_tool-NOTFOUND") + message(FATAL_ERROR "Cannot locate libtool to bundle libraries") + endif() + else() + set(${lib_tool} ${CMAKE_LIBTOOL}) + endif() + set(BUNDLE_TOOL ${lib_tool}) + set(BUNDLE_COMMAND ${BUNDLE_TOOL} /NOLOGO /OUT:${output_lib_path} + ${all_library_paths}) + else() + message(FATAL_ERROR "Unknown bundle scenario!") + endif() + + add_custom_command(COMMAND ${BUNDLE_COMMAND} + OUTPUT ${output_lib_path} + COMMENT "Bundling ${output_lib_path}" + VERBATIM) + + message(STATUS "Creating bundled static library target ${output_target} at ${output_lib_path}") + + add_custom_target(${output_target} ALL DEPENDS ${output_lib_path}) + add_dependencies(${output_target} ${ARG_ROOT} ${ARG_TO_MERGE}) + install(FILES ${output_lib_path} DESTINATION ${CMAKE_INSTALL_LIBDIR}) +endfunction() + # \arg OUTPUTS list to append built targets to function(ADD_ARROW_LIB LIB_NAME) set(options BUILD_SHARED BUILD_STATIC) @@ -832,3 +913,12 @@ function(ARROW_INSTALL_CMAKE_FIND_MODULE MODULE) install(FILES "${ARROW_SOURCE_DIR}/cmake_modules/Find${MODULE}.cmake" DESTINATION "${ARROW_CMAKE_INSTALL_DIR}") endfunction() + +# Implementations of lisp "car" and "cdr" functions +macro(ARROW_CAR var) + SET(${var} ${ARGV1}) +endmacro() + +MACRO(ARROW_CDR var rest) + SET(${var} ${ARGN}) +ENDMACRO() diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index 960d31359c2..20d88f819c1 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -23,6 +23,12 @@ add_custom_target(toolchain) add_custom_target(toolchain-benchmarks) add_custom_target(toolchain-tests) +# Accumulate all bundled targets and we will splice them together later as +# libarrow_dependencies.a so that third party libraries have something usable +# to create statically-linked builds with some BUNDLED dependencies, including +# allocators like jemalloc and mimalloc +set(ARROW_BUNDLED_STATIC_LIBS) + # ---------------------------------------------------------------------- # Toolchain linkage options @@ -685,6 +691,8 @@ macro(build_boost) set(Boost_INCLUDE_DIRS "${BOOST_INCLUDE_DIR}") add_dependencies(toolchain boost_ep) set(BOOST_VENDORED TRUE) + + list(APPEND ARROW_BUNDLED_STATIC_LIBS boost_system_static boost_filesystem_static boost_regex_static) endmacro() if(ARROW_FLIGHT AND ARROW_BUILD_TESTS) @@ -833,6 +841,8 @@ macro(build_snappy) "${SNAPPY_PREFIX}/include") add_dependencies(toolchain snappy_ep) add_dependencies(Snappy::snappy snappy_ep) + + list(APPEND ARROW_BUNDLED_STATIC_LIBS Snappy::snappy) endmacro() if(ARROW_WITH_SNAPPY) @@ -919,6 +929,8 @@ macro(build_brotli) PROPERTIES IMPORTED_LOCATION "${BROTLI_STATIC_LIBRARY_DEC}" INTERFACE_INCLUDE_DIRECTORIES "${BROTLI_INCLUDE_DIR}") add_dependencies(Brotli::brotlidec brotli_ep) + + list(APPEND ARROW_BUNDLED_STATIC_LIBS Brotli::brotlicommon Brotli::brotlienc Brotli::brotlidec) endmacro() if(ARROW_WITH_BROTLI) @@ -1036,6 +1048,8 @@ macro(build_glog) PROPERTIES IMPORTED_LOCATION "${GLOG_STATIC_LIB}" INTERFACE_INCLUDE_DIRECTORIES "${GLOG_INCLUDE_DIR}") add_dependencies(glog::glog glog_ep) + + list(APPEND ARROW_BUNDLED_STATIC_LIBS glog::glog) endmacro() if(ARROW_USE_GLOG) @@ -1105,6 +1119,8 @@ macro(build_gflags) set(GFLAGS_LIBRARIES ${GFLAGS_LIBRARY}) set(GFLAGS_VENDORED TRUE) + + list(APPEND ARROW_BUNDLED_STATIC_LIBS gflags_static) endmacro() if(ARROW_NEED_GFLAGS) @@ -1212,6 +1228,8 @@ macro(build_thrift) add_dependencies(toolchain thrift_ep) add_dependencies(thrift::thrift thrift_ep) set(THRIFT_VERSION ${ARROW_THRIFT_BUILD_VERSION}) + + list(APPEND ARROW_BUNDLED_STATIC_LIBS thrift::thrift) endmacro() if(ARROW_WITH_THRIFT) @@ -1304,6 +1322,8 @@ macro(build_protobuf) add_dependencies(toolchain protobuf_ep) add_dependencies(arrow::protobuf::libprotobuf protobuf_ep) + + list(APPEND ARROW_BUNDLED_STATIC_LIBS arrow::protobuf::libprotobuf) endmacro() if(ARROW_WITH_PROTOBUF) @@ -1440,6 +1460,8 @@ if(ARROW_JEMALLOC) INTERFACE_INCLUDE_DIRECTORIES "${CMAKE_CURRENT_BINARY_DIR}/jemalloc_ep-prefix/src") add_dependencies(jemalloc::jemalloc jemalloc_ep) + + list(APPEND ARROW_BUNDLED_STATIC_LIBS jemalloc::jemalloc) endif() # ---------------------------------------------------------------------- @@ -1490,6 +1512,8 @@ if(ARROW_MIMALLOC) "${MIMALLOC_INCLUDE_DIR}") add_dependencies(mimalloc::mimalloc mimalloc_ep) add_dependencies(toolchain mimalloc_ep) + + list(APPEND ARROW_BUNDLED_STATIC_LIBS mimalloc::mimalloc) endif() # ---------------------------------------------------------------------- @@ -1821,6 +1845,8 @@ macro(build_zlib) add_dependencies(toolchain zlib_ep) add_dependencies(ZLIB::ZLIB zlib_ep) + + list(APPEND ARROW_BUNDLED_STATIC_LIBS ZLIB::ZLIB) endmacro() if(ARROW_WITH_ZLIB) @@ -1883,6 +1909,8 @@ macro(build_lz4) INTERFACE_INCLUDE_DIRECTORIES "${LZ4_PREFIX}/include") add_dependencies(toolchain lz4_ep) add_dependencies(LZ4::lz4 lz4_ep) + + list(APPEND ARROW_BUNDLED_STATIC_LIBS LZ4::lz4) endmacro() if(ARROW_WITH_LZ4) @@ -1945,6 +1973,8 @@ macro(build_zstd) add_dependencies(toolchain zstd_ep) add_dependencies(zstd::libzstd zstd_ep) + + list(APPEND ARROW_BUNDLED_STATIC_LIBS zstd::libzstd) endmacro() if(ARROW_WITH_ZSTD) @@ -1998,6 +2028,8 @@ macro(build_re2) add_dependencies(toolchain re2_ep) add_dependencies(RE2::re2 re2_ep) + + list(APPEND ARROW_BUNDLED_STATIC_LIBS RE2::re2) endmacro() if(ARROW_GANDIVA) @@ -2038,6 +2070,8 @@ macro(build_bzip2) add_dependencies(toolchain bzip2_ep) add_dependencies(BZip2::BZip2 bzip2_ep) + + list(APPEND ARROW_BUNDLED_STATIC_LIBS BZip2::BZip2) endmacro() if(ARROW_WITH_BZ2) @@ -2090,6 +2124,8 @@ macro(build_utf8proc) add_dependencies(toolchain utf8proc_ep) add_dependencies(utf8proc::utf8proc utf8proc_ep) + + list(APPEND ARROW_BUNDLED_STATIC_LIBS utf8proc::utf8proc) endmacro() if(ARROW_WITH_UTF8PROC) @@ -2147,6 +2183,8 @@ macro(build_cares) INTERFACE_INCLUDE_DIRECTORIES "${CARES_INCLUDE_DIR}") set(CARES_VENDORED TRUE) + + list(APPEND ARROW_BUNDLED_STATIC_LIBS c-ares::cares) endmacro() if(ARROW_WITH_GRPC) @@ -2369,6 +2407,8 @@ macro(build_grpc) add_dependencies(gRPC::grpc++ grpc_ep) add_dependencies(gRPC::grpc_cpp_plugin grpc_ep) set(GRPC_VENDORED TRUE) + + list(APPEND ARROW_BUNDLED_STATIC_LIBS gRPC::upb gRPC::gpr gRPC::gprc gRPC::address_sorting gRPC::grpc++) endmacro() if(ARROW_WITH_GRPC) @@ -2513,6 +2553,8 @@ macro(build_orc) add_dependencies(toolchain orc_ep) add_dependencies(orc::liborc orc_ep) + + list(APPEND ARROW_BUNDLED_STATIC_LIBS orc::liborc) endmacro() if(ARROW_ORC) @@ -2573,6 +2615,8 @@ macro(build_awssdk) add_dependencies(toolchain awssdk_ep) set(AWSSDK_LINK_LIBRARIES ${AWSSDK_SHARED_LIBS}) set(AWSSDK_VENDORED TRUE) + + # AWSSDK is shared-only build endmacro() if(ARROW_S3) @@ -2605,6 +2649,8 @@ if(ARROW_S3) endif() endif() +message(STATUS "All bundled static libraries: ${ARROW_BUNDLED_STATIC_LIBS}") + # Write out the package configurations. configure_file("src/arrow/util/config.h.cmake" "src/arrow/util/config.h") diff --git a/cpp/src/arrow/CMakeLists.txt b/cpp/src/arrow/CMakeLists.txt index 818a527a306..308ad7094af 100644 --- a/cpp/src/arrow/CMakeLists.txt +++ b/cpp/src/arrow/CMakeLists.txt @@ -500,6 +500,18 @@ if(ARROW_WITH_BACKTRACE) endforeach() endif() +if (ARROW_BUILD_STATIC AND ARROW_BUNDLED_STATIC_LIBS) + arrow_car(_FIRST_LIB ${ARROW_BUNDLED_STATIC_LIBS}) + arrow_cdr(_OTHER_LIBS ${ARROW_BUNDLED_STATIC_LIBS}) + create_merged_static_lib(arrow_bundled_dependencies + NAME + arrow_bundled_dependencies + ROOT + ${_FIRST_LIB} + TO_MERGE + ${_OTHER_LIBS}) +endif() + if(ARROW_BUILD_TESTS OR ARROW_BUILD_BENCHMARKS OR ARROW_BUILD_INTEGRATION From 5834c6680d7e611f3f2b3456bc32e27042ce32a3 Mon Sep 17 00:00:00 2001 From: Wes McKinney Date: Wed, 8 Jul 2020 23:09:46 -0500 Subject: [PATCH 02/16] Adapt minimal build example to exhibit static linking with dependencies --- cpp/CMakeLists.txt | 12 ----------- cpp/cmake_modules/BuildUtils.cmake | 24 +++++++++++---------- cpp/cmake_modules/ThirdpartyToolchain.cmake | 13 ++++++++--- cpp/examples/minimal_build/CMakeLists.txt | 20 ++++++++++++++++- cpp/src/arrow/CMakeLists.txt | 14 ++++++------ 5 files changed, 49 insertions(+), 34 deletions(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index da916ba3a68..1e4bb235567 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -651,7 +651,6 @@ if(ARROW_USE_OPENSSL) set(ARROW_OPENSSL_LIBS OpenSSL::Crypto OpenSSL::SSL) list(APPEND ARROW_LINK_LIBS ${ARROW_OPENSSL_LIBS}) list(APPEND ARROW_STATIC_LINK_LIBS ${ARROW_OPENSSL_LIBS}) - list(APPEND ARROW_STATIC_INSTALL_INTERFACE_LIBS ${ARROW_OPENSSL_LIBS}) endif() if(ARROW_WITH_BROTLI) @@ -659,45 +658,36 @@ if(ARROW_WITH_BROTLI) set(ARROW_BROTLI_LIBS Brotli::brotlienc Brotli::brotlidec Brotli::brotlicommon) list(APPEND ARROW_LINK_LIBS ${ARROW_BROTLI_LIBS}) list(APPEND ARROW_STATIC_LINK_LIBS ${ARROW_BROTLI_LIBS}) - list(APPEND ARROW_STATIC_INSTALL_INTERFACE_LIBS ${ARROW_BROTLI_LIBS}) endif() if(ARROW_WITH_BZ2) list(APPEND ARROW_STATIC_LINK_LIBS BZip2::BZip2) - list(APPEND ARROW_STATIC_INSTALL_INTERFACE_LIBS BZip2::BZip2) endif() if(ARROW_WITH_LZ4) list(APPEND ARROW_STATIC_LINK_LIBS LZ4::lz4) - list(APPEND ARROW_STATIC_INSTALL_INTERFACE_LIBS LZ4::lz4) endif() if(ARROW_WITH_SNAPPY) list(APPEND ARROW_STATIC_LINK_LIBS Snappy::snappy) - list(APPEND ARROW_STATIC_INSTALL_INTERFACE_LIBS Snappy::snappy) endif() if(ARROW_WITH_ZLIB) list(APPEND ARROW_STATIC_LINK_LIBS ZLIB::ZLIB) - list(APPEND ARROW_STATIC_INSTALL_INTERFACE_LIBS ZLIB::ZLIB) endif() if(ARROW_WITH_ZSTD) list(APPEND ARROW_STATIC_LINK_LIBS ${ARROW_ZSTD_LIBZSTD}) - list(APPEND ARROW_STATIC_INSTALL_INTERFACE_LIBS ${ARROW_ZSTD_LIBZSTD}) endif() if(ARROW_ORC) list(APPEND ARROW_LINK_LIBS orc::liborc ${ARROW_PROTOBUF_LIBPROTOBUF}) list(APPEND ARROW_STATIC_LINK_LIBS orc::liborc ${ARROW_PROTOBUF_LIBPROTOBUF}) - list(APPEND ARROW_STATIC_INSTALL_INTERFACE_LIBS orc::liborc - ${ARROW_PROTOBUF_LIBPROTOBUF}) endif() if(ARROW_USE_GLOG) list(APPEND ARROW_LINK_LIBS glog::glog) list(APPEND ARROW_STATIC_LINK_LIBS glog::glog) - list(APPEND ARROW_STATIC_INSTALL_INTERFACE_LIBS glog::glog) add_definitions("-DARROW_USE_GLOG") endif() @@ -708,7 +698,6 @@ endif() if(ARROW_WITH_UTF8PROC) list(APPEND ARROW_LINK_LIBS utf8proc::utf8proc) list(APPEND ARROW_STATIC_LINK_LIBS utf8proc::utf8proc) - list(APPEND ARROW_STATIC_INSTALL_INTERFACE_LIBS utf8proc::utf8proc) endif() add_custom_target(arrow_dependencies) @@ -832,7 +821,6 @@ endif() list(APPEND ARROW_LINK_LIBS ${ARROW_SYSTEM_LINK_LIBS}) list(APPEND ARROW_STATIC_LINK_LIBS ${ARROW_SYSTEM_LINK_LIBS}) -list(APPEND ARROW_STATIC_INSTALL_INTERFACE_LIBS ${ARROW_SYSTEM_LINK_LIBS}) # # Subdirectories diff --git a/cpp/cmake_modules/BuildUtils.cmake b/cpp/cmake_modules/BuildUtils.cmake index adc3a393461..eb10ebeeb1c 100644 --- a/cpp/cmake_modules/BuildUtils.cmake +++ b/cpp/cmake_modules/BuildUtils.cmake @@ -202,7 +202,9 @@ function(create_merged_static_lib output_target) COMMENT "Bundling ${output_lib_path}" VERBATIM) - message(STATUS "Creating bundled static library target ${output_target} at ${output_lib_path}") + message( + STATUS "Creating bundled static library target ${output_target} at ${output_lib_path}" + ) add_custom_target(${output_target} ALL DEPENDS ${output_lib_path}) add_dependencies(${output_target} ${ARG_ROOT} ${ARG_TO_MERGE}) @@ -432,14 +434,14 @@ function(ADD_ARROW_LIB LIB_NAME) ${LIB_NAME_STATIC}) if(ARG_STATIC_INSTALL_INTERFACE_LIBS) - set(INTERFACE_LIBS ${ARG_STATIC_INSTALL_INTERFACE_LIBS}) - else() - set(INTERFACE_LIBS ${ARG_STATIC_LINK_LIBS}) + target_link_libraries(${LIB_NAME}_static LINK_PUBLIC + "$") endif() - target_link_libraries(${LIB_NAME}_static LINK_PUBLIC - "$" - "$") + if(ARG_STATIC_LINK_LIBS) + target_link_libraries(${LIB_NAME}_static LINK_PRIVATE + "$") + endif() install(TARGETS ${LIB_NAME}_static ${INSTALL_IS_OPTIONAL} EXPORT ${LIB_NAME}_targets @@ -916,9 +918,9 @@ endfunction() # Implementations of lisp "car" and "cdr" functions macro(ARROW_CAR var) - SET(${var} ${ARGV1}) + set(${var} ${ARGV1}) endmacro() -MACRO(ARROW_CDR var rest) - SET(${var} ${ARGN}) -ENDMACRO() +macro(ARROW_CDR var rest) + set(${var} ${ARGN}) +endmacro() diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index 20d88f819c1..d84a1f26527 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -692,7 +692,8 @@ macro(build_boost) add_dependencies(toolchain boost_ep) set(BOOST_VENDORED TRUE) - list(APPEND ARROW_BUNDLED_STATIC_LIBS boost_system_static boost_filesystem_static boost_regex_static) + list(APPEND ARROW_BUNDLED_STATIC_LIBS boost_system_static boost_filesystem_static + boost_regex_static) endmacro() if(ARROW_FLIGHT AND ARROW_BUILD_TESTS) @@ -930,7 +931,8 @@ macro(build_brotli) INTERFACE_INCLUDE_DIRECTORIES "${BROTLI_INCLUDE_DIR}") add_dependencies(Brotli::brotlidec brotli_ep) - list(APPEND ARROW_BUNDLED_STATIC_LIBS Brotli::brotlicommon Brotli::brotlienc Brotli::brotlidec) + list(APPEND ARROW_BUNDLED_STATIC_LIBS Brotli::brotlicommon Brotli::brotlienc + Brotli::brotlidec) endmacro() if(ARROW_WITH_BROTLI) @@ -2408,7 +2410,12 @@ macro(build_grpc) add_dependencies(gRPC::grpc_cpp_plugin grpc_ep) set(GRPC_VENDORED TRUE) - list(APPEND ARROW_BUNDLED_STATIC_LIBS gRPC::upb gRPC::gpr gRPC::gprc gRPC::address_sorting gRPC::grpc++) + list(APPEND ARROW_BUNDLED_STATIC_LIBS + gRPC::upb + gRPC::gpr + gRPC::gprc + gRPC::address_sorting + gRPC::grpc++) endmacro() if(ARROW_WITH_GRPC) diff --git a/cpp/examples/minimal_build/CMakeLists.txt b/cpp/examples/minimal_build/CMakeLists.txt index 4b29f6284b8..96e1f0164b2 100644 --- a/cpp/examples/minimal_build/CMakeLists.txt +++ b/cpp/examples/minimal_build/CMakeLists.txt @@ -19,10 +19,28 @@ cmake_minimum_required(VERSION 3.0) project(ArrowMinimalExample) +option(ARROW_LINK_SHARED "Link to the Arrow shared library" ON) + find_package(Arrow REQUIRED) message(STATUS "Arrow version: ${ARROW_VERSION}") message(STATUS "Arrow SO version: ${ARROW_FULL_SO_VERSION}") add_executable(arrow_example example.cc) -target_link_libraries(arrow_example PRIVATE arrow_shared) + +if (ARROW_LINK_SHARED) + target_link_libraries(arrow_example PRIVATE arrow_shared) +else() + set(THREADS_PREFER_PTHREAD_FLAG ON) + find_package(Threads REQUIRED) + + # We must also link to libarrow_bundled_dependencies.a + get_property(arrow_static_loc TARGET arrow_static PROPERTY LOCATION) + get_filename_component(arrow_lib_dir ${arrow_static_loc} DIRECTORY) + + add_library(arrow_dependencies STATIC IMPORTED) + set_target_properties(arrow_dependencies + PROPERTIES IMPORTED_LOCATION + "${arrow_lib_dir}/${CMAKE_STATIC_LIBRARY_PREFIX}arrow_bundled_dependencies${CMAKE_STATIC_LIBRARY_SUFFIX}") + target_link_libraries(arrow_example PRIVATE arrow_static arrow_dependencies Threads::Threads) +endif() diff --git a/cpp/src/arrow/CMakeLists.txt b/cpp/src/arrow/CMakeLists.txt index 308ad7094af..eea19e54954 100644 --- a/cpp/src/arrow/CMakeLists.txt +++ b/cpp/src/arrow/CMakeLists.txt @@ -500,16 +500,16 @@ if(ARROW_WITH_BACKTRACE) endforeach() endif() -if (ARROW_BUILD_STATIC AND ARROW_BUNDLED_STATIC_LIBS) +if(ARROW_BUILD_STATIC AND ARROW_BUNDLED_STATIC_LIBS) arrow_car(_FIRST_LIB ${ARROW_BUNDLED_STATIC_LIBS}) arrow_cdr(_OTHER_LIBS ${ARROW_BUNDLED_STATIC_LIBS}) create_merged_static_lib(arrow_bundled_dependencies - NAME - arrow_bundled_dependencies - ROOT - ${_FIRST_LIB} - TO_MERGE - ${_OTHER_LIBS}) + NAME + arrow_bundled_dependencies + ROOT + ${_FIRST_LIB} + TO_MERGE + ${_OTHER_LIBS}) endif() if(ARROW_BUILD_TESTS From 1fa8db6bd7c1594b5d4fea409dd170b9f678af65 Mon Sep 17 00:00:00 2001 From: Wes McKinney Date: Thu, 9 Jul 2020 11:57:11 -0500 Subject: [PATCH 03/16] Add run_static.sh for testing the minimal build with static linking --- cpp/CMakeLists.txt | 13 ++-- cpp/examples/minimal_build/Dockerfile | 3 +- cpp/examples/minimal_build/run_static.sh | 86 ++++++++++++++++++++++++ 3 files changed, 97 insertions(+), 5 deletions(-) create mode 100755 cpp/examples/minimal_build/run_static.sh diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 1e4bb235567..37a6fafccfa 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -792,19 +792,24 @@ if(ARROW_BUILD_BENCHMARKS) endif() endif() -set(ARROW_SYSTEM_LINK_LIBS) - if(ARROW_JEMALLOC) add_definitions(-DARROW_JEMALLOC) add_definitions(-DARROW_JEMALLOC_INCLUDE_DIR=${JEMALLOC_INCLUDE_DIR}) - list(APPEND ARROW_SYSTEM_LINK_LIBS jemalloc::jemalloc) + list(APPEND ARROW_LINK_LIBS jemalloc::jemalloc) + list(APPEND ARROW_STATIC_LINK_LIBS jemalloc::jemalloc) endif() if(ARROW_MIMALLOC) add_definitions(-DARROW_MIMALLOC) - list(APPEND ARROW_SYSTEM_LINK_LIBS mimalloc::mimalloc) + list(APPEND ARROW_LINK_LIBS mimalloc::mimalloc) + list(APPEND ARROW_STATIC_LINK_LIBS mimalloc::mimalloc) endif() +# ---------------------------------------------------------------------- +# Handle platform-related libraries like -pthread + +set(ARROW_SYSTEM_LINK_LIBS) + if(THREADS_FOUND) list(APPEND ARROW_SYSTEM_LINK_LIBS Threads::Threads) endif() diff --git a/cpp/examples/minimal_build/Dockerfile b/cpp/examples/minimal_build/Dockerfile index 6f00f475bdc..9b86d9d2389 100644 --- a/cpp/examples/minimal_build/Dockerfile +++ b/cpp/examples/minimal_build/Dockerfile @@ -22,5 +22,6 @@ ENV DEBIAN_FRONTEND=noninteractive RUN apt-get update -y -q && \ apt-get install -y -q --no-install-recommends \ build-essential \ - cmake && \ + cmake \ + ninja-build && \ apt-get clean && rm -rf /var/lib/apt/lists* diff --git a/cpp/examples/minimal_build/run_static.sh b/cpp/examples/minimal_build/run_static.sh new file mode 100755 index 00000000000..67368fd1a41 --- /dev/null +++ b/cpp/examples/minimal_build/run_static.sh @@ -0,0 +1,86 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -e + +cd /io + +: ${ARROW_DIR:=/arrow} +: ${EXAMPLE_DIR:=/io} +: ${ARROW_BUILD_DIR:=/build/arrow} +: ${EXAMPLE_BUILD_DIR:=/build/example} + +echo +echo "==" +echo "== Building Arrow C++ library" +echo "==" +echo + +mkdir -p $ARROW_BUILD_DIR +pushd $ARROW_BUILD_DIR + +cmake $ARROW_DIR/cpp \ + -GNinja \ + -DARROW_DEPENDENCY_SOURCE=BUNDLED \ + -DARROW_BUILD_SHARED=OFF \ + -DARROW_BUILD_STATIC=ON \ + -DARROW_COMPUTE=ON \ + -DARROW_CSV=ON \ + -DARROW_DATASET=ON \ + -DARROW_FILESYSTEM=ON \ + -DARROW_HDFS=ON \ + -DARROW_JEMALLOC=ON \ + -DARROW_JSON=ON \ + -DARROW_ORC=ON \ + -DARROW_PARQUET=ON \ + -DARROW_PLASMA=ON \ + -DARROW_WITH_BROTLI=ON \ + -DARROW_WITH_BZ2=ON \ + -DARROW_WITH_LZ4=ON \ + -DARROW_WITH_SNAPPY=ON \ + -DARROW_WITH_ZLIB=ON \ + -DARROW_WITH_ZSTD=ON \ + $ARROW_CMAKE_OPTIONS + +ninja install + +popd + +/bin/bash + +echo +echo "==" +echo "== Building example project using Arrow C++ library" +echo "==" +echo + +mkdir -p $EXAMPLE_BUILD_DIR +pushd $EXAMPLE_BUILD_DIR + +cmake $EXAMPLE_DIR -DARROW_LINK_SHARED=OFF +make + +popd + +echo +echo "==" +echo "== Running example project" +echo "==" +echo + +${EXAMPLE_BUILD_DIR}/arrow_example From 837f181bab95f4bc0f7d5edf8e436cbc3987d54a Mon Sep 17 00:00:00 2001 From: Wes McKinney Date: Thu, 9 Jul 2020 12:36:54 -0500 Subject: [PATCH 04/16] Build on macOS --- cpp/examples/minimal_build/CMakeLists.txt | 2 ++ cpp/examples/minimal_build/run_static.sh | 2 -- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cpp/examples/minimal_build/CMakeLists.txt b/cpp/examples/minimal_build/CMakeLists.txt index 96e1f0164b2..5df9b74ed0c 100644 --- a/cpp/examples/minimal_build/CMakeLists.txt +++ b/cpp/examples/minimal_build/CMakeLists.txt @@ -23,6 +23,8 @@ option(ARROW_LINK_SHARED "Link to the Arrow shared library" ON) find_package(Arrow REQUIRED) +set(CMAKE_CXX_STANDARD 11) + message(STATUS "Arrow version: ${ARROW_VERSION}") message(STATUS "Arrow SO version: ${ARROW_FULL_SO_VERSION}") diff --git a/cpp/examples/minimal_build/run_static.sh b/cpp/examples/minimal_build/run_static.sh index 67368fd1a41..d5b19fdc260 100755 --- a/cpp/examples/minimal_build/run_static.sh +++ b/cpp/examples/minimal_build/run_static.sh @@ -18,8 +18,6 @@ set -e -cd /io - : ${ARROW_DIR:=/arrow} : ${EXAMPLE_DIR:=/io} : ${ARROW_BUILD_DIR:=/build/arrow} From e2b40851087c64e6127284da0d958150fbb570b1 Mon Sep 17 00:00:00 2001 From: Wes McKinney Date: Thu, 9 Jul 2020 13:21:23 -0500 Subject: [PATCH 05/16] Add run_static.bat for validating minimal Windows build with static bundled dependencies --- cpp/examples/minimal_build/CMakeLists.txt | 1 + cpp/examples/minimal_build/run_static.bat | 88 +++++++++++++++++++++++ 2 files changed, 89 insertions(+) create mode 100644 cpp/examples/minimal_build/run_static.bat diff --git a/cpp/examples/minimal_build/CMakeLists.txt b/cpp/examples/minimal_build/CMakeLists.txt index 5df9b74ed0c..dabfb633edf 100644 --- a/cpp/examples/minimal_build/CMakeLists.txt +++ b/cpp/examples/minimal_build/CMakeLists.txt @@ -24,6 +24,7 @@ option(ARROW_LINK_SHARED "Link to the Arrow shared library" ON) find_package(Arrow REQUIRED) set(CMAKE_CXX_STANDARD 11) +set(CMAKE_BUILD_TYPE Release) message(STATUS "Arrow version: ${ARROW_VERSION}") message(STATUS "Arrow SO version: ${ARROW_FULL_SO_VERSION}") diff --git a/cpp/examples/minimal_build/run_static.bat b/cpp/examples/minimal_build/run_static.bat new file mode 100644 index 00000000000..bbc7ff8f750 --- /dev/null +++ b/cpp/examples/minimal_build/run_static.bat @@ -0,0 +1,88 @@ +@rem Licensed to the Apache Software Foundation (ASF) under one +@rem or more contributor license agreements. See the NOTICE file +@rem distributed with this work for additional information +@rem regarding copyright ownership. The ASF licenses this file +@rem to you under the Apache License, Version 2.0 (the +@rem "License"); you may not use this file except in compliance +@rem with the License. You may obtain a copy of the License at +@rem +@rem http://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, +@rem software distributed under the License is distributed on an +@rem "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +@rem KIND, either express or implied. See the License for the +@rem specific language governing permissions and limitations +@rem under the License. + +@echo on + +@rem clean up prior attempts +if exist "arrow-build" rd arrow-build /s /q +if exist "dist" rd dist /s /q +if exist "example" rd example /s /q + +echo +echo "==" +echo "== Building Arrow C++ library" +echo "==" +echo + +set INSTALL_PREFIX=%cd%\dist + +mkdir arrow-build +pushd arrow-build + +@rem bzip2_ep fails with this method + +cmake ..\..\.. ^ + -GNinja ^ + -DCMAKE_INSTALL_PREFIX=%INSTALL_PREFIX% ^ + -DARROW_DEPENDENCY_SOURCE=BUNDLED ^ + -DARROW_BUILD_SHARED=OFF ^ + -DARROW_BUILD_STATIC=ON ^ + -DARROW_COMPUTE=ON ^ + -DARROW_CSV=ON ^ + -DARROW_DATASET=ON ^ + -DARROW_FILESYSTEM=ON ^ + -DARROW_HDFS=ON ^ + -DARROW_JSON=ON ^ + -DARROW_MIMALLOC=ON ^ + -DARROW_ORC=ON ^ + -DARROW_PARQUET=ON ^ + -DARROW_PLASMA=ON ^ + -DARROW_WITH_BROTLI=ON ^ + -DARROW_WITH_BZ2=OFF ^ + -DARROW_WITH_LZ4=ON ^ + -DARROW_WITH_SNAPPY=ON ^ + -DARROW_WITH_ZLIB=ON ^ + -DARROW_WITH_ZSTD=ON + +ninja install + +popd + +echo +echo "==" +echo "== Building example project using Arrow C++ library" +echo "==" +echo + +mkdir example +pushd example + +cmake .. ^ + -GNinja ^ + -DCMAKE_PREFIX_PATH="%INSTALL_PREFIX%" ^ + -DARROW_LINK_SHARED=OFF +ninja + +popd + +echo +echo "==" +echo "== Running example project" +echo "==" +echo + +call example\arrow_example.exe From e79e5fa76e8d95a76358c84c513533f60926aadd Mon Sep 17 00:00:00 2001 From: Wes McKinney Date: Thu, 9 Jul 2020 14:11:03 -0500 Subject: [PATCH 06/16] Some documentation about static minimal builds --- cpp/examples/minimal_build/README.md | 34 ++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/cpp/examples/minimal_build/README.md b/cpp/examples/minimal_build/README.md index 4c12123b227..85905b40d8f 100644 --- a/cpp/examples/minimal_build/README.md +++ b/cpp/examples/minimal_build/README.md @@ -49,3 +49,37 @@ docker run -it -v $PWD:/io -v $ARROW_ROOT:/arrow arrow_cpp_minimal /io/run.sh Note that this example mounts two volumes inside the Docker image: * `/arrow` points to the Arrow source tree * `/io` points to this example directory + +## Statically-linked builds + +Static linking the Arrow libraries can be tricky depending on your build +configuration. If you are using either `-DARROW_JEMALLOC=ON` or +`-DARROW_MIMALLOC=ON`, then some extra steps are needed to produce a statically +linked build. We've provided an example build configuration here with CMake +that uses the `arrow_bundled_dependencies` static library produced by the Arrow +build system. + +To run it on Linux, you can use the above Docker image: + +```bash +docker run -it -v $PWD:/io -v $ARROW_ROOT:/arrow arrow_cpp_minimal /io/run_static.sh +``` + +On macOS, you can use the `run_static.sh` but you must set some environment +variables to point the script to your Arrow checkout, for example: + +```bash +export ARROW_DIR=path/to/arrow-clone +export EXAMPLE_DIR=$ARROW_DIR/cpp/examples/minimal_build +export ARROW_BUILD_DIR=`pwd`/arrow-build +export EXAMPLE_BUILD_DIR=`pwd`/example + +./run_static.sh +``` + +On Windows, you can run `run_static.bat` from the command prompt with Visual +Studio's command line tools enabled and CMake and ninja build in the path: + +``` +call run_static.bat +``` \ No newline at end of file From 96784aef5e9214541fadadfe2e1c4029a069f884 Mon Sep 17 00:00:00 2001 From: Wes McKinney Date: Thu, 9 Jul 2020 14:23:41 -0500 Subject: [PATCH 07/16] Add some documentation about static linking --- docs/source/developers/cpp/building.rst | 49 +++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/docs/source/developers/cpp/building.rst b/docs/source/developers/cpp/building.rst index 1dc96ef96a9..114e1aa4b3b 100644 --- a/docs/source/developers/cpp/building.rst +++ b/docs/source/developers/cpp/building.rst @@ -309,6 +309,11 @@ version number from ``cpp/thirdparty/versions.txt`` is used. There is also a dependency source downloader script (see below), which can be used to set up offline builds. +When using ``BUNDLED`` for dependency resolution (and if you use either the +jemalloc or mimalloc allocators, which are recommended), statically linking the +Arrow libraries in a third party project is more complex. See below for +instructions about how to configure your build system in this case. + Boost-related Options ~~~~~~~~~~~~~~~~~~~~~ @@ -347,6 +352,50 @@ You can then invoke CMake to create the build directory and it will use the declared environment variable pointing to downloaded archives instead of downloading them (one for each build dir!). +Statically Linking +~~~~~~~~~~~~~~~~~~ + +Static linking the Arrow libraries can be tricky depending on your build +configuration. If you are using either ``-DARROW_JEMALLOC=ON`` or +``-DARROW_MIMALLOC=ON``, or any dependency is set to use the ``BUNDLED`` source, +then some extra steps are needed to produce a statically linked build. + +When ``-DARROW_BUILD_STATIC=ON``, all build dependencies built as static +libraries by the Arrow build system will be merged together to create a static +library ``arrow_bundled_dependencies``. In UNIX-like environments (Linux, macOS, +MinGW), this is called ``libarrow_bundled_dependencies.a`` and on Windows with +Visual Studio ``arrow_bundled_dependencies.lib``. This "dependency bundle" +library is installed in the same place as the other Arrow static libraries. + +For static linking to work, you must include this dependency bundle in your +linker setup. For example, in CMake it could be accomplished like so: + +.. code-block:: cmake + + find_package(Arrow REQUIRED) + get_property(arrow_static_loc TARGET arrow_static PROPERTY LOCATION) + get_filename_component(arrow_lib_dir ${arrow_static_loc} DIRECTORY) + + add_library(arrow_dependencies STATIC IMPORTED) + set_target_properties(arrow_dependencies + PROPERTIES IMPORTED_LOCATION + "${arrow_lib_dir}/${CMAKE_STATIC_LIBRARY_PREFIX}arrow_bundled_dependencies${CMAKE_STATIC_LIBRARY_SUFFIX}") + target_link_libraries(arrow_example PRIVATE arrow_static arrow_dependencies) + +We created an `example CMake-based build configuration +`_ to +help you with this. + +On Linux and macOS, if your application does not link to the ``pthread`` +library already, you must include ``-pthread`` in your linker setup. In CMake +this can be accomplished with the ``Threads`` built-in package: + +.. code-block:: cmake + + set(THREADS_PREFER_PTHREAD_FLAG ON) + find_package(Threads REQUIRED) + target_link_libraries(my_target PRIVATE Threads::Threads) + Extra debugging help ~~~~~~~~~~~~~~~~~~~~ From a96c70714b7cd8efb50d7055b4b56b2809d45b45 Mon Sep 17 00:00:00 2001 From: Wes McKinney Date: Thu, 9 Jul 2020 15:44:10 -0500 Subject: [PATCH 08/16] Review comments, use make instead of ninja --- cpp/CMakeLists.txt | 1 - cpp/examples/minimal_build/Dockerfile | 3 +-- cpp/examples/minimal_build/README.md | 12 ++++++------ cpp/examples/minimal_build/run_static.sh | 8 ++++---- docs/source/developers/cpp/building.rst | 9 +++++---- 5 files changed, 16 insertions(+), 17 deletions(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 37a6fafccfa..3629e57bdff 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -641,7 +641,6 @@ endif() # Linker and Dependencies # -# TODO: Also rework how these libs work # Libraries to link statically with libarrow.so set(ARROW_LINK_LIBS) set(ARROW_STATIC_LINK_LIBS) diff --git a/cpp/examples/minimal_build/Dockerfile b/cpp/examples/minimal_build/Dockerfile index 9b86d9d2389..4c586ebd464 100644 --- a/cpp/examples/minimal_build/Dockerfile +++ b/cpp/examples/minimal_build/Dockerfile @@ -22,6 +22,5 @@ ENV DEBIAN_FRONTEND=noninteractive RUN apt-get update -y -q && \ apt-get install -y -q --no-install-recommends \ build-essential \ - cmake \ - ninja-build && \ + cmake &&\ apt-get clean && rm -rf /var/lib/apt/lists* diff --git a/cpp/examples/minimal_build/README.md b/cpp/examples/minimal_build/README.md index 85905b40d8f..2048f98888b 100644 --- a/cpp/examples/minimal_build/README.md +++ b/cpp/examples/minimal_build/README.md @@ -52,12 +52,12 @@ Note that this example mounts two volumes inside the Docker image: ## Statically-linked builds -Static linking the Arrow libraries can be tricky depending on your build -configuration. If you are using either `-DARROW_JEMALLOC=ON` or -`-DARROW_MIMALLOC=ON`, then some extra steps are needed to produce a statically -linked build. We've provided an example build configuration here with CMake -that uses the `arrow_bundled_dependencies` static library produced by the Arrow -build system. +Static linking the Arrow libraries may require extra steps depending on your +build configuration. If you are using either `-DARROW_JEMALLOC=ON`, +`-DARROW_MIMALLOC=ON`, or if any dependency is built in `BUNDLED` mode, then +some extra steps are needed to produce a statically linked build. We've +provided an example build configuration here with CMake that uses the +`arrow_bundled_dependencies` static library produced by the Arrow build system. To run it on Linux, you can use the above Docker image: diff --git a/cpp/examples/minimal_build/run_static.sh b/cpp/examples/minimal_build/run_static.sh index d5b19fdc260..3b6ad46f209 100755 --- a/cpp/examples/minimal_build/run_static.sh +++ b/cpp/examples/minimal_build/run_static.sh @@ -32,8 +32,9 @@ echo mkdir -p $ARROW_BUILD_DIR pushd $ARROW_BUILD_DIR +NPROC=$(nproc) + cmake $ARROW_DIR/cpp \ - -GNinja \ -DARROW_DEPENDENCY_SOURCE=BUNDLED \ -DARROW_BUILD_SHARED=OFF \ -DARROW_BUILD_STATIC=ON \ @@ -55,12 +56,11 @@ cmake $ARROW_DIR/cpp \ -DARROW_WITH_ZSTD=ON \ $ARROW_CMAKE_OPTIONS -ninja install +make -j$NPROC +make install popd -/bin/bash - echo echo "==" echo "== Building example project using Arrow C++ library" diff --git a/docs/source/developers/cpp/building.rst b/docs/source/developers/cpp/building.rst index 114e1aa4b3b..f88fde5b183 100644 --- a/docs/source/developers/cpp/building.rst +++ b/docs/source/developers/cpp/building.rst @@ -355,10 +355,11 @@ downloading them (one for each build dir!). Statically Linking ~~~~~~~~~~~~~~~~~~ -Static linking the Arrow libraries can be tricky depending on your build -configuration. If you are using either ``-DARROW_JEMALLOC=ON`` or -``-DARROW_MIMALLOC=ON``, or any dependency is set to use the ``BUNDLED`` source, -then some extra steps are needed to produce a statically linked build. +Static linking the Arrow libraries may require extra steps depending on your +build configuration. If you are using either ``-DARROW_JEMALLOC=ON`` or +``-DARROW_MIMALLOC=ON``, or any dependency is set to use the ``BUNDLED`` +source, then you must do some extra configuration to include these bundled +dependencies when linking. When ``-DARROW_BUILD_STATIC=ON``, all build dependencies built as static libraries by the Arrow build system will be merged together to create a static From 7afaae16baf062e084e660bb08d3f8f7f73f0b1f Mon Sep 17 00:00:00 2001 From: Neal Richardson Date: Thu, 9 Jul 2020 14:59:22 -0700 Subject: [PATCH 09/16] Try to wire up the new bundled lib everywhere (for R) and turn on jemalloc/mimalloc --- ci/scripts/PKGBUILD | 1 + dev/tasks/homebrew-formulae/apache-arrow.rb | 1 + .../homebrew-formulae/autobrew/apache-arrow.rb | 2 +- r/configure | 2 +- r/configure.win | 2 +- r/inst/build_arrow_static.sh | 14 -------------- 6 files changed, 5 insertions(+), 17 deletions(-) diff --git a/ci/scripts/PKGBUILD b/ci/scripts/PKGBUILD index ca29f0d84db..4bf08b7151d 100644 --- a/ci/scripts/PKGBUILD +++ b/ci/scripts/PKGBUILD @@ -90,6 +90,7 @@ build() { -DARROW_HDFS=OFF \ -DARROW_JEMALLOC=OFF \ -DARROW_JSON=ON \ + -DARROW_MIMALLOC=ON \ -DARROW_PACKAGE_PREFIX="${MINGW_PREFIX}" \ -DARROW_PARQUET=ON \ -DARROW_USE_GLOG=OFF \ diff --git a/dev/tasks/homebrew-formulae/apache-arrow.rb b/dev/tasks/homebrew-formulae/apache-arrow.rb index c5dbb526c8b..81949d16c28 100644 --- a/dev/tasks/homebrew-formulae/apache-arrow.rb +++ b/dev/tasks/homebrew-formulae/apache-arrow.rb @@ -26,6 +26,7 @@ def install args = %W[ -DARROW_FLIGHT=ON -DARROW_GANDIVA=ON + -DARROW_JEMALLOC=ON -DARROW_ORC=ON -DARROW_PARQUET=ON -DARROW_PLASMA=ON diff --git a/dev/tasks/homebrew-formulae/autobrew/apache-arrow.rb b/dev/tasks/homebrew-formulae/autobrew/apache-arrow.rb index 691366b2be0..9e78c1ae481 100644 --- a/dev/tasks/homebrew-formulae/autobrew/apache-arrow.rb +++ b/dev/tasks/homebrew-formulae/autobrew/apache-arrow.rb @@ -47,7 +47,7 @@ def install -DARROW_JSON=ON -DARROW_PARQUET=ON -DARROW_BUILD_SHARED=OFF - -DARROW_JEMALLOC=OFF + -DARROW_JEMALLOC=ON -DARROW_USE_GLOG=OFF -DARROW_PYTHON=OFF -DARROW_S3=OFF diff --git a/r/configure b/r/configure index 85f16e08126..71c885a593d 100755 --- a/r/configure +++ b/r/configure @@ -114,7 +114,7 @@ else fi fi PKG_CFLAGS="-I$BREWDIR/opt/$PKG_BREW_NAME/include" - PKG_LIBS="-L$BREWDIR/opt/$PKG_BREW_NAME/lib $PKG_LIBS" + PKG_LIBS="-L$BREWDIR/opt/$PKG_BREW_NAME/lib $PKG_LIBS -larrow_bundled_dependencies" elif [ "$UNAME" = "Linux" ]; then # Set some default values/backwards compatibility if [ "${LIBARROW_DOWNLOAD}" = "" ] && [ "${NOT_CRAN}" != "" ]; then diff --git a/r/configure.win b/r/configure.win index 3072d0e9e6d..682f93ddfdd 100644 --- a/r/configure.win +++ b/r/configure.win @@ -47,7 +47,7 @@ OPENSSL_LIBS="-lcrypto -lcrypt32" # NOTE: If you make changes to the libraries below, you should also change # ci/scripts/r_windows_build.sh and ci/scripts/PKGBUILD PKG_CFLAGS="-I${RWINLIB}/include -DARROW_STATIC -DPARQUET_STATIC -DARROW_DS_STATIC -DARROW_R_WITH_ARROW" -PKG_LIBS="-L${RWINLIB}/lib"'$(subst gcc,,$(COMPILED_BY))$(R_ARCH) '"-L${RWINLIB}/lib"'$(R_ARCH) '"-lparquet -larrow_dataset -larrow -lthrift -lsnappy -lz -lzstd -llz4 ${OPENSSL_LIBS} -lws2_32" +PKG_LIBS="-L${RWINLIB}/lib"'$(subst gcc,,$(COMPILED_BY))$(R_ARCH) '"-L${RWINLIB}/lib"'$(R_ARCH) '"-lparquet -larrow_dataset -larrow -larrow_bundled_dependencies -lthrift -lsnappy -lz -lzstd -llz4 ${OPENSSL_LIBS} -lws2_32" # Set any user-defined CXXFLAGS if [ "$ARROW_R_CXXFLAGS" ]; then diff --git a/r/inst/build_arrow_static.sh b/r/inst/build_arrow_static.sh index 73aba450ed7..8593b896ec6 100755 --- a/r/inst/build_arrow_static.sh +++ b/r/inst/build_arrow_static.sh @@ -94,18 +94,4 @@ if [ $? -ne 0 ]; then fi fi -if [ -d "${DEST_DIR}/lib" ]; then - # If the build failed and the CMAKE_INSTALL_LIBDIR doesn't exist, don't try - # to do this stuff because the errors will be misleading - - # Copy the bundled static libs from the build to the install dir - # See https://issues.apache.org/jira/browse/ARROW-7499 for moving this to CMake - find . -regex .*/.*/lib/.*\\.a\$ | xargs -I{} cp -u {} ${DEST_DIR}/lib - # jemalloc makes both libjemalloc.a and libjemalloc_pic.a; we can't use the former, only the latter - rm ${DEST_DIR}/lib/libjemalloc.a || true - # -lbrotlicommon-static needs to come after the other brotli libs, so rename it so alpha sort works - if [ -f "${DEST_DIR}/lib/libbrotlicommon-static.a" ]; then - mv "${DEST_DIR}/lib/libbrotlicommon-static.a" "${DEST_DIR}/lib/libbrotlizzz-static.a" - fi -fi popd From 21c6141e008273c7e494b83603ee271a16e9a992 Mon Sep 17 00:00:00 2001 From: Neal Richardson Date: Thu, 9 Jul 2020 16:12:12 -0700 Subject: [PATCH 10/16] Verbose thirdparty for autobrew --- dev/tasks/homebrew-formulae/autobrew/apache-arrow.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/dev/tasks/homebrew-formulae/autobrew/apache-arrow.rb b/dev/tasks/homebrew-formulae/autobrew/apache-arrow.rb index 9e78c1ae481..755c01cfa00 100644 --- a/dev/tasks/homebrew-formulae/autobrew/apache-arrow.rb +++ b/dev/tasks/homebrew-formulae/autobrew/apache-arrow.rb @@ -56,6 +56,7 @@ def install -DARROW_WITH_SNAPPY=ON -DARROW_WITH_UTF8PROC=OFF -DARROW_BUILD_UTILITIES=ON + -DARROW_VERBOSE_THIRDPARTY_BUILD=ON -DCMAKE_UNITY_BUILD=ON -DPARQUET_BUILD_EXECUTABLES=ON -DLZ4_HOME=#{Formula["lz4"].prefix} From 4812a7cc3a65b31c34f36b549b0dd5cdd77c38f2 Mon Sep 17 00:00:00 2001 From: Wes McKinney Date: Thu, 9 Jul 2020 18:28:12 -0500 Subject: [PATCH 11/16] Do not build mimalloc shared library --- cpp/cmake_modules/ThirdpartyToolchain.cmake | 1 + 1 file changed, 1 insertion(+) diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index d84a1f26527..d6008ebc4ab 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -1494,6 +1494,7 @@ if(ARROW_MIMALLOC) -DMI_OVERRIDE=OFF -DMI_LOCAL_DYNAMIC_TLS=ON -DMI_BUILD_OBJECT=OFF + -DMI_BUILD_SHARED=OFF -DMI_BUILD_TESTS=OFF) externalproject_add(mimalloc_ep From e62d5aef620dfa3e1000518adbfcf22da777c90c Mon Sep 17 00:00:00 2001 From: Wes McKinney Date: Thu, 9 Jul 2020 19:45:46 -0500 Subject: [PATCH 12/16] Add mimalloc transitive dependencies --- r/configure.win | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/r/configure.win b/r/configure.win index 682f93ddfdd..5caf8e24663 100644 --- a/r/configure.win +++ b/r/configure.win @@ -43,11 +43,12 @@ else RWINLIB="../windows/$(ls windows/ | grep ^arrow-)" fi OPENSSL_LIBS="-lcrypto -lcrypt32" +MIMALLOC_LIBS="-lbcrypt -lpsapi" # NOTE: If you make changes to the libraries below, you should also change # ci/scripts/r_windows_build.sh and ci/scripts/PKGBUILD PKG_CFLAGS="-I${RWINLIB}/include -DARROW_STATIC -DPARQUET_STATIC -DARROW_DS_STATIC -DARROW_R_WITH_ARROW" -PKG_LIBS="-L${RWINLIB}/lib"'$(subst gcc,,$(COMPILED_BY))$(R_ARCH) '"-L${RWINLIB}/lib"'$(R_ARCH) '"-lparquet -larrow_dataset -larrow -larrow_bundled_dependencies -lthrift -lsnappy -lz -lzstd -llz4 ${OPENSSL_LIBS} -lws2_32" +PKG_LIBS="-L${RWINLIB}/lib"'$(subst gcc,,$(COMPILED_BY))$(R_ARCH) '"-L${RWINLIB}/lib"'$(R_ARCH) '"-lparquet -larrow_dataset -larrow -larrow_bundled_dependencies -lthrift -lsnappy -lz -lzstd -llz4 ${MIMALLOC_LIBS} ${OPENSSL_LIBS} -lws2_32" # Set any user-defined CXXFLAGS if [ "$ARROW_R_CXXFLAGS" ]; then From 1f24b9e60587f1b42eeeeee0a8d903e7cc9fc051 Mon Sep 17 00:00:00 2001 From: Wes McKinney Date: Fri, 10 Jul 2020 14:43:39 -0500 Subject: [PATCH 13/16] Apply Kou's suggestions to ArrowConfig.cmake.in and try to fix Linux package builds --- cpp/cmake_modules/ThirdpartyToolchain.cmake | 1 + cpp/examples/minimal_build/CMakeLists.txt | 11 +------- cpp/examples/minimal_build/Dockerfile | 2 +- cpp/examples/minimal_build/README.md | 8 ++---- cpp/examples/minimal_build/run_static.sh | 2 ++ cpp/src/arrow/ArrowConfig.cmake.in | 20 +++++++++++++ .../debian.ubuntu-xenial/libarrow-dev.install | 1 + .../apache-arrow/debian/libarrow-dev.install | 1 + .../apache-arrow/yum/arrow.spec.in | 1 + docs/source/developers/cpp/building.rst | 28 ++++--------------- 10 files changed, 35 insertions(+), 40 deletions(-) diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index d6008ebc4ab..8f10a43c341 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -1428,6 +1428,7 @@ if(ARROW_JEMALLOC) "--with-jemalloc-prefix=je_arrow_" "--with-private-namespace=je_arrow_private_" "--without-export" + "--disable-shared" # Don't override operator new() "--disable-cxx" "--disable-libdl" # See https://github.com/jemalloc/jemalloc/issues/1237 diff --git a/cpp/examples/minimal_build/CMakeLists.txt b/cpp/examples/minimal_build/CMakeLists.txt index dabfb633edf..9fc20c70fe0 100644 --- a/cpp/examples/minimal_build/CMakeLists.txt +++ b/cpp/examples/minimal_build/CMakeLists.txt @@ -36,14 +36,5 @@ if (ARROW_LINK_SHARED) else() set(THREADS_PREFER_PTHREAD_FLAG ON) find_package(Threads REQUIRED) - - # We must also link to libarrow_bundled_dependencies.a - get_property(arrow_static_loc TARGET arrow_static PROPERTY LOCATION) - get_filename_component(arrow_lib_dir ${arrow_static_loc} DIRECTORY) - - add_library(arrow_dependencies STATIC IMPORTED) - set_target_properties(arrow_dependencies - PROPERTIES IMPORTED_LOCATION - "${arrow_lib_dir}/${CMAKE_STATIC_LIBRARY_PREFIX}arrow_bundled_dependencies${CMAKE_STATIC_LIBRARY_SUFFIX}") - target_link_libraries(arrow_example PRIVATE arrow_static arrow_dependencies Threads::Threads) + target_link_libraries(arrow_example PRIVATE arrow_static Threads::Threads) endif() diff --git a/cpp/examples/minimal_build/Dockerfile b/cpp/examples/minimal_build/Dockerfile index 4c586ebd464..6f00f475bdc 100644 --- a/cpp/examples/minimal_build/Dockerfile +++ b/cpp/examples/minimal_build/Dockerfile @@ -22,5 +22,5 @@ ENV DEBIAN_FRONTEND=noninteractive RUN apt-get update -y -q && \ apt-get install -y -q --no-install-recommends \ build-essential \ - cmake &&\ + cmake && \ apt-get clean && rm -rf /var/lib/apt/lists* diff --git a/cpp/examples/minimal_build/README.md b/cpp/examples/minimal_build/README.md index 2048f98888b..b79d20c2a0e 100644 --- a/cpp/examples/minimal_build/README.md +++ b/cpp/examples/minimal_build/README.md @@ -52,12 +52,8 @@ Note that this example mounts two volumes inside the Docker image: ## Statically-linked builds -Static linking the Arrow libraries may require extra steps depending on your -build configuration. If you are using either `-DARROW_JEMALLOC=ON`, -`-DARROW_MIMALLOC=ON`, or if any dependency is built in `BUNDLED` mode, then -some extra steps are needed to produce a statically linked build. We've -provided an example build configuration here with CMake that uses the -`arrow_bundled_dependencies` static library produced by the Arrow build system. +We've provided an example build configuration here with CMake to show how to +create a statically-linked executable . To run it on Linux, you can use the above Docker image: diff --git a/cpp/examples/minimal_build/run_static.sh b/cpp/examples/minimal_build/run_static.sh index 3b6ad46f209..941a29f3186 100755 --- a/cpp/examples/minimal_build/run_static.sh +++ b/cpp/examples/minimal_build/run_static.sh @@ -81,4 +81,6 @@ echo "== Running example project" echo "==" echo +cd $EXAMPLE_DIR + ${EXAMPLE_BUILD_DIR}/arrow_example diff --git a/cpp/src/arrow/ArrowConfig.cmake.in b/cpp/src/arrow/ArrowConfig.cmake.in index 0e595066dda..c4b35d5aba3 100644 --- a/cpp/src/arrow/ArrowConfig.cmake.in +++ b/cpp/src/arrow/ArrowConfig.cmake.in @@ -40,4 +40,24 @@ include("${CMAKE_CURRENT_LIST_DIR}/ArrowOptions.cmake") # already existent target error. if(NOT (TARGET arrow_shared OR TARGET arrow_static)) include("${CMAKE_CURRENT_LIST_DIR}/ArrowTargets.cmake") + + if(TARGET arrow_static) + get_property(arrow_static_loc TARGET arrow_static PROPERTY LOCATION) + get_filename_component(arrow_lib_dir ${arrow_static_loc} DIRECTORY) + + add_library(arrow_bundled_dependencies STATIC IMPORTED) + set_target_properties( + arrow_bundled_dependencies + PROPERTIES + IMPORTED_LOCATION + "${arrow_lib_dir}/${CMAKE_STATIC_LIBRARY_PREFIX}arrow_bundled_dependencies${CMAKE_STATIC_LIBRARY_SUFFIX}" + ) + + get_property(arrow_static_interface_link_libraries + TARGET arrow_static + PROPERTY INTERFACE_LINK_LIBRARIES) + set_target_properties( + arrow_static PROPERTIES INTERFACE_LINK_LIBRARIES + "${arrow_static_interface_link_libraries};arrow_bundled_dependencies") + endif() endif() diff --git a/dev/tasks/linux-packages/apache-arrow/debian.ubuntu-xenial/libarrow-dev.install b/dev/tasks/linux-packages/apache-arrow/debian.ubuntu-xenial/libarrow-dev.install index 4f1670a39a8..443e70cf0c1 100644 --- a/dev/tasks/linux-packages/apache-arrow/debian.ubuntu-xenial/libarrow-dev.install +++ b/dev/tasks/linux-packages/apache-arrow/debian.ubuntu-xenial/libarrow-dev.install @@ -6,6 +6,7 @@ usr/lib/*/cmake/arrow/FindArrow.cmake usr/lib/*/cmake/arrow/arrow-config.cmake usr/lib/*/libarrow.a usr/lib/*/libarrow.so +usr/lib/*/libarrow_bundled_dependencies.a usr/lib/*/pkgconfig/arrow-compute.pc usr/lib/*/pkgconfig/arrow-csv.pc usr/lib/*/pkgconfig/arrow-filesystem.pc diff --git a/dev/tasks/linux-packages/apache-arrow/debian/libarrow-dev.install b/dev/tasks/linux-packages/apache-arrow/debian/libarrow-dev.install index 4f1670a39a8..443e70cf0c1 100644 --- a/dev/tasks/linux-packages/apache-arrow/debian/libarrow-dev.install +++ b/dev/tasks/linux-packages/apache-arrow/debian/libarrow-dev.install @@ -6,6 +6,7 @@ usr/lib/*/cmake/arrow/FindArrow.cmake usr/lib/*/cmake/arrow/arrow-config.cmake usr/lib/*/libarrow.a usr/lib/*/libarrow.so +usr/lib/*/libarrow_bundled_dependencies.a usr/lib/*/pkgconfig/arrow-compute.pc usr/lib/*/pkgconfig/arrow-csv.pc usr/lib/*/pkgconfig/arrow-filesystem.pc diff --git a/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in b/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in index f36fdfd66de..77a665c2dea 100644 --- a/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in +++ b/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in @@ -268,6 +268,7 @@ Libraries and header files for Apache Arrow C++. %{_libdir}/cmake/arrow/arrow-config.cmake %{_libdir}/libarrow.a %{_libdir}/libarrow.so +%{_libdir}/libarrow_bundled_dependencies.a %{_libdir}/pkgconfig/arrow-compute.pc %{_libdir}/pkgconfig/arrow-csv.pc %{_libdir}/pkgconfig/arrow-filesystem.pc diff --git a/docs/source/developers/cpp/building.rst b/docs/source/developers/cpp/building.rst index f88fde5b183..fa2c4b05453 100644 --- a/docs/source/developers/cpp/building.rst +++ b/docs/source/developers/cpp/building.rst @@ -355,12 +355,6 @@ downloading them (one for each build dir!). Statically Linking ~~~~~~~~~~~~~~~~~~ -Static linking the Arrow libraries may require extra steps depending on your -build configuration. If you are using either ``-DARROW_JEMALLOC=ON`` or -``-DARROW_MIMALLOC=ON``, or any dependency is set to use the ``BUNDLED`` -source, then you must do some extra configuration to include these bundled -dependencies when linking. - When ``-DARROW_BUILD_STATIC=ON``, all build dependencies built as static libraries by the Arrow build system will be merged together to create a static library ``arrow_bundled_dependencies``. In UNIX-like environments (Linux, macOS, @@ -368,24 +362,12 @@ MinGW), this is called ``libarrow_bundled_dependencies.a`` and on Windows with Visual Studio ``arrow_bundled_dependencies.lib``. This "dependency bundle" library is installed in the same place as the other Arrow static libraries. -For static linking to work, you must include this dependency bundle in your -linker setup. For example, in CMake it could be accomplished like so: - -.. code-block:: cmake - - find_package(Arrow REQUIRED) - get_property(arrow_static_loc TARGET arrow_static PROPERTY LOCATION) - get_filename_component(arrow_lib_dir ${arrow_static_loc} DIRECTORY) - - add_library(arrow_dependencies STATIC IMPORTED) - set_target_properties(arrow_dependencies - PROPERTIES IMPORTED_LOCATION - "${arrow_lib_dir}/${CMAKE_STATIC_LIBRARY_PREFIX}arrow_bundled_dependencies${CMAKE_STATIC_LIBRARY_SUFFIX}") - target_link_libraries(arrow_example PRIVATE arrow_static arrow_dependencies) - -We created an `example CMake-based build configuration +If you are using CMake, the bundled dependencies will automatically be included +when linking if you use the ``arrow_static`` CMake target. In other build +systems, you may need to explicitly link to the dependency bundle. We created +an `example CMake-based build configuration `_ to -help you with this. +show you a working example. On Linux and macOS, if your application does not link to the ``pthread`` library already, you must include ``-pthread`` in your linker setup. In CMake From 09c80571a035e339c9f0e40259d5cc5ef384bfa6 Mon Sep 17 00:00:00 2001 From: Wes McKinney Date: Fri, 10 Jul 2020 15:52:10 -0500 Subject: [PATCH 14/16] Attempted bundling of gRPC and its dependencies --- cpp/cmake_modules/ThirdpartyToolchain.cmake | 31 +++++++++++++++++---- cpp/src/arrow/flight/CMakeLists.txt | 2 +- 2 files changed, 27 insertions(+), 6 deletions(-) diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index 8f10a43c341..a63350cdeff 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -2412,12 +2412,33 @@ macro(build_grpc) add_dependencies(gRPC::grpc_cpp_plugin grpc_ep) set(GRPC_VENDORED TRUE) + # ar -M rejects with the "libgrpc++.a" filename because "+" is a line + # continuation character in these scripts, so we have to create a copy of the + # static lib that we will bundle later + + set(GRPC_STATIC_LIBRARY_GRPCPP_FOR_AR + "${CMAKE_CURRENT_BINARY_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}grpcpp${CMAKE_STATIC_LIBRARY_SUFFIX}") + add_custom_command(OUTPUT ${GRPC_STATIC_LIBRARY_GRPCPP_FOR_AR} + COMMAND ${CMAKE_COMMAND} -E copy $ + ${GRPC_STATIC_LIBRARY_GRPCPP_FOR_AR} + DEPENDS grpc_ep) + add_library(gRPC::grpcpp_for_bundling STATIC IMPORTED) + set_target_properties( + gRPC::grpcpp_for_bundling + PROPERTIES IMPORTED_LOCATION + "${GRPC_STATIC_LIBRARY_GRPCPP_FOR_AR}") + + set_source_files_properties("${GRPC_STATIC_LIBRARY_GRPCPP_FOR_AR}" PROPERTIES GENERATED TRUE) + add_custom_target(grpc_copy_grpc++ ALL DEPENDS "${GRPC_STATIC_LIBRARY_GRPCPP_FOR_AR}") + add_dependencies(gRPC::grpcpp_for_bundling grpc_copy_grpc++) + list(APPEND ARROW_BUNDLED_STATIC_LIBS - gRPC::upb - gRPC::gpr - gRPC::gprc - gRPC::address_sorting - gRPC::grpc++) + ${ABSL_LIBRARIES} + gRPC::upb + gRPC::gpr + gRPC::grpc + gRPC::address_sorting + gRPC::grpcpp_for_bundling) endmacro() if(ARROW_WITH_GRPC) diff --git a/cpp/src/arrow/flight/CMakeLists.txt b/cpp/src/arrow/flight/CMakeLists.txt index 4cdadb42458..17aade6e584 100644 --- a/cpp/src/arrow/flight/CMakeLists.txt +++ b/cpp/src/arrow/flight/CMakeLists.txt @@ -32,7 +32,6 @@ if(ARROW_TEST_LINKAGE STREQUAL "static") ${ARROW_TEST_LINK_LIBS}) else() set(ARROW_FLIGHT_TEST_LINK_LIBS arrow_flight_shared arrow_flight_testing_shared - ${ARROW_FLIGHT_STATIC_LINK_LIBS} ${ARROW_TEST_LINK_LIBS}) endif() @@ -92,6 +91,7 @@ add_arrow_lib(arrow_flight ${ARROW_VERSION_SCRIPT_FLAGS} # Defined in cpp/arrow/CMakeLists.txt SHARED_LINK_LIBS arrow_shared + SHARED_PRIVATE_LINK_LIBS ${ARROW_FLIGHT_STATIC_LINK_LIBS} STATIC_LINK_LIBS arrow_static From 9d3aa76fd5324b67c522f66d4c484872000d586c Mon Sep 17 00:00:00 2001 From: Wes McKinney Date: Fri, 10 Jul 2020 16:00:18 -0500 Subject: [PATCH 15/16] Dependencies of libarrow_flight.so currently need to see their transitive dependencies --- cpp/src/arrow/flight/CMakeLists.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/cpp/src/arrow/flight/CMakeLists.txt b/cpp/src/arrow/flight/CMakeLists.txt index 17aade6e584..9ce4654f90b 100644 --- a/cpp/src/arrow/flight/CMakeLists.txt +++ b/cpp/src/arrow/flight/CMakeLists.txt @@ -91,7 +91,6 @@ add_arrow_lib(arrow_flight ${ARROW_VERSION_SCRIPT_FLAGS} # Defined in cpp/arrow/CMakeLists.txt SHARED_LINK_LIBS arrow_shared - SHARED_PRIVATE_LINK_LIBS ${ARROW_FLIGHT_STATIC_LINK_LIBS} STATIC_LINK_LIBS arrow_static From f43ce4606996ed44980dba514c824786e70729d3 Mon Sep 17 00:00:00 2001 From: Wes McKinney Date: Fri, 10 Jul 2020 16:02:10 -0500 Subject: [PATCH 16/16] cmake-format --- cpp/cmake_modules/ThirdpartyToolchain.cmake | 37 ++++++++++++--------- 1 file changed, 21 insertions(+), 16 deletions(-) diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index a63350cdeff..3c229d270c2 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -2416,29 +2416,34 @@ macro(build_grpc) # continuation character in these scripts, so we have to create a copy of the # static lib that we will bundle later - set(GRPC_STATIC_LIBRARY_GRPCPP_FOR_AR - "${CMAKE_CURRENT_BINARY_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}grpcpp${CMAKE_STATIC_LIBRARY_SUFFIX}") + set( + GRPC_STATIC_LIBRARY_GRPCPP_FOR_AR + "${CMAKE_CURRENT_BINARY_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}grpcpp${CMAKE_STATIC_LIBRARY_SUFFIX}" + ) add_custom_command(OUTPUT ${GRPC_STATIC_LIBRARY_GRPCPP_FOR_AR} - COMMAND ${CMAKE_COMMAND} -E copy $ - ${GRPC_STATIC_LIBRARY_GRPCPP_FOR_AR} - DEPENDS grpc_ep) + COMMAND ${CMAKE_COMMAND} + -E + copy + $ + ${GRPC_STATIC_LIBRARY_GRPCPP_FOR_AR} + DEPENDS grpc_ep) add_library(gRPC::grpcpp_for_bundling STATIC IMPORTED) - set_target_properties( - gRPC::grpcpp_for_bundling - PROPERTIES IMPORTED_LOCATION - "${GRPC_STATIC_LIBRARY_GRPCPP_FOR_AR}") + set_target_properties(gRPC::grpcpp_for_bundling + PROPERTIES IMPORTED_LOCATION + "${GRPC_STATIC_LIBRARY_GRPCPP_FOR_AR}") - set_source_files_properties("${GRPC_STATIC_LIBRARY_GRPCPP_FOR_AR}" PROPERTIES GENERATED TRUE) + set_source_files_properties("${GRPC_STATIC_LIBRARY_GRPCPP_FOR_AR}" PROPERTIES GENERATED + TRUE) add_custom_target(grpc_copy_grpc++ ALL DEPENDS "${GRPC_STATIC_LIBRARY_GRPCPP_FOR_AR}") add_dependencies(gRPC::grpcpp_for_bundling grpc_copy_grpc++) list(APPEND ARROW_BUNDLED_STATIC_LIBS - ${ABSL_LIBRARIES} - gRPC::upb - gRPC::gpr - gRPC::grpc - gRPC::address_sorting - gRPC::grpcpp_for_bundling) + ${ABSL_LIBRARIES} + gRPC::upb + gRPC::gpr + gRPC::grpc + gRPC::address_sorting + gRPC::grpcpp_for_bundling) endmacro() if(ARROW_WITH_GRPC)