From 7a3b6e0f3d41dfcb411809aed596e3a0c7b36663 Mon Sep 17 00:00:00 2001 From: Wes McKinney Date: Thu, 16 Jan 2020 18:46:34 -0600 Subject: [PATCH 1/6] Tinkering --- cpp/cmake_modules/BuildUtils.cmake | 53 ++++++++++++++++++++++++++++++ cpp/src/arrow/CMakeLists.txt | 2 ++ 2 files changed, 55 insertions(+) diff --git a/cpp/cmake_modules/BuildUtils.cmake b/cpp/cmake_modules/BuildUtils.cmake index e4e13cb7061..8f855e7adac 100644 --- a/cpp/cmake_modules/BuildUtils.cmake +++ b/cpp/cmake_modules/BuildUtils.cmake @@ -128,6 +128,58 @@ function(REUSE_PRECOMPILED_HEADER_LIB TARGET_NAME LIB_NAME) endif() endfunction() +function(create_merged_static_lib output_target) + set(options) + set(one_value_args NAME ROOT) + set(multi_value_args TO_MERGE) + cmake_parse_arguments(ARG + "${options}" + "${one_value_args}" + "${multi_value_args}" + ${ARGN}) + if(ARG_UNPARSED_ARGUMENTS) + message(SEND_ERROR "Error: unrecognized arguments: ${ARG_UNPARSED_ARGUMENTS}") + endif() + + set(output_lib_path + ${CMAKE_BINARY_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}${ARG_NAME}${CMAKE_STATIC_LIBRARY_SUFFIX}) + + set(ar_script_path ${CMAKE_BINARY_DIR}/${ARG_NAME}.ar) + + file(WRITE ${ar_script_path}.in "CREATE ${output_lib_path}\n" ) + + file(APPEND ${ar_script_path}.in "ADDLIB $\n") + foreach(lib ${ARG_TO_MERGE}) + file(APPEND ${ar_script_path}.in "ADDLIB $\n") + endforeach() + file(APPEND ${ar_script_path}.in "SAVE\nEND\n") + + file(GENERATE + OUTPUT ${ar_script_path} + INPUT ${ar_script_path}.in) + + set(ar_tool ${CMAKE_AR}) + if (CMAKE_INTERPROCEDURAL_OPTIMIZATION) + set(ar_tool ${CMAKE_CXX_COMPILER_AR}) + endif() + + add_custom_command( + COMMAND ${ar_tool} -M < ${ar_script_path} + OUTPUT ${output_lib_path} + COMMENT "Bundling ${bundled_tgt_name}" + VERBATIM) + + add_custom_target(${output_target}_bundling ALL DEPENDS ${output_lib_path}) + add_dependencies(${output_target}_bundling ${ARG_ROOT} ${ARG_TO_MERGE}) + + add_library(${output_target} STATIC IMPORTED) + set_target_properties(${output_target} + PROPERTIES + IMPORTED_LOCATION ${output_lib_path} + INTERFACE_INCLUDE_DIRECTORIES $) + add_dependencies(${output_target} ${output_target}_bundling) +endfunction() + # \arg OUTPUTS list to append built targets to function(ADD_ARROW_LIB LIB_NAME) set(options BUILD_SHARED BUILD_STATIC) @@ -140,6 +192,7 @@ function(ADD_ARROW_LIB LIB_NAME) STATIC_LINK_LIBS SHARED_LINK_LIBS SHARED_PRIVATE_LINK_LIBS + BUNDLE_STATIC_LIBS EXTRA_INCLUDES PRIVATE_INCLUDES DEPENDENCIES diff --git a/cpp/src/arrow/CMakeLists.txt b/cpp/src/arrow/CMakeLists.txt index a9fcbbc596b..1098149490d 100644 --- a/cpp/src/arrow/CMakeLists.txt +++ b/cpp/src/arrow/CMakeLists.txt @@ -437,6 +437,8 @@ add_arrow_lib(arrow ${ARROW_SHARED_PRIVATE_LINK_LIBS} STATIC_LINK_LIBS ${ARROW_STATIC_LINK_LIBS} + BUNDLE_STATIC_LIBS + ${ARROW_BUNDLE_STATIC_LIBS} SHARED_INSTALL_INTERFACE_LIBS ${ARROW_SHARED_INSTALL_INTERFACE_LIBS} STATIC_INSTALL_INTERFACE_LIBS From 6deb268032e2c4e7b131c164ba538cbda681c69b Mon Sep 17 00:00:00 2001 From: Wes McKinney Date: Thu, 16 Jan 2020 19:13:38 -0600 Subject: [PATCH 2/6] Prototype working on Linux --- cpp/cmake_modules/BuildUtils.cmake | 51 ++++++++++++++++++------------ cpp/src/arrow/CMakeLists.txt | 8 +++++ 2 files changed, 38 insertions(+), 21 deletions(-) diff --git a/cpp/cmake_modules/BuildUtils.cmake b/cpp/cmake_modules/BuildUtils.cmake index 8f855e7adac..d0e4476bbee 100644 --- a/cpp/cmake_modules/BuildUtils.cmake +++ b/cpp/cmake_modules/BuildUtils.cmake @@ -141,43 +141,37 @@ function(create_merged_static_lib output_target) message(SEND_ERROR "Error: unrecognized arguments: ${ARG_UNPARSED_ARGUMENTS}") endif() - set(output_lib_path - ${CMAKE_BINARY_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}${ARG_NAME}${CMAKE_STATIC_LIBRARY_SUFFIX}) + set( + output_lib_path + ${BUILD_OUTPUT_ROOT_DIRECTORY}/${CMAKE_STATIC_LIBRARY_PREFIX}${ARG_NAME}${CMAKE_STATIC_LIBRARY_SUFFIX} + ) set(ar_script_path ${CMAKE_BINARY_DIR}/${ARG_NAME}.ar) - file(WRITE ${ar_script_path}.in "CREATE ${output_lib_path}\n" ) + file(WRITE ${ar_script_path}.in "CREATE ${output_lib_path}\n") - file(APPEND ${ar_script_path}.in "ADDLIB $\n") + file(APPEND ${ar_script_path}.in "ADDLIB $\n") foreach(lib ${ARG_TO_MERGE}) file(APPEND ${ar_script_path}.in "ADDLIB $\n") endforeach() file(APPEND ${ar_script_path}.in "SAVE\nEND\n") - file(GENERATE - OUTPUT ${ar_script_path} - INPUT ${ar_script_path}.in) + file(GENERATE OUTPUT ${ar_script_path} INPUT ${ar_script_path}.in) set(ar_tool ${CMAKE_AR}) - if (CMAKE_INTERPROCEDURAL_OPTIMIZATION) + if(CMAKE_INTERPROCEDURAL_OPTIMIZATION) set(ar_tool ${CMAKE_CXX_COMPILER_AR}) endif() - add_custom_command( - COMMAND ${ar_tool} -M < ${ar_script_path} - OUTPUT ${output_lib_path} - COMMENT "Bundling ${bundled_tgt_name}" - VERBATIM) + add_custom_command(COMMAND ${ar_tool} -M < ${ar_script_path} + OUTPUT ${output_lib_path} + COMMENT "Bundling ${output_lib_path}" + VERBATIM) - add_custom_target(${output_target}_bundling ALL DEPENDS ${output_lib_path}) - add_dependencies(${output_target}_bundling ${ARG_ROOT} ${ARG_TO_MERGE}) + message(STATUS "bundled target: ${output_target}") - add_library(${output_target} STATIC IMPORTED) - set_target_properties(${output_target} - PROPERTIES - IMPORTED_LOCATION ${output_lib_path} - INTERFACE_INCLUDE_DIRECTORIES $) - add_dependencies(${output_target} ${output_target}_bundling) + add_custom_target(${output_target} ALL DEPENDS ${output_lib_path}) + add_dependencies(${output_target} ${ARG_ROOT} ${ARG_TO_MERGE}) endfunction() # \arg OUTPUTS list to append built targets to @@ -394,6 +388,11 @@ function(ADD_ARROW_LIB LIB_NAME) set(LIB_NAME_STATIC ${LIB_NAME}) endif() + if(ARG_BUNDLE_STATIC_LIBS) + set(LIB_NAME_BUNDLED ${LIB_NAME_STATIC}) + set(LIB_NAME_STATIC ${LIB_NAME_STATIC}_unbundled) + endif() + if(ARROW_BUILD_STATIC AND WIN32) target_compile_definitions(${LIB_NAME}_static PUBLIC ARROW_STATIC) endif() @@ -412,6 +411,16 @@ function(ADD_ARROW_LIB LIB_NAME) "$" "$") + if(ARG_BUNDLE_STATIC_LIBS) + create_merged_static_lib(${LIB_NAME}_static_bundled + NAME + ${LIB_NAME_BUNDLED} + ROOT + ${LIB_NAME}_static + TO_MERGE + ${ARG_BUNDLE_STATIC_LIBS}) + endif() + install(TARGETS ${LIB_NAME}_static ${INSTALL_IS_OPTIONAL} EXPORT ${LIB_NAME}_targets RUNTIME DESTINATION ${RUNTIME_INSTALL_DIR} diff --git a/cpp/src/arrow/CMakeLists.txt b/cpp/src/arrow/CMakeLists.txt index 1098149490d..10588204ade 100644 --- a/cpp/src/arrow/CMakeLists.txt +++ b/cpp/src/arrow/CMakeLists.txt @@ -416,6 +416,14 @@ endif() set(ARROW_ALL_SRCS ${ARROW_SRCS} ${ARROW_C_SRCS}) +if(ARROW_JEMALLOC) + set(ARROW_BUNDLE_STATIC_LIBS jemalloc::jemalloc) +endif() + +if(ARROW_MIMALLOC) + set(ARROW_BUNDLE_STATIC_LIBS mimalloc::mimalloc) +endif() + add_arrow_lib(arrow CMAKE_PACKAGE_NAME Arrow From c3c0da1bbc23877690fcaf4d196a3eca3c229722 Mon Sep 17 00:00:00 2001 From: Wes McKinney Date: Fri, 17 Jan 2020 10:58:18 -0600 Subject: [PATCH 3/6] Get bundling working on macOS --- cpp/cmake_modules/BuildUtils.cmake | 67 ++++++++++++++++++++++-------- 1 file changed, 49 insertions(+), 18 deletions(-) diff --git a/cpp/cmake_modules/BuildUtils.cmake b/cpp/cmake_modules/BuildUtils.cmake index d0e4476bbee..ff3dd9dece4 100644 --- a/cpp/cmake_modules/BuildUtils.cmake +++ b/cpp/cmake_modules/BuildUtils.cmake @@ -143,35 +143,66 @@ function(create_merged_static_lib output_target) set( output_lib_path - ${BUILD_OUTPUT_ROOT_DIRECTORY}/${CMAKE_STATIC_LIBRARY_PREFIX}${ARG_NAME}${CMAKE_STATIC_LIBRARY_SUFFIX} + ${BUILD_OUTPUT_ROOT_DIRECTORY}${CMAKE_STATIC_LIBRARY_PREFIX}${ARG_NAME}${CMAKE_STATIC_LIBRARY_SUFFIX} ) - set(ar_script_path ${CMAKE_BINARY_DIR}/${ARG_NAME}.ar) - - file(WRITE ${ar_script_path}.in "CREATE ${output_lib_path}\n") - - file(APPEND ${ar_script_path}.in "ADDLIB $\n") + set(all_library_paths $) foreach(lib ${ARG_TO_MERGE}) - file(APPEND ${ar_script_path}.in "ADDLIB $\n") + list(APPEND all_library_paths $) endforeach() - file(APPEND ${ar_script_path}.in "SAVE\nEND\n") - - file(GENERATE OUTPUT ${ar_script_path} INPUT ${ar_script_path}.in) - - set(ar_tool ${CMAKE_AR}) - if(CMAKE_INTERPROCEDURAL_OPTIMIZATION) - set(ar_tool ${CMAKE_CXX_COMPILER_AR}) + + if(APPLE) + set(BUNDLE_COMMAND "libtool" "-no_warning_for_no_symbols" "-static" "-o" + ${output_lib_path} ${all_library_paths}) + elseif(CMAKE_CXX_COMPILER_ID MATCHES "^(Clang|GNU)$") + set(ar_script_path ${CMAKE_BINARY_DIR}/${ARG_NAME}.ar) + + file(WRITE ${ar_script_path}.in "CREATE ${output_lib_path}\n") + file(APPEND ${ar_script_path}.in "ADDLIB $\n") + + foreach(lib ${ARG_TO_MERGE}) + file(APPEND ${ar_script_path}.in "ADDLIB $\n") + endforeach() + + file(APPEND ${ar_script_path}.in "SAVE\nEND\n") + file(GENERATE OUTPUT ${ar_script_path} INPUT ${ar_script_path}.in) + set(ar_tool ${CMAKE_AR}) + + if(CMAKE_INTERPROCEDURAL_OPTIMIZATION) + set(ar_tool ${CMAKE_CXX_COMPILER_AR}) + endif() + + set(BUNDLE_COMMAND ${ar_tool} -M < ${ar_script_path}) + + elseif(MSVC) + if(NOT CMAKE_LIBTOOL) + find_program(lib_tool lib HINTS "${CMAKE_CXX_COMPILER}/..") + if("${lib_tool}" STREQUAL "lib_tool-NOTFOUND") + message(FATAL_ERROR "Cannot locate libtool to bundle libraries") + endif() + else() + set(${lib_tool} ${CMAKE_LIBTOOL}) + endif() + set(BUNDLE_TOOL ${lib_tool}) + set(BUNDLE_COMMAND ${BUNDLE_TOOL} /NOLOGO /OUT:${output_lib_path} ${all_library_paths}) + else() + message(FATAL_ERROR "Unknown bundle scenario!") endif() - - add_custom_command(COMMAND ${ar_tool} -M < ${ar_script_path} + + add_custom_command(COMMAND ${BUNDLE_COMMAND} OUTPUT ${output_lib_path} COMMENT "Bundling ${output_lib_path}" VERBATIM) - message(STATUS "bundled target: ${output_target}") - add_custom_target(${output_target} ALL DEPENDS ${output_lib_path}) add_dependencies(${output_target} ${ARG_ROOT} ${ARG_TO_MERGE}) + + add_library(${output_target}_lib STATIC IMPORTED) + set_target_properties(${output_target}_lib + PROPERTIES + IMPORTED_LOCATION ${bundled_tgt_full_name} + INTERFACE_INCLUDE_DIRECTORIES $) + add_dependencies(${bundled_tgt_name} ${tgt_name}) endfunction() # \arg OUTPUTS list to append built targets to From ee5f086aced8aebaee50602e5bb285a58e35b0df Mon Sep 17 00:00:00 2001 From: Wes McKinney Date: Fri, 17 Jan 2020 11:16:37 -0600 Subject: [PATCH 4/6] Install bundled static library --- cpp/cmake_modules/BuildUtils.cmake | 46 +++++++++++++++--------------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/cpp/cmake_modules/BuildUtils.cmake b/cpp/cmake_modules/BuildUtils.cmake index ff3dd9dece4..8c211779cd1 100644 --- a/cpp/cmake_modules/BuildUtils.cmake +++ b/cpp/cmake_modules/BuildUtils.cmake @@ -150,10 +150,15 @@ function(create_merged_static_lib output_target) foreach(lib ${ARG_TO_MERGE}) list(APPEND all_library_paths $) endforeach() - + if(APPLE) - set(BUNDLE_COMMAND "libtool" "-no_warning_for_no_symbols" "-static" "-o" - ${output_lib_path} ${all_library_paths}) + set(BUNDLE_COMMAND + "libtool" + "-no_warning_for_no_symbols" + "-static" + "-o" + ${output_lib_path} + ${all_library_paths}) elseif(CMAKE_CXX_COMPILER_ID MATCHES "^(Clang|GNU)$") set(ar_script_path ${CMAKE_BINARY_DIR}/${ARG_NAME}.ar) @@ -171,24 +176,25 @@ function(create_merged_static_lib output_target) if(CMAKE_INTERPROCEDURAL_OPTIMIZATION) set(ar_tool ${CMAKE_CXX_COMPILER_AR}) endif() - + set(BUNDLE_COMMAND ${ar_tool} -M < ${ar_script_path}) - elseif(MSVC) - if(NOT CMAKE_LIBTOOL) - find_program(lib_tool lib HINTS "${CMAKE_CXX_COMPILER}/..") - if("${lib_tool}" STREQUAL "lib_tool-NOTFOUND") - message(FATAL_ERROR "Cannot locate libtool to bundle libraries") - endif() - else() - set(${lib_tool} ${CMAKE_LIBTOOL}) + elseif(MSVC) + if(NOT CMAKE_LIBTOOL) + find_program(lib_tool lib HINTS "${CMAKE_CXX_COMPILER}/..") + if("${lib_tool}" STREQUAL "lib_tool-NOTFOUND") + message(FATAL_ERROR "Cannot locate libtool to bundle libraries") endif() - set(BUNDLE_TOOL ${lib_tool}) - set(BUNDLE_COMMAND ${BUNDLE_TOOL} /NOLOGO /OUT:${output_lib_path} ${all_library_paths}) else() - message(FATAL_ERROR "Unknown bundle scenario!") + set(${lib_tool} ${CMAKE_LIBTOOL}) + endif() + set(BUNDLE_TOOL ${lib_tool}) + set(BUNDLE_COMMAND ${BUNDLE_TOOL} /NOLOGO /OUT:${output_lib_path} + ${all_library_paths}) + else() + message(FATAL_ERROR "Unknown bundle scenario!") endif() - + add_custom_command(COMMAND ${BUNDLE_COMMAND} OUTPUT ${output_lib_path} COMMENT "Bundling ${output_lib_path}" @@ -196,13 +202,7 @@ function(create_merged_static_lib output_target) add_custom_target(${output_target} ALL DEPENDS ${output_lib_path}) add_dependencies(${output_target} ${ARG_ROOT} ${ARG_TO_MERGE}) - - add_library(${output_target}_lib STATIC IMPORTED) - set_target_properties(${output_target}_lib - PROPERTIES - IMPORTED_LOCATION ${bundled_tgt_full_name} - INTERFACE_INCLUDE_DIRECTORIES $) - add_dependencies(${bundled_tgt_name} ${tgt_name}) + install(FILES ${output_lib_path} DESTINATION ${CMAKE_INSTALL_LIBDIR}) endfunction() # \arg OUTPUTS list to append built targets to From b57ee519b7d7b8fc2a1c5c573836d90aee843358 Mon Sep 17 00:00:00 2001 From: Wes McKinney Date: Fri, 17 Jan 2020 11:19:01 -0600 Subject: [PATCH 5/6] Add license --- LICENSE.txt | 28 ++++++++++++++++++++++++++++ cpp/cmake_modules/BuildUtils.cmake | 2 ++ 2 files changed, 30 insertions(+) diff --git a/LICENSE.txt b/LICENSE.txt index a078f5e6b80..0eb703f1030 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -1986,3 +1986,31 @@ SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +-------------------------------------------------------------------------------- + +The file cpp/cmake_modules/BuildUtils.cmake contains code from + +https://gist.github.com/cristianadam/ef920342939a89fae3e8a85ca9459b49 + +which is made available under the MIT license + +Copyright (c) 2019 Cristian Adam + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/cpp/cmake_modules/BuildUtils.cmake b/cpp/cmake_modules/BuildUtils.cmake index 8c211779cd1..2a953fb0cb6 100644 --- a/cpp/cmake_modules/BuildUtils.cmake +++ b/cpp/cmake_modules/BuildUtils.cmake @@ -128,6 +128,8 @@ function(REUSE_PRECOMPILED_HEADER_LIB TARGET_NAME LIB_NAME) endif() endfunction() +# Based on MIT-licensed +# https://gist.github.com/cristianadam/ef920342939a89fae3e8a85ca9459b49 function(create_merged_static_lib output_target) set(options) set(one_value_args NAME ROOT) From f7783ba8e30931002f0bd22204c264262c60feac Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Wed, 22 Apr 2020 18:03:19 +0200 Subject: [PATCH 6/6] Apply Kou's suggestion and remove duplicate code --- cpp/CMakeLists.txt | 4 ---- cpp/src/arrow/CMakeLists.txt | 8 ++++++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index b9aedaf18ca..da22f2aa66d 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -626,10 +626,6 @@ endif() # # TODO: Also rework how these libs work -set(ARROW_LINK_LIBS) -set(ARROW_SHARED_INSTALL_INTERFACE_LIBS) -set(ARROW_STATIC_INSTALL_INTERFACE_LIBS) - # Libraries to link statically with libarrow.so set(ARROW_LINK_LIBS) set(ARROW_STATIC_LINK_LIBS) diff --git a/cpp/src/arrow/CMakeLists.txt b/cpp/src/arrow/CMakeLists.txt index 10588204ade..52ad223e5eb 100644 --- a/cpp/src/arrow/CMakeLists.txt +++ b/cpp/src/arrow/CMakeLists.txt @@ -416,12 +416,16 @@ endif() set(ARROW_ALL_SRCS ${ARROW_SRCS} ${ARROW_C_SRCS}) +# We bundle some private static dependencies into the Arrow static library, +# for convenience. +set(ARROW_BUNDLE_STATIC_LIBS) + if(ARROW_JEMALLOC) - set(ARROW_BUNDLE_STATIC_LIBS jemalloc::jemalloc) + list(APPEND ARROW_BUNDLE_STATIC_LIBS jemalloc::jemalloc) endif() if(ARROW_MIMALLOC) - set(ARROW_BUNDLE_STATIC_LIBS mimalloc::mimalloc) + list(APPEND ARROW_BUNDLE_STATIC_LIBS mimalloc::mimalloc) endif() add_arrow_lib(arrow