diff --git a/LICENSE.txt b/LICENSE.txt index 13faeb2ee8e..673b146c807 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -2179,3 +2179,31 @@ IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +-------------------------------------------------------------------------------- + +The file cpp/cmake_modules/BuildUtils.cmake contains code from + +https://gist.github.com/cristianadam/ef920342939a89fae3e8a85ca9459b49 + +which is made available under the MIT license + +Copyright (c) 2019 Cristian Adam + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/ci/scripts/PKGBUILD b/ci/scripts/PKGBUILD index ca29f0d84db..4bf08b7151d 100644 --- a/ci/scripts/PKGBUILD +++ b/ci/scripts/PKGBUILD @@ -90,6 +90,7 @@ build() { -DARROW_HDFS=OFF \ -DARROW_JEMALLOC=OFF \ -DARROW_JSON=ON \ + -DARROW_MIMALLOC=ON \ -DARROW_PACKAGE_PREFIX="${MINGW_PREFIX}" \ -DARROW_PARQUET=ON \ -DARROW_USE_GLOG=OFF \ diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 32a149492a0..3629e57bdff 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -641,11 +641,6 @@ endif() # Linker and Dependencies # -# TODO: Also rework how these libs work -set(ARROW_LINK_LIBS) -set(ARROW_SHARED_INSTALL_INTERFACE_LIBS) -set(ARROW_STATIC_INSTALL_INTERFACE_LIBS) - # Libraries to link statically with libarrow.so set(ARROW_LINK_LIBS) set(ARROW_STATIC_LINK_LIBS) @@ -655,7 +650,6 @@ if(ARROW_USE_OPENSSL) set(ARROW_OPENSSL_LIBS OpenSSL::Crypto OpenSSL::SSL) list(APPEND ARROW_LINK_LIBS ${ARROW_OPENSSL_LIBS}) list(APPEND ARROW_STATIC_LINK_LIBS ${ARROW_OPENSSL_LIBS}) - list(APPEND ARROW_STATIC_INSTALL_INTERFACE_LIBS ${ARROW_OPENSSL_LIBS}) endif() if(ARROW_WITH_BROTLI) @@ -663,45 +657,36 @@ if(ARROW_WITH_BROTLI) set(ARROW_BROTLI_LIBS Brotli::brotlienc Brotli::brotlidec Brotli::brotlicommon) list(APPEND ARROW_LINK_LIBS ${ARROW_BROTLI_LIBS}) list(APPEND ARROW_STATIC_LINK_LIBS ${ARROW_BROTLI_LIBS}) - list(APPEND ARROW_STATIC_INSTALL_INTERFACE_LIBS ${ARROW_BROTLI_LIBS}) endif() if(ARROW_WITH_BZ2) list(APPEND ARROW_STATIC_LINK_LIBS BZip2::BZip2) - list(APPEND ARROW_STATIC_INSTALL_INTERFACE_LIBS BZip2::BZip2) endif() if(ARROW_WITH_LZ4) list(APPEND ARROW_STATIC_LINK_LIBS LZ4::lz4) - list(APPEND ARROW_STATIC_INSTALL_INTERFACE_LIBS LZ4::lz4) endif() if(ARROW_WITH_SNAPPY) list(APPEND ARROW_STATIC_LINK_LIBS Snappy::snappy) - list(APPEND ARROW_STATIC_INSTALL_INTERFACE_LIBS Snappy::snappy) endif() if(ARROW_WITH_ZLIB) list(APPEND ARROW_STATIC_LINK_LIBS ZLIB::ZLIB) - list(APPEND ARROW_STATIC_INSTALL_INTERFACE_LIBS ZLIB::ZLIB) endif() if(ARROW_WITH_ZSTD) list(APPEND ARROW_STATIC_LINK_LIBS ${ARROW_ZSTD_LIBZSTD}) - list(APPEND ARROW_STATIC_INSTALL_INTERFACE_LIBS ${ARROW_ZSTD_LIBZSTD}) endif() if(ARROW_ORC) list(APPEND ARROW_LINK_LIBS orc::liborc ${ARROW_PROTOBUF_LIBPROTOBUF}) list(APPEND ARROW_STATIC_LINK_LIBS orc::liborc ${ARROW_PROTOBUF_LIBPROTOBUF}) - list(APPEND ARROW_STATIC_INSTALL_INTERFACE_LIBS orc::liborc - ${ARROW_PROTOBUF_LIBPROTOBUF}) endif() if(ARROW_USE_GLOG) list(APPEND ARROW_LINK_LIBS glog::glog) list(APPEND ARROW_STATIC_LINK_LIBS glog::glog) - list(APPEND ARROW_STATIC_INSTALL_INTERFACE_LIBS glog::glog) add_definitions("-DARROW_USE_GLOG") endif() @@ -712,7 +697,6 @@ endif() if(ARROW_WITH_UTF8PROC) list(APPEND ARROW_LINK_LIBS utf8proc::utf8proc) list(APPEND ARROW_STATIC_LINK_LIBS utf8proc::utf8proc) - list(APPEND ARROW_STATIC_INSTALL_INTERFACE_LIBS utf8proc::utf8proc) endif() add_custom_target(arrow_dependencies) @@ -807,19 +791,24 @@ if(ARROW_BUILD_BENCHMARKS) endif() endif() -set(ARROW_SYSTEM_LINK_LIBS) - if(ARROW_JEMALLOC) add_definitions(-DARROW_JEMALLOC) add_definitions(-DARROW_JEMALLOC_INCLUDE_DIR=${JEMALLOC_INCLUDE_DIR}) - list(APPEND ARROW_SYSTEM_LINK_LIBS jemalloc::jemalloc) + list(APPEND ARROW_LINK_LIBS jemalloc::jemalloc) + list(APPEND ARROW_STATIC_LINK_LIBS jemalloc::jemalloc) endif() if(ARROW_MIMALLOC) add_definitions(-DARROW_MIMALLOC) - list(APPEND ARROW_SYSTEM_LINK_LIBS mimalloc::mimalloc) + list(APPEND ARROW_LINK_LIBS mimalloc::mimalloc) + list(APPEND ARROW_STATIC_LINK_LIBS mimalloc::mimalloc) endif() +# ---------------------------------------------------------------------- +# Handle platform-related libraries like -pthread + +set(ARROW_SYSTEM_LINK_LIBS) + if(THREADS_FOUND) list(APPEND ARROW_SYSTEM_LINK_LIBS Threads::Threads) endif() @@ -836,7 +825,6 @@ endif() list(APPEND ARROW_LINK_LIBS ${ARROW_SYSTEM_LINK_LIBS}) list(APPEND ARROW_STATIC_LINK_LIBS ${ARROW_SYSTEM_LINK_LIBS}) -list(APPEND ARROW_STATIC_INSTALL_INTERFACE_LIBS ${ARROW_SYSTEM_LINK_LIBS}) # # Subdirectories diff --git a/cpp/cmake_modules/BuildUtils.cmake b/cpp/cmake_modules/BuildUtils.cmake index 7adc5624bd4..eb10ebeeb1c 100644 --- a/cpp/cmake_modules/BuildUtils.cmake +++ b/cpp/cmake_modules/BuildUtils.cmake @@ -128,6 +128,89 @@ function(REUSE_PRECOMPILED_HEADER_LIB TARGET_NAME LIB_NAME) endif() endfunction() +# Based on MIT-licensed +# https://gist.github.com/cristianadam/ef920342939a89fae3e8a85ca9459b49 +function(create_merged_static_lib output_target) + set(options) + set(one_value_args NAME ROOT) + set(multi_value_args TO_MERGE) + cmake_parse_arguments(ARG + "${options}" + "${one_value_args}" + "${multi_value_args}" + ${ARGN}) + if(ARG_UNPARSED_ARGUMENTS) + message(SEND_ERROR "Error: unrecognized arguments: ${ARG_UNPARSED_ARGUMENTS}") + endif() + + set( + output_lib_path + ${BUILD_OUTPUT_ROOT_DIRECTORY}${CMAKE_STATIC_LIBRARY_PREFIX}${ARG_NAME}${CMAKE_STATIC_LIBRARY_SUFFIX} + ) + + set(all_library_paths $) + foreach(lib ${ARG_TO_MERGE}) + list(APPEND all_library_paths $) + endforeach() + + if(APPLE) + set(BUNDLE_COMMAND + "libtool" + "-no_warning_for_no_symbols" + "-static" + "-o" + ${output_lib_path} + ${all_library_paths}) + elseif(CMAKE_CXX_COMPILER_ID MATCHES "^(Clang|GNU)$") + set(ar_script_path ${CMAKE_BINARY_DIR}/${ARG_NAME}.ar) + + file(WRITE ${ar_script_path}.in "CREATE ${output_lib_path}\n") + file(APPEND ${ar_script_path}.in "ADDLIB $\n") + + foreach(lib ${ARG_TO_MERGE}) + file(APPEND ${ar_script_path}.in "ADDLIB $\n") + endforeach() + + file(APPEND ${ar_script_path}.in "SAVE\nEND\n") + file(GENERATE OUTPUT ${ar_script_path} INPUT ${ar_script_path}.in) + set(ar_tool ${CMAKE_AR}) + + if(CMAKE_INTERPROCEDURAL_OPTIMIZATION) + set(ar_tool ${CMAKE_CXX_COMPILER_AR}) + endif() + + set(BUNDLE_COMMAND ${ar_tool} -M < ${ar_script_path}) + + elseif(MSVC) + if(NOT CMAKE_LIBTOOL) + find_program(lib_tool lib HINTS "${CMAKE_CXX_COMPILER}/..") + if("${lib_tool}" STREQUAL "lib_tool-NOTFOUND") + message(FATAL_ERROR "Cannot locate libtool to bundle libraries") + endif() + else() + set(${lib_tool} ${CMAKE_LIBTOOL}) + endif() + set(BUNDLE_TOOL ${lib_tool}) + set(BUNDLE_COMMAND ${BUNDLE_TOOL} /NOLOGO /OUT:${output_lib_path} + ${all_library_paths}) + else() + message(FATAL_ERROR "Unknown bundle scenario!") + endif() + + add_custom_command(COMMAND ${BUNDLE_COMMAND} + OUTPUT ${output_lib_path} + COMMENT "Bundling ${output_lib_path}" + VERBATIM) + + message( + STATUS "Creating bundled static library target ${output_target} at ${output_lib_path}" + ) + + add_custom_target(${output_target} ALL DEPENDS ${output_lib_path}) + add_dependencies(${output_target} ${ARG_ROOT} ${ARG_TO_MERGE}) + install(FILES ${output_lib_path} DESTINATION ${CMAKE_INSTALL_LIBDIR}) +endfunction() + # \arg OUTPUTS list to append built targets to function(ADD_ARROW_LIB LIB_NAME) set(options BUILD_SHARED BUILD_STATIC) @@ -351,14 +434,14 @@ function(ADD_ARROW_LIB LIB_NAME) ${LIB_NAME_STATIC}) if(ARG_STATIC_INSTALL_INTERFACE_LIBS) - set(INTERFACE_LIBS ${ARG_STATIC_INSTALL_INTERFACE_LIBS}) - else() - set(INTERFACE_LIBS ${ARG_STATIC_LINK_LIBS}) + target_link_libraries(${LIB_NAME}_static LINK_PUBLIC + "$") endif() - target_link_libraries(${LIB_NAME}_static LINK_PUBLIC - "$" - "$") + if(ARG_STATIC_LINK_LIBS) + target_link_libraries(${LIB_NAME}_static LINK_PRIVATE + "$") + endif() install(TARGETS ${LIB_NAME}_static ${INSTALL_IS_OPTIONAL} EXPORT ${LIB_NAME}_targets @@ -832,3 +915,12 @@ function(ARROW_INSTALL_CMAKE_FIND_MODULE MODULE) install(FILES "${ARROW_SOURCE_DIR}/cmake_modules/Find${MODULE}.cmake" DESTINATION "${ARROW_CMAKE_INSTALL_DIR}") endfunction() + +# Implementations of lisp "car" and "cdr" functions +macro(ARROW_CAR var) + set(${var} ${ARGV1}) +endmacro() + +macro(ARROW_CDR var rest) + set(${var} ${ARGN}) +endmacro() diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index 960d31359c2..3c229d270c2 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -23,6 +23,12 @@ add_custom_target(toolchain) add_custom_target(toolchain-benchmarks) add_custom_target(toolchain-tests) +# Accumulate all bundled targets and we will splice them together later as +# libarrow_dependencies.a so that third party libraries have something usable +# to create statically-linked builds with some BUNDLED dependencies, including +# allocators like jemalloc and mimalloc +set(ARROW_BUNDLED_STATIC_LIBS) + # ---------------------------------------------------------------------- # Toolchain linkage options @@ -685,6 +691,9 @@ macro(build_boost) set(Boost_INCLUDE_DIRS "${BOOST_INCLUDE_DIR}") add_dependencies(toolchain boost_ep) set(BOOST_VENDORED TRUE) + + list(APPEND ARROW_BUNDLED_STATIC_LIBS boost_system_static boost_filesystem_static + boost_regex_static) endmacro() if(ARROW_FLIGHT AND ARROW_BUILD_TESTS) @@ -833,6 +842,8 @@ macro(build_snappy) "${SNAPPY_PREFIX}/include") add_dependencies(toolchain snappy_ep) add_dependencies(Snappy::snappy snappy_ep) + + list(APPEND ARROW_BUNDLED_STATIC_LIBS Snappy::snappy) endmacro() if(ARROW_WITH_SNAPPY) @@ -919,6 +930,9 @@ macro(build_brotli) PROPERTIES IMPORTED_LOCATION "${BROTLI_STATIC_LIBRARY_DEC}" INTERFACE_INCLUDE_DIRECTORIES "${BROTLI_INCLUDE_DIR}") add_dependencies(Brotli::brotlidec brotli_ep) + + list(APPEND ARROW_BUNDLED_STATIC_LIBS Brotli::brotlicommon Brotli::brotlienc + Brotli::brotlidec) endmacro() if(ARROW_WITH_BROTLI) @@ -1036,6 +1050,8 @@ macro(build_glog) PROPERTIES IMPORTED_LOCATION "${GLOG_STATIC_LIB}" INTERFACE_INCLUDE_DIRECTORIES "${GLOG_INCLUDE_DIR}") add_dependencies(glog::glog glog_ep) + + list(APPEND ARROW_BUNDLED_STATIC_LIBS glog::glog) endmacro() if(ARROW_USE_GLOG) @@ -1105,6 +1121,8 @@ macro(build_gflags) set(GFLAGS_LIBRARIES ${GFLAGS_LIBRARY}) set(GFLAGS_VENDORED TRUE) + + list(APPEND ARROW_BUNDLED_STATIC_LIBS gflags_static) endmacro() if(ARROW_NEED_GFLAGS) @@ -1212,6 +1230,8 @@ macro(build_thrift) add_dependencies(toolchain thrift_ep) add_dependencies(thrift::thrift thrift_ep) set(THRIFT_VERSION ${ARROW_THRIFT_BUILD_VERSION}) + + list(APPEND ARROW_BUNDLED_STATIC_LIBS thrift::thrift) endmacro() if(ARROW_WITH_THRIFT) @@ -1304,6 +1324,8 @@ macro(build_protobuf) add_dependencies(toolchain protobuf_ep) add_dependencies(arrow::protobuf::libprotobuf protobuf_ep) + + list(APPEND ARROW_BUNDLED_STATIC_LIBS arrow::protobuf::libprotobuf) endmacro() if(ARROW_WITH_PROTOBUF) @@ -1406,6 +1428,7 @@ if(ARROW_JEMALLOC) "--with-jemalloc-prefix=je_arrow_" "--with-private-namespace=je_arrow_private_" "--without-export" + "--disable-shared" # Don't override operator new() "--disable-cxx" "--disable-libdl" # See https://github.com/jemalloc/jemalloc/issues/1237 @@ -1440,6 +1463,8 @@ if(ARROW_JEMALLOC) INTERFACE_INCLUDE_DIRECTORIES "${CMAKE_CURRENT_BINARY_DIR}/jemalloc_ep-prefix/src") add_dependencies(jemalloc::jemalloc jemalloc_ep) + + list(APPEND ARROW_BUNDLED_STATIC_LIBS jemalloc::jemalloc) endif() # ---------------------------------------------------------------------- @@ -1470,6 +1495,7 @@ if(ARROW_MIMALLOC) -DMI_OVERRIDE=OFF -DMI_LOCAL_DYNAMIC_TLS=ON -DMI_BUILD_OBJECT=OFF + -DMI_BUILD_SHARED=OFF -DMI_BUILD_TESTS=OFF) externalproject_add(mimalloc_ep @@ -1490,6 +1516,8 @@ if(ARROW_MIMALLOC) "${MIMALLOC_INCLUDE_DIR}") add_dependencies(mimalloc::mimalloc mimalloc_ep) add_dependencies(toolchain mimalloc_ep) + + list(APPEND ARROW_BUNDLED_STATIC_LIBS mimalloc::mimalloc) endif() # ---------------------------------------------------------------------- @@ -1821,6 +1849,8 @@ macro(build_zlib) add_dependencies(toolchain zlib_ep) add_dependencies(ZLIB::ZLIB zlib_ep) + + list(APPEND ARROW_BUNDLED_STATIC_LIBS ZLIB::ZLIB) endmacro() if(ARROW_WITH_ZLIB) @@ -1883,6 +1913,8 @@ macro(build_lz4) INTERFACE_INCLUDE_DIRECTORIES "${LZ4_PREFIX}/include") add_dependencies(toolchain lz4_ep) add_dependencies(LZ4::lz4 lz4_ep) + + list(APPEND ARROW_BUNDLED_STATIC_LIBS LZ4::lz4) endmacro() if(ARROW_WITH_LZ4) @@ -1945,6 +1977,8 @@ macro(build_zstd) add_dependencies(toolchain zstd_ep) add_dependencies(zstd::libzstd zstd_ep) + + list(APPEND ARROW_BUNDLED_STATIC_LIBS zstd::libzstd) endmacro() if(ARROW_WITH_ZSTD) @@ -1998,6 +2032,8 @@ macro(build_re2) add_dependencies(toolchain re2_ep) add_dependencies(RE2::re2 re2_ep) + + list(APPEND ARROW_BUNDLED_STATIC_LIBS RE2::re2) endmacro() if(ARROW_GANDIVA) @@ -2038,6 +2074,8 @@ macro(build_bzip2) add_dependencies(toolchain bzip2_ep) add_dependencies(BZip2::BZip2 bzip2_ep) + + list(APPEND ARROW_BUNDLED_STATIC_LIBS BZip2::BZip2) endmacro() if(ARROW_WITH_BZ2) @@ -2090,6 +2128,8 @@ macro(build_utf8proc) add_dependencies(toolchain utf8proc_ep) add_dependencies(utf8proc::utf8proc utf8proc_ep) + + list(APPEND ARROW_BUNDLED_STATIC_LIBS utf8proc::utf8proc) endmacro() if(ARROW_WITH_UTF8PROC) @@ -2147,6 +2187,8 @@ macro(build_cares) INTERFACE_INCLUDE_DIRECTORIES "${CARES_INCLUDE_DIR}") set(CARES_VENDORED TRUE) + + list(APPEND ARROW_BUNDLED_STATIC_LIBS c-ares::cares) endmacro() if(ARROW_WITH_GRPC) @@ -2369,6 +2411,39 @@ macro(build_grpc) add_dependencies(gRPC::grpc++ grpc_ep) add_dependencies(gRPC::grpc_cpp_plugin grpc_ep) set(GRPC_VENDORED TRUE) + + # ar -M rejects with the "libgrpc++.a" filename because "+" is a line + # continuation character in these scripts, so we have to create a copy of the + # static lib that we will bundle later + + set( + GRPC_STATIC_LIBRARY_GRPCPP_FOR_AR + "${CMAKE_CURRENT_BINARY_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}grpcpp${CMAKE_STATIC_LIBRARY_SUFFIX}" + ) + add_custom_command(OUTPUT ${GRPC_STATIC_LIBRARY_GRPCPP_FOR_AR} + COMMAND ${CMAKE_COMMAND} + -E + copy + $ + ${GRPC_STATIC_LIBRARY_GRPCPP_FOR_AR} + DEPENDS grpc_ep) + add_library(gRPC::grpcpp_for_bundling STATIC IMPORTED) + set_target_properties(gRPC::grpcpp_for_bundling + PROPERTIES IMPORTED_LOCATION + "${GRPC_STATIC_LIBRARY_GRPCPP_FOR_AR}") + + set_source_files_properties("${GRPC_STATIC_LIBRARY_GRPCPP_FOR_AR}" PROPERTIES GENERATED + TRUE) + add_custom_target(grpc_copy_grpc++ ALL DEPENDS "${GRPC_STATIC_LIBRARY_GRPCPP_FOR_AR}") + add_dependencies(gRPC::grpcpp_for_bundling grpc_copy_grpc++) + + list(APPEND ARROW_BUNDLED_STATIC_LIBS + ${ABSL_LIBRARIES} + gRPC::upb + gRPC::gpr + gRPC::grpc + gRPC::address_sorting + gRPC::grpcpp_for_bundling) endmacro() if(ARROW_WITH_GRPC) @@ -2513,6 +2588,8 @@ macro(build_orc) add_dependencies(toolchain orc_ep) add_dependencies(orc::liborc orc_ep) + + list(APPEND ARROW_BUNDLED_STATIC_LIBS orc::liborc) endmacro() if(ARROW_ORC) @@ -2573,6 +2650,8 @@ macro(build_awssdk) add_dependencies(toolchain awssdk_ep) set(AWSSDK_LINK_LIBRARIES ${AWSSDK_SHARED_LIBS}) set(AWSSDK_VENDORED TRUE) + + # AWSSDK is shared-only build endmacro() if(ARROW_S3) @@ -2605,6 +2684,8 @@ if(ARROW_S3) endif() endif() +message(STATUS "All bundled static libraries: ${ARROW_BUNDLED_STATIC_LIBS}") + # Write out the package configurations. configure_file("src/arrow/util/config.h.cmake" "src/arrow/util/config.h") diff --git a/cpp/examples/minimal_build/CMakeLists.txt b/cpp/examples/minimal_build/CMakeLists.txt index 4b29f6284b8..9fc20c70fe0 100644 --- a/cpp/examples/minimal_build/CMakeLists.txt +++ b/cpp/examples/minimal_build/CMakeLists.txt @@ -19,10 +19,22 @@ cmake_minimum_required(VERSION 3.0) project(ArrowMinimalExample) +option(ARROW_LINK_SHARED "Link to the Arrow shared library" ON) + find_package(Arrow REQUIRED) +set(CMAKE_CXX_STANDARD 11) +set(CMAKE_BUILD_TYPE Release) + message(STATUS "Arrow version: ${ARROW_VERSION}") message(STATUS "Arrow SO version: ${ARROW_FULL_SO_VERSION}") add_executable(arrow_example example.cc) -target_link_libraries(arrow_example PRIVATE arrow_shared) + +if (ARROW_LINK_SHARED) + target_link_libraries(arrow_example PRIVATE arrow_shared) +else() + set(THREADS_PREFER_PTHREAD_FLAG ON) + find_package(Threads REQUIRED) + target_link_libraries(arrow_example PRIVATE arrow_static Threads::Threads) +endif() diff --git a/cpp/examples/minimal_build/README.md b/cpp/examples/minimal_build/README.md index 4c12123b227..b79d20c2a0e 100644 --- a/cpp/examples/minimal_build/README.md +++ b/cpp/examples/minimal_build/README.md @@ -49,3 +49,33 @@ docker run -it -v $PWD:/io -v $ARROW_ROOT:/arrow arrow_cpp_minimal /io/run.sh Note that this example mounts two volumes inside the Docker image: * `/arrow` points to the Arrow source tree * `/io` points to this example directory + +## Statically-linked builds + +We've provided an example build configuration here with CMake to show how to +create a statically-linked executable . + +To run it on Linux, you can use the above Docker image: + +```bash +docker run -it -v $PWD:/io -v $ARROW_ROOT:/arrow arrow_cpp_minimal /io/run_static.sh +``` + +On macOS, you can use the `run_static.sh` but you must set some environment +variables to point the script to your Arrow checkout, for example: + +```bash +export ARROW_DIR=path/to/arrow-clone +export EXAMPLE_DIR=$ARROW_DIR/cpp/examples/minimal_build +export ARROW_BUILD_DIR=`pwd`/arrow-build +export EXAMPLE_BUILD_DIR=`pwd`/example + +./run_static.sh +``` + +On Windows, you can run `run_static.bat` from the command prompt with Visual +Studio's command line tools enabled and CMake and ninja build in the path: + +``` +call run_static.bat +``` \ No newline at end of file diff --git a/cpp/examples/minimal_build/run_static.bat b/cpp/examples/minimal_build/run_static.bat new file mode 100644 index 00000000000..bbc7ff8f750 --- /dev/null +++ b/cpp/examples/minimal_build/run_static.bat @@ -0,0 +1,88 @@ +@rem Licensed to the Apache Software Foundation (ASF) under one +@rem or more contributor license agreements. See the NOTICE file +@rem distributed with this work for additional information +@rem regarding copyright ownership. The ASF licenses this file +@rem to you under the Apache License, Version 2.0 (the +@rem "License"); you may not use this file except in compliance +@rem with the License. You may obtain a copy of the License at +@rem +@rem http://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, +@rem software distributed under the License is distributed on an +@rem "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +@rem KIND, either express or implied. See the License for the +@rem specific language governing permissions and limitations +@rem under the License. + +@echo on + +@rem clean up prior attempts +if exist "arrow-build" rd arrow-build /s /q +if exist "dist" rd dist /s /q +if exist "example" rd example /s /q + +echo +echo "==" +echo "== Building Arrow C++ library" +echo "==" +echo + +set INSTALL_PREFIX=%cd%\dist + +mkdir arrow-build +pushd arrow-build + +@rem bzip2_ep fails with this method + +cmake ..\..\.. ^ + -GNinja ^ + -DCMAKE_INSTALL_PREFIX=%INSTALL_PREFIX% ^ + -DARROW_DEPENDENCY_SOURCE=BUNDLED ^ + -DARROW_BUILD_SHARED=OFF ^ + -DARROW_BUILD_STATIC=ON ^ + -DARROW_COMPUTE=ON ^ + -DARROW_CSV=ON ^ + -DARROW_DATASET=ON ^ + -DARROW_FILESYSTEM=ON ^ + -DARROW_HDFS=ON ^ + -DARROW_JSON=ON ^ + -DARROW_MIMALLOC=ON ^ + -DARROW_ORC=ON ^ + -DARROW_PARQUET=ON ^ + -DARROW_PLASMA=ON ^ + -DARROW_WITH_BROTLI=ON ^ + -DARROW_WITH_BZ2=OFF ^ + -DARROW_WITH_LZ4=ON ^ + -DARROW_WITH_SNAPPY=ON ^ + -DARROW_WITH_ZLIB=ON ^ + -DARROW_WITH_ZSTD=ON + +ninja install + +popd + +echo +echo "==" +echo "== Building example project using Arrow C++ library" +echo "==" +echo + +mkdir example +pushd example + +cmake .. ^ + -GNinja ^ + -DCMAKE_PREFIX_PATH="%INSTALL_PREFIX%" ^ + -DARROW_LINK_SHARED=OFF +ninja + +popd + +echo +echo "==" +echo "== Running example project" +echo "==" +echo + +call example\arrow_example.exe diff --git a/cpp/examples/minimal_build/run_static.sh b/cpp/examples/minimal_build/run_static.sh new file mode 100755 index 00000000000..941a29f3186 --- /dev/null +++ b/cpp/examples/minimal_build/run_static.sh @@ -0,0 +1,86 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -e + +: ${ARROW_DIR:=/arrow} +: ${EXAMPLE_DIR:=/io} +: ${ARROW_BUILD_DIR:=/build/arrow} +: ${EXAMPLE_BUILD_DIR:=/build/example} + +echo +echo "==" +echo "== Building Arrow C++ library" +echo "==" +echo + +mkdir -p $ARROW_BUILD_DIR +pushd $ARROW_BUILD_DIR + +NPROC=$(nproc) + +cmake $ARROW_DIR/cpp \ + -DARROW_DEPENDENCY_SOURCE=BUNDLED \ + -DARROW_BUILD_SHARED=OFF \ + -DARROW_BUILD_STATIC=ON \ + -DARROW_COMPUTE=ON \ + -DARROW_CSV=ON \ + -DARROW_DATASET=ON \ + -DARROW_FILESYSTEM=ON \ + -DARROW_HDFS=ON \ + -DARROW_JEMALLOC=ON \ + -DARROW_JSON=ON \ + -DARROW_ORC=ON \ + -DARROW_PARQUET=ON \ + -DARROW_PLASMA=ON \ + -DARROW_WITH_BROTLI=ON \ + -DARROW_WITH_BZ2=ON \ + -DARROW_WITH_LZ4=ON \ + -DARROW_WITH_SNAPPY=ON \ + -DARROW_WITH_ZLIB=ON \ + -DARROW_WITH_ZSTD=ON \ + $ARROW_CMAKE_OPTIONS + +make -j$NPROC +make install + +popd + +echo +echo "==" +echo "== Building example project using Arrow C++ library" +echo "==" +echo + +mkdir -p $EXAMPLE_BUILD_DIR +pushd $EXAMPLE_BUILD_DIR + +cmake $EXAMPLE_DIR -DARROW_LINK_SHARED=OFF +make + +popd + +echo +echo "==" +echo "== Running example project" +echo "==" +echo + +cd $EXAMPLE_DIR + +${EXAMPLE_BUILD_DIR}/arrow_example diff --git a/cpp/src/arrow/ArrowConfig.cmake.in b/cpp/src/arrow/ArrowConfig.cmake.in index 0e595066dda..c4b35d5aba3 100644 --- a/cpp/src/arrow/ArrowConfig.cmake.in +++ b/cpp/src/arrow/ArrowConfig.cmake.in @@ -40,4 +40,24 @@ include("${CMAKE_CURRENT_LIST_DIR}/ArrowOptions.cmake") # already existent target error. if(NOT (TARGET arrow_shared OR TARGET arrow_static)) include("${CMAKE_CURRENT_LIST_DIR}/ArrowTargets.cmake") + + if(TARGET arrow_static) + get_property(arrow_static_loc TARGET arrow_static PROPERTY LOCATION) + get_filename_component(arrow_lib_dir ${arrow_static_loc} DIRECTORY) + + add_library(arrow_bundled_dependencies STATIC IMPORTED) + set_target_properties( + arrow_bundled_dependencies + PROPERTIES + IMPORTED_LOCATION + "${arrow_lib_dir}/${CMAKE_STATIC_LIBRARY_PREFIX}arrow_bundled_dependencies${CMAKE_STATIC_LIBRARY_SUFFIX}" + ) + + get_property(arrow_static_interface_link_libraries + TARGET arrow_static + PROPERTY INTERFACE_LINK_LIBRARIES) + set_target_properties( + arrow_static PROPERTIES INTERFACE_LINK_LIBRARIES + "${arrow_static_interface_link_libraries};arrow_bundled_dependencies") + endif() endif() diff --git a/cpp/src/arrow/CMakeLists.txt b/cpp/src/arrow/CMakeLists.txt index 818a527a306..eea19e54954 100644 --- a/cpp/src/arrow/CMakeLists.txt +++ b/cpp/src/arrow/CMakeLists.txt @@ -500,6 +500,18 @@ if(ARROW_WITH_BACKTRACE) endforeach() endif() +if(ARROW_BUILD_STATIC AND ARROW_BUNDLED_STATIC_LIBS) + arrow_car(_FIRST_LIB ${ARROW_BUNDLED_STATIC_LIBS}) + arrow_cdr(_OTHER_LIBS ${ARROW_BUNDLED_STATIC_LIBS}) + create_merged_static_lib(arrow_bundled_dependencies + NAME + arrow_bundled_dependencies + ROOT + ${_FIRST_LIB} + TO_MERGE + ${_OTHER_LIBS}) +endif() + if(ARROW_BUILD_TESTS OR ARROW_BUILD_BENCHMARKS OR ARROW_BUILD_INTEGRATION diff --git a/cpp/src/arrow/flight/CMakeLists.txt b/cpp/src/arrow/flight/CMakeLists.txt index 4cdadb42458..9ce4654f90b 100644 --- a/cpp/src/arrow/flight/CMakeLists.txt +++ b/cpp/src/arrow/flight/CMakeLists.txt @@ -32,7 +32,6 @@ if(ARROW_TEST_LINKAGE STREQUAL "static") ${ARROW_TEST_LINK_LIBS}) else() set(ARROW_FLIGHT_TEST_LINK_LIBS arrow_flight_shared arrow_flight_testing_shared - ${ARROW_FLIGHT_STATIC_LINK_LIBS} ${ARROW_TEST_LINK_LIBS}) endif() diff --git a/dev/tasks/homebrew-formulae/apache-arrow.rb b/dev/tasks/homebrew-formulae/apache-arrow.rb index c5dbb526c8b..81949d16c28 100644 --- a/dev/tasks/homebrew-formulae/apache-arrow.rb +++ b/dev/tasks/homebrew-formulae/apache-arrow.rb @@ -26,6 +26,7 @@ def install args = %W[ -DARROW_FLIGHT=ON -DARROW_GANDIVA=ON + -DARROW_JEMALLOC=ON -DARROW_ORC=ON -DARROW_PARQUET=ON -DARROW_PLASMA=ON diff --git a/dev/tasks/homebrew-formulae/autobrew/apache-arrow.rb b/dev/tasks/homebrew-formulae/autobrew/apache-arrow.rb index 691366b2be0..755c01cfa00 100644 --- a/dev/tasks/homebrew-formulae/autobrew/apache-arrow.rb +++ b/dev/tasks/homebrew-formulae/autobrew/apache-arrow.rb @@ -47,7 +47,7 @@ def install -DARROW_JSON=ON -DARROW_PARQUET=ON -DARROW_BUILD_SHARED=OFF - -DARROW_JEMALLOC=OFF + -DARROW_JEMALLOC=ON -DARROW_USE_GLOG=OFF -DARROW_PYTHON=OFF -DARROW_S3=OFF @@ -56,6 +56,7 @@ def install -DARROW_WITH_SNAPPY=ON -DARROW_WITH_UTF8PROC=OFF -DARROW_BUILD_UTILITIES=ON + -DARROW_VERBOSE_THIRDPARTY_BUILD=ON -DCMAKE_UNITY_BUILD=ON -DPARQUET_BUILD_EXECUTABLES=ON -DLZ4_HOME=#{Formula["lz4"].prefix} diff --git a/dev/tasks/linux-packages/apache-arrow/debian.ubuntu-xenial/libarrow-dev.install b/dev/tasks/linux-packages/apache-arrow/debian.ubuntu-xenial/libarrow-dev.install index 4f1670a39a8..443e70cf0c1 100644 --- a/dev/tasks/linux-packages/apache-arrow/debian.ubuntu-xenial/libarrow-dev.install +++ b/dev/tasks/linux-packages/apache-arrow/debian.ubuntu-xenial/libarrow-dev.install @@ -6,6 +6,7 @@ usr/lib/*/cmake/arrow/FindArrow.cmake usr/lib/*/cmake/arrow/arrow-config.cmake usr/lib/*/libarrow.a usr/lib/*/libarrow.so +usr/lib/*/libarrow_bundled_dependencies.a usr/lib/*/pkgconfig/arrow-compute.pc usr/lib/*/pkgconfig/arrow-csv.pc usr/lib/*/pkgconfig/arrow-filesystem.pc diff --git a/dev/tasks/linux-packages/apache-arrow/debian/libarrow-dev.install b/dev/tasks/linux-packages/apache-arrow/debian/libarrow-dev.install index 4f1670a39a8..443e70cf0c1 100644 --- a/dev/tasks/linux-packages/apache-arrow/debian/libarrow-dev.install +++ b/dev/tasks/linux-packages/apache-arrow/debian/libarrow-dev.install @@ -6,6 +6,7 @@ usr/lib/*/cmake/arrow/FindArrow.cmake usr/lib/*/cmake/arrow/arrow-config.cmake usr/lib/*/libarrow.a usr/lib/*/libarrow.so +usr/lib/*/libarrow_bundled_dependencies.a usr/lib/*/pkgconfig/arrow-compute.pc usr/lib/*/pkgconfig/arrow-csv.pc usr/lib/*/pkgconfig/arrow-filesystem.pc diff --git a/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in b/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in index f36fdfd66de..77a665c2dea 100644 --- a/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in +++ b/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in @@ -268,6 +268,7 @@ Libraries and header files for Apache Arrow C++. %{_libdir}/cmake/arrow/arrow-config.cmake %{_libdir}/libarrow.a %{_libdir}/libarrow.so +%{_libdir}/libarrow_bundled_dependencies.a %{_libdir}/pkgconfig/arrow-compute.pc %{_libdir}/pkgconfig/arrow-csv.pc %{_libdir}/pkgconfig/arrow-filesystem.pc diff --git a/docs/source/developers/cpp/building.rst b/docs/source/developers/cpp/building.rst index 1dc96ef96a9..fa2c4b05453 100644 --- a/docs/source/developers/cpp/building.rst +++ b/docs/source/developers/cpp/building.rst @@ -309,6 +309,11 @@ version number from ``cpp/thirdparty/versions.txt`` is used. There is also a dependency source downloader script (see below), which can be used to set up offline builds. +When using ``BUNDLED`` for dependency resolution (and if you use either the +jemalloc or mimalloc allocators, which are recommended), statically linking the +Arrow libraries in a third party project is more complex. See below for +instructions about how to configure your build system in this case. + Boost-related Options ~~~~~~~~~~~~~~~~~~~~~ @@ -347,6 +352,33 @@ You can then invoke CMake to create the build directory and it will use the declared environment variable pointing to downloaded archives instead of downloading them (one for each build dir!). +Statically Linking +~~~~~~~~~~~~~~~~~~ + +When ``-DARROW_BUILD_STATIC=ON``, all build dependencies built as static +libraries by the Arrow build system will be merged together to create a static +library ``arrow_bundled_dependencies``. In UNIX-like environments (Linux, macOS, +MinGW), this is called ``libarrow_bundled_dependencies.a`` and on Windows with +Visual Studio ``arrow_bundled_dependencies.lib``. This "dependency bundle" +library is installed in the same place as the other Arrow static libraries. + +If you are using CMake, the bundled dependencies will automatically be included +when linking if you use the ``arrow_static`` CMake target. In other build +systems, you may need to explicitly link to the dependency bundle. We created +an `example CMake-based build configuration +`_ to +show you a working example. + +On Linux and macOS, if your application does not link to the ``pthread`` +library already, you must include ``-pthread`` in your linker setup. In CMake +this can be accomplished with the ``Threads`` built-in package: + +.. code-block:: cmake + + set(THREADS_PREFER_PTHREAD_FLAG ON) + find_package(Threads REQUIRED) + target_link_libraries(my_target PRIVATE Threads::Threads) + Extra debugging help ~~~~~~~~~~~~~~~~~~~~ diff --git a/r/configure b/r/configure index 85f16e08126..71c885a593d 100755 --- a/r/configure +++ b/r/configure @@ -114,7 +114,7 @@ else fi fi PKG_CFLAGS="-I$BREWDIR/opt/$PKG_BREW_NAME/include" - PKG_LIBS="-L$BREWDIR/opt/$PKG_BREW_NAME/lib $PKG_LIBS" + PKG_LIBS="-L$BREWDIR/opt/$PKG_BREW_NAME/lib $PKG_LIBS -larrow_bundled_dependencies" elif [ "$UNAME" = "Linux" ]; then # Set some default values/backwards compatibility if [ "${LIBARROW_DOWNLOAD}" = "" ] && [ "${NOT_CRAN}" != "" ]; then diff --git a/r/configure.win b/r/configure.win index 3072d0e9e6d..5caf8e24663 100644 --- a/r/configure.win +++ b/r/configure.win @@ -43,11 +43,12 @@ else RWINLIB="../windows/$(ls windows/ | grep ^arrow-)" fi OPENSSL_LIBS="-lcrypto -lcrypt32" +MIMALLOC_LIBS="-lbcrypt -lpsapi" # NOTE: If you make changes to the libraries below, you should also change # ci/scripts/r_windows_build.sh and ci/scripts/PKGBUILD PKG_CFLAGS="-I${RWINLIB}/include -DARROW_STATIC -DPARQUET_STATIC -DARROW_DS_STATIC -DARROW_R_WITH_ARROW" -PKG_LIBS="-L${RWINLIB}/lib"'$(subst gcc,,$(COMPILED_BY))$(R_ARCH) '"-L${RWINLIB}/lib"'$(R_ARCH) '"-lparquet -larrow_dataset -larrow -lthrift -lsnappy -lz -lzstd -llz4 ${OPENSSL_LIBS} -lws2_32" +PKG_LIBS="-L${RWINLIB}/lib"'$(subst gcc,,$(COMPILED_BY))$(R_ARCH) '"-L${RWINLIB}/lib"'$(R_ARCH) '"-lparquet -larrow_dataset -larrow -larrow_bundled_dependencies -lthrift -lsnappy -lz -lzstd -llz4 ${MIMALLOC_LIBS} ${OPENSSL_LIBS} -lws2_32" # Set any user-defined CXXFLAGS if [ "$ARROW_R_CXXFLAGS" ]; then diff --git a/r/inst/build_arrow_static.sh b/r/inst/build_arrow_static.sh index 73aba450ed7..8593b896ec6 100755 --- a/r/inst/build_arrow_static.sh +++ b/r/inst/build_arrow_static.sh @@ -94,18 +94,4 @@ if [ $? -ne 0 ]; then fi fi -if [ -d "${DEST_DIR}/lib" ]; then - # If the build failed and the CMAKE_INSTALL_LIBDIR doesn't exist, don't try - # to do this stuff because the errors will be misleading - - # Copy the bundled static libs from the build to the install dir - # See https://issues.apache.org/jira/browse/ARROW-7499 for moving this to CMake - find . -regex .*/.*/lib/.*\\.a\$ | xargs -I{} cp -u {} ${DEST_DIR}/lib - # jemalloc makes both libjemalloc.a and libjemalloc_pic.a; we can't use the former, only the latter - rm ${DEST_DIR}/lib/libjemalloc.a || true - # -lbrotlicommon-static needs to come after the other brotli libs, so rename it so alpha sort works - if [ -f "${DEST_DIR}/lib/libbrotlicommon-static.a" ]; then - mv "${DEST_DIR}/lib/libbrotlicommon-static.a" "${DEST_DIR}/lib/libbrotlizzz-static.a" - fi -fi popd