From c8231c7685fbb7e76e172013b8021a51d0228fb9 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Wed, 30 Jun 2021 10:35:45 +0900 Subject: [PATCH 1/4] ARROW-6312: [C++] Add support for "pkg-config --static arrow" --- cpp/CMakeLists.txt | 10 ++- cpp/cmake_modules/ThirdpartyToolchain.cmake | 65 ++++++++++++++----- cpp/examples/minimal_build/minimal.dockerfile | 3 +- cpp/examples/minimal_build/run_static.sh | 33 +++++++++- .../system_dependency.dockerfile | 1 + cpp/src/arrow/CMakeLists.txt | 17 ++++- cpp/src/arrow/arrow.pc.in | 2 + 7 files changed, 110 insertions(+), 21 deletions(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 91b3528bf70..8a358db8b95 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -507,6 +507,11 @@ endif() include(BuildUtils) enable_testing() +# For arrow.pc. Requires.private and Libs.private are used when +# "pkg-config --libs --static arrow" is used. +set(ARROW_PC_REQUIRES_PRIVATE) +set(ARROW_PC_LIBS_PRIVATE) + include(ThirdpartyToolchain) # Add common flags @@ -855,8 +860,9 @@ endif() set(ARROW_SYSTEM_LINK_LIBS) -if(THREADS_FOUND) - list(APPEND ARROW_SYSTEM_LINK_LIBS Threads::Threads) +list(APPEND ARROW_SYSTEM_LINK_LIBS Threads::Threads) +if(CMAKE_THREAD_LIBS_INIT) + string(APPEND ARROW_PC_LIBS_PRIVATE " ${CMAKE_THREAD_LIBS_INIT}") endif() if(WIN32) diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index 40f73f92129..a4ed6695f01 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -196,6 +196,7 @@ endmacro() macro(resolve_dependency DEPENDENCY_NAME) set(options) set(one_value_args HAVE_ALT IS_RUNTIME_DEPENDENCY REQUIRED_VERSION USE_CONFIG) + set(multi_value_args PC_PACKAGE_NAMES) cmake_parse_arguments(ARG "${options}" "${one_value_args}" @@ -236,6 +237,9 @@ macro(resolve_dependency DEPENDENCY_NAME) if(${DEPENDENCY_NAME}_SOURCE STREQUAL "SYSTEM" AND ARG_IS_RUNTIME_DEPENDENCY) provide_find_module(${PACKAGE_NAME}) list(APPEND ARROW_SYSTEM_DEPENDENCIES ${PACKAGE_NAME}) + foreach(ARG_PC_PACKAGE_NAME ${ARG_PC_PACKAGE_NAMES}) + string(APPEND ARROW_PC_REQUIRES_PRIVATE " ${ARG_PC_PACKAGE_NAME}") + endforeach() endif() endmacro() @@ -933,7 +937,7 @@ macro(build_snappy) endmacro() if(ARROW_WITH_SNAPPY) - resolve_dependency(Snappy) + resolve_dependency(Snappy PC_PACKAGE_NAMES snappy) # TODO: Don't use global includes but rather target_include_directories get_target_property(SNAPPY_INCLUDE_DIRS Snappy::snappy INTERFACE_INCLUDE_DIRECTORIES) include_directories(SYSTEM ${SNAPPY_INCLUDE_DIRS}) @@ -998,7 +1002,7 @@ macro(build_brotli) endmacro() if(ARROW_WITH_BROTLI) - resolve_dependency(Brotli) + resolve_dependency(Brotli PC_PACKAGE_NAMES libbrotlidec libbrotlienc) # TODO: Don't use global includes but rather target_include_directories get_target_property(BROTLI_INCLUDE_DIR Brotli::brotlicommon INTERFACE_INCLUDE_DIRECTORIES) @@ -1079,9 +1083,9 @@ macro(build_glog) ) set(GLOG_CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fPIC") set(GLOG_CMAKE_C_FLAGS "${EP_C_FLAGS} -fPIC") - if(Threads::Threads) - set(GLOG_CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fPIC -pthread") - set(GLOG_CMAKE_C_FLAGS "${EP_C_FLAGS} -fPIC -pthread") + if(CMAKE_THREAD_LIBS_INIT) + set(GLOG_CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CMAKE_THREAD_LIBS_INIT}") + set(GLOG_CMAKE_C_FLAGS "${EP_C_FLAGS} ${CMAKE_THREAD_LIBS_INIT}") endif() if(APPLE) @@ -1117,7 +1121,7 @@ macro(build_glog) endmacro() if(ARROW_USE_GLOG) - resolve_dependency(GLOG) + resolve_dependency(GLOG PC_PACKAGE_NAMES libglog) # TODO: Don't use global includes but rather target_include_directories get_target_property(GLOG_INCLUDE_DIR glog::glog INTERFACE_INCLUDE_DIRECTORIES) include_directories(SYSTEM ${GLOG_INCLUDE_DIR}) @@ -1292,7 +1296,11 @@ if(ARROW_WITH_THRIFT) # to build Boost, so don't look again if already found. if(NOT Thrift_FOUND AND NOT THRIFT_FOUND) # Thrift c++ code generated by 0.13 requires 0.11 or greater - resolve_dependency(Thrift REQUIRED_VERSION 0.11.0) + resolve_dependency(Thrift + REQUIRED_VERSION + 0.11.0 + PC_PACKAGE_NAMES + thrift) endif() # TODO: Don't use global includes but rather target_include_directories include_directories(SYSTEM ${THRIFT_INCLUDE_DIR}) @@ -1392,7 +1400,11 @@ if(ARROW_WITH_PROTOBUF) else() set(ARROW_PROTOBUF_REQUIRED_VERSION "2.6.1") endif() - resolve_dependency(Protobuf REQUIRED_VERSION ${ARROW_PROTOBUF_REQUIRED_VERSION}) + resolve_dependency(Protobuf + REQUIRED_VERSION + ${ARROW_PROTOBUF_REQUIRED_VERSION} + PC_PACKAGE_NAMES + protobuf) if(ARROW_PROTOBUF_USE_SHARED AND MSVC_TOOLCHAIN) add_definitions(-DPROTOBUF_USE_DLLS) @@ -1825,7 +1837,11 @@ if(ARROW_BUILD_BENCHMARKS) # ci/conda_env_cpp.yml. set(BENCHMARK_REQUIRED_VERSION 0.0.0) endif() - resolve_dependency(benchmark REQUIRED_VERSION ${BENCHMARK_REQUIRED_VERSION}) + resolve_dependency(benchmark + REQUIRED_VERSION + ${BENCHMARK_REQUIRED_VERSION} + IS_RUNTIME_DEPENDENCY + FALSE) # TODO: Don't use global includes but rather target_include_directories get_target_property(BENCHMARK_INCLUDE_DIR benchmark::benchmark INTERFACE_INCLUDE_DIRECTORIES) @@ -1940,7 +1956,7 @@ macro(build_zlib) endmacro() if(ARROW_WITH_ZLIB) - resolve_dependency(ZLIB) + resolve_dependency(ZLIB PC_PACKAGE_NAMES zlib) # TODO: Don't use global includes but rather target_include_directories get_target_property(ZLIB_INCLUDE_DIR ZLIB::ZLIB INTERFACE_INCLUDE_DIRECTORIES) @@ -1996,7 +2012,7 @@ macro(build_lz4) endmacro() if(ARROW_WITH_LZ4) - resolve_dependency(Lz4) + resolve_dependency(Lz4 PC_PACKAGE_NAMES liblz4) # TODO: Don't use global includes but rather target_include_directories get_target_property(LZ4_INCLUDE_DIR LZ4::lz4 INTERFACE_INCLUDE_DIRECTORIES) @@ -2060,7 +2076,7 @@ macro(build_zstd) endmacro() if(ARROW_WITH_ZSTD) - resolve_dependency(zstd) + resolve_dependency(zstd PC_PACKAGE_NAMES libzstd) if(TARGET zstd::libzstd) set(ARROW_ZSTD_LIBZSTD zstd::libzstd) @@ -2120,7 +2136,11 @@ macro(build_re2) endmacro() if(ARROW_WITH_RE2) - resolve_dependency(re2 HAVE_ALT TRUE) + resolve_dependency(re2 + HAVE_ALT + TRUE + PC_PACKAGE_NAMES + re2) add_definitions(-DARROW_WITH_RE2) # TODO: Don't use global includes but rather target_include_directories @@ -2169,6 +2189,9 @@ endmacro() if(ARROW_WITH_BZ2) resolve_dependency(BZip2) + if(${BZip2_SOURCE} STREQUAL "SYSTEM") + string(APPEND ARROW_PC_LIBS_PRIVATE " ${BZIP2_LIBRARIES}") + endif() if(NOT TARGET BZip2::BZip2) add_library(BZip2::BZip2 UNKNOWN IMPORTED) @@ -2219,7 +2242,11 @@ macro(build_utf8proc) endmacro() if(ARROW_WITH_UTF8PROC) - resolve_dependency(utf8proc REQUIRED_VERSION "2.2.0") + resolve_dependency(utf8proc + REQUIRED_VERSION + "2.2.0" + PC_PACKAGE_NAMES + libutf8proc) add_definitions(-DARROW_WITH_UTF8PROC) @@ -2287,7 +2314,11 @@ endmacro() # Dependencies for Arrow Flight RPC macro(build_grpc) - resolve_dependency(c-ares HAVE_ALT TRUE) + resolve_dependency(c-ares + HAVE_ALT + TRUE + PC_PACKAGE_NAMES + libcares) # TODO: Don't use global includes but rather target_include_directories get_target_property(c-ares_INCLUDE_DIR c-ares::cares INTERFACE_INCLUDE_DIRECTORIES) include_directories(SYSTEM ${c-ares_INCLUDE_DIR}) @@ -2551,7 +2582,9 @@ if(ARROW_WITH_GRPC) HAVE_ALT TRUE REQUIRED_VERSION - ${ARROW_GRPC_REQUIRED_VERSION}) + ${ARROW_GRPC_REQUIRED_VERSION} + PC_PACKAGE_NAMES + grpc++) # TODO: Don't use global includes but rather target_include_directories get_target_property(GRPC_INCLUDE_DIR gRPC::grpc++ INTERFACE_INCLUDE_DIRECTORIES) diff --git a/cpp/examples/minimal_build/minimal.dockerfile b/cpp/examples/minimal_build/minimal.dockerfile index 95f73e9a549..9361fc5e81d 100644 --- a/cpp/examples/minimal_build/minimal.dockerfile +++ b/cpp/examples/minimal_build/minimal.dockerfile @@ -22,5 +22,6 @@ ENV DEBIAN_FRONTEND=noninteractive RUN apt-get update -y -q && \ apt-get install -y -q --no-install-recommends \ build-essential \ - cmake && \ + cmake \ + pkg-config && \ apt-get clean && rm -rf /var/lib/apt/lists* diff --git a/cpp/examples/minimal_build/run_static.sh b/cpp/examples/minimal_build/run_static.sh index 05804a0366c..ff3bb894570 100755 --- a/cpp/examples/minimal_build/run_static.sh +++ b/cpp/examples/minimal_build/run_static.sh @@ -67,10 +67,12 @@ popd echo echo "==" +echo "== CMake:" echo "== Building example project using Arrow C++ library" echo "==" echo +rm -rf $EXAMPLE_BUILD_DIR mkdir -p $EXAMPLE_BUILD_DIR pushd $EXAMPLE_BUILD_DIR @@ -81,10 +83,39 @@ popd echo echo "==" +echo "== CMake:" echo "== Running example project" echo "==" echo pushd $EXAMPLE_DIR -${EXAMPLE_BUILD_DIR}/arrow_example +$EXAMPLE_BUILD_DIR/arrow_example + +echo +echo "==" +echo "== pkg-config" +echo "== Building example project using Arrow C++ library" +echo "==" +echo + +rm -rf $EXAMPLE_BUILD_DIR +mkdir -p $EXAMPLE_BUILD_DIR +${CXX:-c++} \ + -o $EXAMPLE_BUILD_DIR/arrow_example \ + $EXAMPLE_DIR/example.cc \ + $(PKG_CONFIG_PATH=$ARROW_BUILD_DIR/lib/pkgconfig \ + pkg-config --cflags --libs --static arrow) + +popd + +echo +echo "==" +echo "== pkg-config:" +echo "== Running example project" +echo "==" +echo + +pushd $EXAMPLE_DIR + +$EXAMPLE_BUILD_DIR/arrow_example diff --git a/cpp/examples/minimal_build/system_dependency.dockerfile b/cpp/examples/minimal_build/system_dependency.dockerfile index f0b29cef990..926fcaf6f4b 100644 --- a/cpp/examples/minimal_build/system_dependency.dockerfile +++ b/cpp/examples/minimal_build/system_dependency.dockerfile @@ -37,6 +37,7 @@ RUN apt-get update -y -q && \ libthrift-dev \ libutf8proc-dev \ libzstd-dev \ + pkg-config \ protobuf-compiler \ rapidjson-dev \ zlib1g-dev && \ diff --git a/cpp/src/arrow/CMakeLists.txt b/cpp/src/arrow/CMakeLists.txt index 79b48461f9b..e22e800d812 100644 --- a/cpp/src/arrow/CMakeLists.txt +++ b/cpp/src/arrow/CMakeLists.txt @@ -489,6 +489,21 @@ endif() set(ARROW_ALL_SRCS ${ARROW_SRCS} ${ARROW_C_SRCS}) +if(ARROW_BUILD_STATIC AND ARROW_BUNDLED_STATIC_LIBS) + set(ARROW_BUILD_BUNDLED_DEPENDENCIES TRUE) +else() + set(ARROW_BUILD_BUNDLED_DEPENDENCIES FALSE) +endif() + +if(ARROW_BUILD_BUNDLED_DEPENDENCIES) + string(APPEND ARROW_PC_LIBS_PRIVATE " -larrow_bundled_dependencies") +endif() +# Need -latomic on Raspbian. +# See also: https://issues.apache.org/jira/browse/ARROW-12860 +if(${CMAKE_SYSTEM_NAME} STREQUAL "Linux" AND ${CMAKE_SYSTEM_PROCESSOR} MATCHES "armv7") + string(APPEND ARROW_PC_LIBS_PRIVATE " -latomic") +endif() + add_arrow_lib(arrow CMAKE_PACKAGE_NAME Arrow @@ -535,7 +550,7 @@ if(ARROW_WITH_BACKTRACE) endforeach() endif() -if(ARROW_BUILD_STATIC AND ARROW_BUNDLED_STATIC_LIBS) +if(ARROW_BUILD_BUNDLED_DEPENDENCIES) arrow_car(_FIRST_LIB ${ARROW_BUNDLED_STATIC_LIBS}) arrow_cdr(_OTHER_LIBS ${ARROW_BUNDLED_STATIC_LIBS}) create_merged_static_lib(arrow_bundled_dependencies diff --git a/cpp/src/arrow/arrow.pc.in b/cpp/src/arrow/arrow.pc.in index 947d534fdbf..ef995fdc3db 100644 --- a/cpp/src/arrow/arrow.pc.in +++ b/cpp/src/arrow/arrow.pc.in @@ -25,5 +25,7 @@ full_so_version=@ARROW_FULL_SO_VERSION@ Name: Apache Arrow Description: Arrow is a set of technologies that enable big-data systems to process and move data fast. Version: @ARROW_VERSION@ +Requires.private:@ARROW_PC_REQUIRES_PRIVATE@ Libs: -L${libdir} -larrow +Libs.private:@ARROW_PC_LIBS_PRIVATE@ Cflags: -I${includedir} From 5eec5b90fc9486c2392b6f7e1355523aeb187563 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Wed, 30 Jun 2021 11:49:25 +0900 Subject: [PATCH 2/4] Don't require .pc for system Snappy and re2 --- cpp/cmake_modules/ThirdpartyToolchain.cmake | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index a4ed6695f01..89125cae9b8 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -237,8 +237,12 @@ macro(resolve_dependency DEPENDENCY_NAME) if(${DEPENDENCY_NAME}_SOURCE STREQUAL "SYSTEM" AND ARG_IS_RUNTIME_DEPENDENCY) provide_find_module(${PACKAGE_NAME}) list(APPEND ARROW_SYSTEM_DEPENDENCIES ${PACKAGE_NAME}) + find_package(PkgConfig QUIET) foreach(ARG_PC_PACKAGE_NAME ${ARG_PC_PACKAGE_NAMES}) - string(APPEND ARROW_PC_REQUIRES_PRIVATE " ${ARG_PC_PACKAGE_NAME}") + pkg_check_modules(${ARG_PC_PACKAGE_NAME}_PC ${ARG_PC_PACKAGE_NAME} QUIET) + if(${${ARG_PC_PACKAGE_NAME}_PC_FOUND}) + string(APPEND ARROW_PC_REQUIRES_PRIVATE " ${ARG_PC_PACKAGE_NAME}") + endif() endforeach() endif() endmacro() @@ -938,6 +942,10 @@ endmacro() if(ARROW_WITH_SNAPPY) resolve_dependency(Snappy PC_PACKAGE_NAMES snappy) + if(${Snappy_SOURCE} STREQUAL "SYSTEM" AND NOT snappy_PC_FOUND) + get_target_property(SNAPPY_LIB Snappy::snappy IMPORTED_LOCATION) + string(APPEND ARROW_PC_LIBS_PRIVATE " ${SNAPPY_LIB}") + endif() # TODO: Don't use global includes but rather target_include_directories get_target_property(SNAPPY_INCLUDE_DIRS Snappy::snappy INTERFACE_INCLUDE_DIRECTORIES) include_directories(SYSTEM ${SNAPPY_INCLUDE_DIRS}) @@ -2141,6 +2149,10 @@ if(ARROW_WITH_RE2) TRUE PC_PACKAGE_NAMES re2) + if(${re2_SOURCE} STREQUAL "SYSTEM" AND NOT re2_PC_FOUND) + get_target_property(RE2_LIB re2::re2 IMPORTED_LOCATION) + string(APPEND ARROW_PC_LIBS_PRIVATE " ${RE2_LIB}") + endif() add_definitions(-DARROW_WITH_RE2) # TODO: Don't use global includes but rather target_include_directories From 0feb16af7173999eaf8217caf3e0fde36a841f83 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Wed, 30 Jun 2021 13:56:19 +0900 Subject: [PATCH 3/4] Don't use re2.pc --- cpp/cmake_modules/ThirdpartyToolchain.cmake | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index 89125cae9b8..ab2dd168a09 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -2144,12 +2144,11 @@ macro(build_re2) endmacro() if(ARROW_WITH_RE2) - resolve_dependency(re2 - HAVE_ALT - TRUE - PC_PACKAGE_NAMES - re2) - if(${re2_SOURCE} STREQUAL "SYSTEM" AND NOT re2_PC_FOUND) + # Don't specify "PC_PACKAGE_NAMES re2" here because re2.pc may + # include -std=c++11. It's not compatible with C source and C++ + # source not uses C++ 11. + resolve_dependency(re2 HAVE_ALT TRUE) + if(${re2_SOURCE} STREQUAL "SYSTEM") get_target_property(RE2_LIB re2::re2 IMPORTED_LOCATION) string(APPEND ARROW_PC_LIBS_PRIVATE " ${RE2_LIB}") endif() From 086e50573eac7dd01eebf4636a99b06f1dd08303 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Fri, 2 Jul 2021 09:44:56 +0900 Subject: [PATCH 4/4] Document about pkg-config support --- .../cpp/{cmake.rst => build_system.rst} | 72 +++++++++++++++++-- docs/source/cpp/getting_started.rst | 2 +- 2 files changed, 69 insertions(+), 5 deletions(-) rename docs/source/cpp/{cmake.rst => build_system.rst} (57%) diff --git a/docs/source/cpp/cmake.rst b/docs/source/cpp/build_system.rst similarity index 57% rename from docs/source/cpp/cmake.rst rename to docs/source/cpp/build_system.rst index f192988fc0c..c0d05e9dab3 100644 --- a/docs/source/cpp/cmake.rst +++ b/docs/source/cpp/build_system.rst @@ -18,6 +18,7 @@ .. default-domain:: cpp .. highlight:: cpp +=================================== Using Arrow C++ in your own project =================================== @@ -25,10 +26,16 @@ This section assumes you already have the Arrow C++ libraries on your system, either after installing them using a package manager or after :ref:`building them yourself `. -The recommended way to integrate the Arrow C++ libraries in your own C++ -project is to use CMake's -`find_package `_ -function for locating and integrating dependencies. +The recommended way to integrate the Arrow C++ libraries in your own +C++ project is to use CMake's `find_package +`_ +function for locating and integrating dependencies. If you don't use +CMake as a build system, you can use `pkg-config +`_ to find +installed the Arrow C++ libraries. + +CMake +===== Basic usage ----------- @@ -70,3 +77,60 @@ In most cases, it is recommended to use the Arrow shared libraries. .. seealso:: A Docker-based :doc:`minimal build example `. + +pkg-config +========== + +Basic usage +----------- + +You can get suitable build flags by the following command line: + +.. code-block:: shell + + pkg-config --cflags --libs arrow + +If you want to link the Arrow C++ static library, you need to add +``--static`` option: + +.. code-block:: shell + + pkg-config --cflags --libs --static arrow + +This minimal ``Makefile`` file compiles a ``my_example.cc`` source +file into an executable linked with the Arrow C++ shared library: + +.. code-block:: makefile + + my_example: my_example.cc + $(CXX) -o $@ $(CXXFLAGS) $< $$(pkg-config --cflags --libs arrow) + +Many build systems support pkg-config. For example: + + * `GNU Autotools `_ + * `CMake `_ + (But you should use ``find_package(Arrow)`` instead.) + * `Meson `_ + +Available packages +------------------ + +The Arrow C++ provides a pkg-config package for each module. Here are +all available packages: + + * ``arrow-csv`` + * ``arrow-cuda`` + * ``arrow-dataset`` + * ``arrow-filesystem`` + * ``arrow-flight-testing`` + * ``arrow-flight`` + * ``arrow-json`` + * ``arrow-orc`` + * ``arrow-python-flight`` + * ``arrow-python`` + * ``arrow-tensorflow`` + * ``arrow-testing`` + * ``arrow`` + * ``gandiva`` + * ``parquet`` + * ``plasma`` diff --git a/docs/source/cpp/getting_started.rst b/docs/source/cpp/getting_started.rst index d6cfb177044..3c7b7f94f01 100644 --- a/docs/source/cpp/getting_started.rst +++ b/docs/source/cpp/getting_started.rst @@ -25,7 +25,7 @@ User Guide overview conventions - cmake + build_system memory arrays datatypes