From 6b3444f66bc09e9aeee69cee8118857ab2314f9a Mon Sep 17 00:00:00 2001 From: David Li Date: Wed, 15 Dec 2021 09:23:10 -0500 Subject: [PATCH 1/6] ARROW-15066: [C++] Enable use of OpenTelemetry from conda-forge --- cpp/cmake_modules/ThirdpartyToolchain.cmake | 74 ++++++++++++--------- 1 file changed, 42 insertions(+), 32 deletions(-) diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index 81524653db6..c7cc7b03d6d 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -63,6 +63,7 @@ set(ARROW_THIRDPARTY_DEPENDENCIES GTest LLVM Lz4 + nlohmann_json opentelemetry-cpp ORC re2 @@ -162,6 +163,8 @@ macro(build_dependency DEPENDENCY_NAME) build_gtest() elseif("${DEPENDENCY_NAME}" STREQUAL "Lz4") build_lz4() + elseif("${DEPENDENCY_NAME}" STREQUAL "nlohmann_json") + build_nlohmann_json() elseif("${DEPENDENCY_NAME}" STREQUAL "opentelemetry-cpp") build_opentelemetry() elseif("${DEPENDENCY_NAME}" STREQUAL "ORC") @@ -276,6 +279,7 @@ if(PARQUET_REQUIRE_ENCRYPTION) endif() if(ARROW_WITH_OPENTELEMETRY) + set(ARROW_WITH_NLOHMANN_JSON ON) set(ARROW_WITH_PROTOBUF ON) endif() @@ -304,6 +308,7 @@ endif() if(ARROW_GCS) set(ARROW_WITH_GOOGLE_CLOUD_CPP ON) + set(ARROW_WITH_NLOHMANN_JSON ON) endif() if(ARROW_JSON) @@ -3749,36 +3754,42 @@ macro(build_crc32c_once) endif() endmacro() -macro(build_nlohmann_json_once) - if(NOT TARGET nlohmann_json_ep) - message(STATUS "Building nlohmann-json from source") - # "Build" nlohmann-json - set(NLOHMANN_JSON_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/nlohmann_json_ep-install") - set(NLOHMANN_JSON_INCLUDE_DIR "${NLOHMANN_JSON_PREFIX}/include") - set(NLOHMANN_JSON_CMAKE_ARGS - ${EP_COMMON_CMAKE_ARGS} "-DCMAKE_INSTALL_PREFIX=" - -DBUILD_TESTING=OFF -DJSON_BuildTests=OFF) +macro(build_nlohmann_json) + message(STATUS "Building nlohmann-json from source") + # "Build" nlohmann-json + set(NLOHMANN_JSON_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/nlohmann_json_ep-install") + set(NLOHMANN_JSON_INCLUDE_DIR "${NLOHMANN_JSON_PREFIX}/include") + set(NLOHMANN_JSON_CMAKE_ARGS + ${EP_COMMON_CMAKE_ARGS} "-DCMAKE_INSTALL_PREFIX=" -DBUILD_TESTING=OFF + -DJSON_BuildTests=OFF) - set(NLOHMANN_JSON_BUILD_BYPRODUCTS ${NLOHMANN_JSON_PREFIX}/include/nlohmann/json.hpp) + set(NLOHMANN_JSON_BUILD_BYPRODUCTS ${NLOHMANN_JSON_PREFIX}/include/nlohmann/json.hpp) - externalproject_add(nlohmann_json_ep - ${EP_LOG_OPTIONS} - INSTALL_DIR ${NLOHMANN_JSON_PREFIX} - URL ${NLOHMANN_JSON_SOURCE_URL} - URL_HASH "SHA256=${ARROW_NLOHMANN_JSON_BUILD_SHA256_CHECKSUM}" - CMAKE_ARGS ${NLOHMANN_JSON_CMAKE_ARGS} - BUILD_BYPRODUCTS ${NLOHMANN_JSON_BUILD_BYPRODUCTS}) + externalproject_add(nlohmann_json_ep + ${EP_LOG_OPTIONS} + INSTALL_DIR ${NLOHMANN_JSON_PREFIX} + URL ${NLOHMANN_JSON_SOURCE_URL} + URL_HASH "SHA256=${ARROW_NLOHMANN_JSON_BUILD_SHA256_CHECKSUM}" + CMAKE_ARGS ${NLOHMANN_JSON_CMAKE_ARGS} + BUILD_BYPRODUCTS ${NLOHMANN_JSON_BUILD_BYPRODUCTS}) - # Work around https://gitlab.kitware.com/cmake/cmake/issues/15052 - file(MAKE_DIRECTORY ${NLOHMANN_JSON_INCLUDE_DIR}) + # Work around https://gitlab.kitware.com/cmake/cmake/issues/15052 + file(MAKE_DIRECTORY ${NLOHMANN_JSON_INCLUDE_DIR}) - add_library(nlohmann_json::nlohmann_json INTERFACE IMPORTED) - set_target_properties(nlohmann_json::nlohmann_json - PROPERTIES INTERFACE_INCLUDE_DIRECTORIES - "${NLOHMANN_JSON_INCLUDE_DIR}") - add_dependencies(nlohmann_json::nlohmann_json nlohmann_json_ep) - endif() + add_library(nlohmann_json::nlohmann_json INTERFACE IMPORTED) + set_target_properties(nlohmann_json::nlohmann_json + PROPERTIES INTERFACE_INCLUDE_DIRECTORIES + "${NLOHMANN_JSON_INCLUDE_DIR}") + add_dependencies(nlohmann_json::nlohmann_json nlohmann_json_ep) endmacro() +if(ARROW_WITH_NLOHMANN_JSON) + set(nlohmann_json_SOURCE "AUTO") + resolve_dependency(nlohmann_json) + get_target_property(nlohmann_json_INCLUDE_DIR nlohmann_json::nlohmann_json + INTERFACE_INCLUDE_DIRECTORIES) + include_directories(SYSTEM ${nlohmann_json_INCLUDE_DIR}) + message(STATUS "Found nlohmann_json headers: ${nlohmann_json_INCLUDE_DIR}") +endif() macro(build_google_cloud_cpp_storage) message(STATUS "Building google-cloud-cpp from source") @@ -3787,7 +3798,6 @@ macro(build_google_cloud_cpp_storage) # List of dependencies taken from https://github.com/googleapis/google-cloud-cpp/blob/master/doc/packaging.md build_absl_once() build_crc32c_once() - build_nlohmann_json_once() # Curl is required on all platforms, but building it internally might also trip over S3's copy. # For now, force its inclusion from the underlying system or fail. @@ -3827,7 +3837,7 @@ macro(build_google_cloud_cpp_storage) add_dependencies(google_cloud_cpp_dependencies absl_ep) add_dependencies(google_cloud_cpp_dependencies crc32c_ep) - add_dependencies(google_cloud_cpp_dependencies nlohmann_json_ep) + add_dependencies(google_cloud_cpp_dependencies nlohmann_json::nlohmann_json) set(GOOGLE_CLOUD_CPP_STATIC_LIBRARY_STORAGE "${GOOGLE_CLOUD_CPP_INSTALL_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}google_cloud_cpp_storage${CMAKE_STATIC_LIBRARY_SUFFIX}" @@ -4020,9 +4030,6 @@ endif() macro(build_opentelemetry) message(STATUS "Building OpenTelemetry from source") - build_nlohmann_json_once() - find_curl() - set(OPENTELEMETRY_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/opentelemetry_ep-install") set(OPENTELEMETRY_INCLUDE_DIR "${OPENTELEMETRY_PREFIX}/include") set(OPENTELEMETRY_STATIC_LIB @@ -4130,8 +4137,8 @@ macro(build_opentelemetry) INSTALL_COMMAND "" EXCLUDE_FROM_ALL OFF) - add_dependencies(opentelemetry_dependencies nlohmann_json_ep opentelemetry_proto_ep - ${ARROW_PROTOBUF_LIBPROTOBUF}) + add_dependencies(opentelemetry_dependencies nlohmann_json::nlohmann_json + opentelemetry_proto_ep ${ARROW_PROTOBUF_LIBPROTOBUF}) set(OPENTELEMETRY_PREFIX_PATH_LIST_SEP_CHAR "|") # JOIN is CMake >=3.12 only @@ -4222,6 +4229,9 @@ macro(build_opentelemetry) endmacro() if(ARROW_WITH_OPENTELEMETRY) + # cURL is required whether we build from source or use an existing installation + # (OTel's cmake files do not call find_curl for you) + find_curl() set(opentelemetry-cpp_SOURCE "AUTO") resolve_dependency(opentelemetry-cpp) get_target_property(OPENTELEMETRY_INCLUDE_DIR opentelemetry-cpp::api From 13b7a23e22332cbcfa89ae4e12a772d13ec39a4d Mon Sep 17 00:00:00 2001 From: David Li Date: Tue, 8 Mar 2022 08:56:58 -0500 Subject: [PATCH 2/6] ARROW-15066: [C++] Remove unnecessary config --- cpp/cmake_modules/ThirdpartyToolchain.cmake | 1 - 1 file changed, 1 deletion(-) diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index c7cc7b03d6d..df5bc0223de 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -3783,7 +3783,6 @@ macro(build_nlohmann_json) add_dependencies(nlohmann_json::nlohmann_json nlohmann_json_ep) endmacro() if(ARROW_WITH_NLOHMANN_JSON) - set(nlohmann_json_SOURCE "AUTO") resolve_dependency(nlohmann_json) get_target_property(nlohmann_json_INCLUDE_DIR nlohmann_json::nlohmann_json INTERFACE_INCLUDE_DIRECTORIES) From bc6693c046c819307e0171f913ad052c3f5ee709 Mon Sep 17 00:00:00 2001 From: David Li Date: Tue, 8 Mar 2022 09:26:07 -0500 Subject: [PATCH 3/6] Revert "ARROW-15066: [C++] Remove unnecessary config" This reverts commit 13b7a23e22332cbcfa89ae4e12a772d13ec39a4d. --- cpp/cmake_modules/ThirdpartyToolchain.cmake | 1 + 1 file changed, 1 insertion(+) diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index df5bc0223de..c7cc7b03d6d 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -3783,6 +3783,7 @@ macro(build_nlohmann_json) add_dependencies(nlohmann_json::nlohmann_json nlohmann_json_ep) endmacro() if(ARROW_WITH_NLOHMANN_JSON) + set(nlohmann_json_SOURCE "AUTO") resolve_dependency(nlohmann_json) get_target_property(nlohmann_json_INCLUDE_DIR nlohmann_json::nlohmann_json INTERFACE_INCLUDE_DIRECTORIES) From 93883c5fc1ff106eeebed846b140c0c5b6d71503 Mon Sep 17 00:00:00 2001 From: David Li Date: Tue, 8 Mar 2022 17:15:08 -0500 Subject: [PATCH 4/6] ARROW-15066: [C++] Install nlohmann_json packages where necessary --- ci/docker/debian-10-cpp.dockerfile | 1 + ci/docker/debian-11-cpp.dockerfile | 1 + ci/docker/fedora-33-cpp.dockerfile | 1 + ci/docker/ubuntu-20.04-cpp.dockerfile | 1 + ci/scripts/msys2_setup.sh | 3 ++- cpp/cmake_modules/ThirdpartyToolchain.cmake | 1 - 6 files changed, 6 insertions(+), 2 deletions(-) diff --git a/ci/docker/debian-10-cpp.dockerfile b/ci/docker/debian-10-cpp.dockerfile index b5fa2716907..15c85b868ba 100644 --- a/ci/docker/debian-10-cpp.dockerfile +++ b/ci/docker/debian-10-cpp.dockerfile @@ -62,6 +62,7 @@ RUN apt-get update -y -q && \ llvm-${llvm}-dev \ make \ ninja-build \ + nlohmann-json3-dev \ pkg-config \ protobuf-compiler \ python3-pip \ diff --git a/ci/docker/debian-11-cpp.dockerfile b/ci/docker/debian-11-cpp.dockerfile index 87bb5b50ba5..1bb67e334e8 100644 --- a/ci/docker/debian-11-cpp.dockerfile +++ b/ci/docker/debian-11-cpp.dockerfile @@ -61,6 +61,7 @@ RUN apt-get update -y -q && \ llvm-${llvm}-dev \ make \ ninja-build \ + nlohmann-json3-dev \ pkg-config \ protobuf-compiler-grpc \ python3-pip \ diff --git a/ci/docker/fedora-33-cpp.dockerfile b/ci/docker/fedora-33-cpp.dockerfile index a605275330c..0a6cb59f6bd 100644 --- a/ci/docker/fedora-33-cpp.dockerfile +++ b/ci/docker/fedora-33-cpp.dockerfile @@ -44,6 +44,7 @@ RUN dnf update -y && \ gtest-devel \ java-latest-openjdk-devel \ java-latest-openjdk-headless \ + json \ libzstd-devel \ llvm-devel \ llvm-static \ diff --git a/ci/docker/ubuntu-20.04-cpp.dockerfile b/ci/docker/ubuntu-20.04-cpp.dockerfile index c16aa49ad24..dfeb93bedd9 100644 --- a/ci/docker/ubuntu-20.04-cpp.dockerfile +++ b/ci/docker/ubuntu-20.04-cpp.dockerfile @@ -93,6 +93,7 @@ RUN apt-get update -y -q && \ libzstd-dev \ make \ ninja-build \ + nlohmann-json3-dev \ pkg-config \ protobuf-compiler \ python3-pip \ diff --git a/ci/scripts/msys2_setup.sh b/ci/scripts/msys2_setup.sh index 01cd5fa9ee3..aabb17b3b82 100755 --- a/ci/scripts/msys2_setup.sh +++ b/ci/scripts/msys2_setup.sh @@ -41,6 +41,7 @@ case "${target}" in packages+=(${MINGW_PACKAGE_PREFIX}-make) packages+=(${MINGW_PACKAGE_PREFIX}-mlir) packages+=(${MINGW_PACKAGE_PREFIX}-ninja) + packages+=(${MINGW_PACKAGE_PREFIX}-nlohmann-json) packages+=(${MINGW_PACKAGE_PREFIX}-polly) packages+=(${MINGW_PACKAGE_PREFIX}-protobuf) packages+=(${MINGW_PACKAGE_PREFIX}-python3-numpy) @@ -67,7 +68,7 @@ case "${target}" in packages+=(${MINGW_PACKAGE_PREFIX}-gcc) packages+=(${MINGW_PACKAGE_PREFIX}-go) packages+=(${MINGW_PACKAGE_PREFIX}-toolchain) - packages+=(base-devel) + packages+=(base-devel) ;; esac diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index c7cc7b03d6d..df5bc0223de 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -3783,7 +3783,6 @@ macro(build_nlohmann_json) add_dependencies(nlohmann_json::nlohmann_json nlohmann_json_ep) endmacro() if(ARROW_WITH_NLOHMANN_JSON) - set(nlohmann_json_SOURCE "AUTO") resolve_dependency(nlohmann_json) get_target_property(nlohmann_json_INCLUDE_DIR nlohmann_json::nlohmann_json INTERFACE_INCLUDE_DIRECTORIES) From 1b5183e8e8cbbe307bcf0ff8de0e9eb8f6b413a5 Mon Sep 17 00:00:00 2001 From: David Li Date: Wed, 9 Mar 2022 09:06:57 -0500 Subject: [PATCH 5/6] ARROW-15066: [C++] Fix fedora --- ci/docker/fedora-33-cpp.dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/docker/fedora-33-cpp.dockerfile b/ci/docker/fedora-33-cpp.dockerfile index 0a6cb59f6bd..2f729eed562 100644 --- a/ci/docker/fedora-33-cpp.dockerfile +++ b/ci/docker/fedora-33-cpp.dockerfile @@ -44,7 +44,7 @@ RUN dnf update -y && \ gtest-devel \ java-latest-openjdk-devel \ java-latest-openjdk-headless \ - json \ + json-devel \ libzstd-devel \ llvm-devel \ llvm-static \ From 6095fdd71e0a7a3fd7158c6818cfb61586a7b061 Mon Sep 17 00:00:00 2001 From: David Li Date: Wed, 9 Mar 2022 13:16:04 -0500 Subject: [PATCH 6/6] ARROW-15066: [C++] Install nlohmann_json on Ubuntu too --- ci/docker/ubuntu-20.04-verify-rc.dockerfile | 1 + ci/docker/ubuntu-21.04-cpp.dockerfile | 1 + 2 files changed, 2 insertions(+) diff --git a/ci/docker/ubuntu-20.04-verify-rc.dockerfile b/ci/docker/ubuntu-20.04-verify-rc.dockerfile index 9dd6f10f80c..e3415bd745b 100644 --- a/ci/docker/ubuntu-20.04-verify-rc.dockerfile +++ b/ci/docker/ubuntu-20.04-verify-rc.dockerfile @@ -34,6 +34,7 @@ RUN apt-get update -y -q && \ llvm-dev \ maven \ ninja-build \ + nlohmann-json3-dev \ openjdk-11-jdk \ pkg-config \ python3-dev \ diff --git a/ci/docker/ubuntu-21.04-cpp.dockerfile b/ci/docker/ubuntu-21.04-cpp.dockerfile index bd4e33f52ad..ff0979ea647 100644 --- a/ci/docker/ubuntu-21.04-cpp.dockerfile +++ b/ci/docker/ubuntu-21.04-cpp.dockerfile @@ -92,6 +92,7 @@ RUN apt-get update -y -q && \ libzstd-dev \ make \ ninja-build \ + nlohmann-json3-dev \ pkg-config \ protobuf-compiler \ protobuf-compiler-grpc \