ARROW-15066: [C++] Enable use of non-bundled OpenTelemetry

ci/docker/debian-10-cpp.dockerfile

@@ -62,6 +62,7 @@ RUN apt-get update -y -q && \
         llvm-${llvm}-dev \
         make \
         ninja-build \
+        nlohmann-json3-dev \
         pkg-config \
         protobuf-compiler \
         python3-pip \

ci/docker/debian-11-cpp.dockerfile

@@ -61,6 +61,7 @@ RUN apt-get update -y -q && \
         llvm-${llvm}-dev \
         make \
         ninja-build \
+        nlohmann-json3-dev \
         pkg-config \
         protobuf-compiler-grpc \
         python3-pip \

ci/docker/fedora-33-cpp.dockerfile

@@ -44,6 +44,7 @@ RUN dnf update -y && \
         gtest-devel \
         java-latest-openjdk-devel \
         java-latest-openjdk-headless \
+        json-devel \
         libzstd-devel \
         llvm-devel \
         llvm-static \

ci/docker/ubuntu-20.04-cpp.dockerfile

@@ -93,6 +93,7 @@ RUN apt-get update -y -q && \
         libzstd-dev \
         make \
         ninja-build \
+        nlohmann-json3-dev \
         pkg-config \
         protobuf-compiler \
         python3-pip \

ci/docker/ubuntu-20.04-verify-rc.dockerfile

@@ -34,6 +34,7 @@ RUN apt-get update -y -q && \
         llvm-dev \
         maven \
         ninja-build \
+        nlohmann-json3-dev \
         openjdk-11-jdk \
         pkg-config \
         python3-dev \

ci/docker/ubuntu-21.04-cpp.dockerfile

@@ -92,6 +92,7 @@ RUN apt-get update -y -q && \
         libzstd-dev \
         make \
         ninja-build \
+        nlohmann-json3-dev \
         pkg-config \
         protobuf-compiler \
         protobuf-compiler-grpc \

ci/scripts/msys2_setup.sh

@@ -41,6 +41,7 @@ case "${target}" in
     packages+=(${MINGW_PACKAGE_PREFIX}-make)
     packages+=(${MINGW_PACKAGE_PREFIX}-mlir)
     packages+=(${MINGW_PACKAGE_PREFIX}-ninja)
+    packages+=(${MINGW_PACKAGE_PREFIX}-nlohmann-json)
     packages+=(${MINGW_PACKAGE_PREFIX}-polly)
     packages+=(${MINGW_PACKAGE_PREFIX}-protobuf)
     packages+=(${MINGW_PACKAGE_PREFIX}-python3-numpy)
@@ -67,7 +68,7 @@ case "${target}" in
     packages+=(${MINGW_PACKAGE_PREFIX}-gcc)
     packages+=(${MINGW_PACKAGE_PREFIX}-go)
     packages+=(${MINGW_PACKAGE_PREFIX}-toolchain)
-    packages+=(base-devel)    
+    packages+=(base-devel)
     ;;
 esac
 

cpp/cmake_modules/ThirdpartyToolchain.cmake

@@ -63,6 +63,7 @@ set(ARROW_THIRDPARTY_DEPENDENCIES
     GTest
     LLVM
     Lz4
+    nlohmann_json
     opentelemetry-cpp
     ORC
     re2
@@ -162,6 +163,8 @@ macro(build_dependency DEPENDENCY_NAME)
     build_gtest()
   elseif("${DEPENDENCY_NAME}" STREQUAL "Lz4")
     build_lz4()
+  elseif("${DEPENDENCY_NAME}" STREQUAL "nlohmann_json")
+    build_nlohmann_json()
   elseif("${DEPENDENCY_NAME}" STREQUAL "opentelemetry-cpp")
     build_opentelemetry()
   elseif("${DEPENDENCY_NAME}" STREQUAL "ORC")
@@ -276,6 +279,7 @@ if(PARQUET_REQUIRE_ENCRYPTION)
 endif()
 
 if(ARROW_WITH_OPENTELEMETRY)
+  set(ARROW_WITH_NLOHMANN_JSON ON)
   set(ARROW_WITH_PROTOBUF ON)
 endif()
 
@@ -304,6 +308,7 @@ endif()
 
 if(ARROW_GCS)
   set(ARROW_WITH_GOOGLE_CLOUD_CPP ON)
+  set(ARROW_WITH_NLOHMANN_JSON ON)
 endif()
 
 if(ARROW_JSON)
@@ -3749,36 +3754,41 @@ macro(build_crc32c_once)
   endif()
 endmacro()
 
-macro(build_nlohmann_json_once)
-  if(NOT TARGET nlohmann_json_ep)
-    message(STATUS "Building nlohmann-json from source")
-    # "Build" nlohmann-json
-    set(NLOHMANN_JSON_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/nlohmann_json_ep-install")
-    set(NLOHMANN_JSON_INCLUDE_DIR "${NLOHMANN_JSON_PREFIX}/include")
-    set(NLOHMANN_JSON_CMAKE_ARGS
-        ${EP_COMMON_CMAKE_ARGS} "-DCMAKE_INSTALL_PREFIX=<INSTALL_DIR>"
-        -DBUILD_TESTING=OFF -DJSON_BuildTests=OFF)
+macro(build_nlohmann_json)
+  message(STATUS "Building nlohmann-json from source")
+  # "Build" nlohmann-json
+  set(NLOHMANN_JSON_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/nlohmann_json_ep-install")
+  set(NLOHMANN_JSON_INCLUDE_DIR "${NLOHMANN_JSON_PREFIX}/include")
+  set(NLOHMANN_JSON_CMAKE_ARGS
+      ${EP_COMMON_CMAKE_ARGS} "-DCMAKE_INSTALL_PREFIX=<INSTALL_DIR>" -DBUILD_TESTING=OFF
+      -DJSON_BuildTests=OFF)
 
-    set(NLOHMANN_JSON_BUILD_BYPRODUCTS ${NLOHMANN_JSON_PREFIX}/include/nlohmann/json.hpp)
+  set(NLOHMANN_JSON_BUILD_BYPRODUCTS ${NLOHMANN_JSON_PREFIX}/include/nlohmann/json.hpp)
 
-    externalproject_add(nlohmann_json_ep
-                        ${EP_LOG_OPTIONS}
-                        INSTALL_DIR ${NLOHMANN_JSON_PREFIX}
-                        URL ${NLOHMANN_JSON_SOURCE_URL}
-                        URL_HASH "SHA256=${ARROW_NLOHMANN_JSON_BUILD_SHA256_CHECKSUM}"
-                        CMAKE_ARGS ${NLOHMANN_JSON_CMAKE_ARGS}
-                        BUILD_BYPRODUCTS ${NLOHMANN_JSON_BUILD_BYPRODUCTS})
+  externalproject_add(nlohmann_json_ep
+                      ${EP_LOG_OPTIONS}
+                      INSTALL_DIR ${NLOHMANN_JSON_PREFIX}
+                      URL ${NLOHMANN_JSON_SOURCE_URL}
+                      URL_HASH "SHA256=${ARROW_NLOHMANN_JSON_BUILD_SHA256_CHECKSUM}"
+                      CMAKE_ARGS ${NLOHMANN_JSON_CMAKE_ARGS}
+                      BUILD_BYPRODUCTS ${NLOHMANN_JSON_BUILD_BYPRODUCTS})
 
-    # Work around https://gitlab.kitware.com/cmake/cmake/issues/15052
-    file(MAKE_DIRECTORY ${NLOHMANN_JSON_INCLUDE_DIR})
+  # Work around https://gitlab.kitware.com/cmake/cmake/issues/15052
+  file(MAKE_DIRECTORY ${NLOHMANN_JSON_INCLUDE_DIR})
 
-    add_library(nlohmann_json::nlohmann_json INTERFACE IMPORTED)
-    set_target_properties(nlohmann_json::nlohmann_json
-                          PROPERTIES INTERFACE_INCLUDE_DIRECTORIES
-                                     "${NLOHMANN_JSON_INCLUDE_DIR}")
-    add_dependencies(nlohmann_json::nlohmann_json nlohmann_json_ep)
-  endif()
+  add_library(nlohmann_json::nlohmann_json INTERFACE IMPORTED)
+  set_target_properties(nlohmann_json::nlohmann_json
+                        PROPERTIES INTERFACE_INCLUDE_DIRECTORIES
+                                   "${NLOHMANN_JSON_INCLUDE_DIR}")
+  add_dependencies(nlohmann_json::nlohmann_json nlohmann_json_ep)
 endmacro()
+if(ARROW_WITH_NLOHMANN_JSON)
+  resolve_dependency(nlohmann_json)
+  get_target_property(nlohmann_json_INCLUDE_DIR nlohmann_json::nlohmann_json
+                      INTERFACE_INCLUDE_DIRECTORIES)
+  include_directories(SYSTEM ${nlohmann_json_INCLUDE_DIR})
+  message(STATUS "Found nlohmann_json headers: ${nlohmann_json_INCLUDE_DIR}")
+endif()
 
 macro(build_google_cloud_cpp_storage)
   message(STATUS "Building google-cloud-cpp from source")
@@ -3787,7 +3797,6 @@ macro(build_google_cloud_cpp_storage)
   # List of dependencies taken from https://github.com/googleapis/google-cloud-cpp/blob/master/doc/packaging.md
   build_absl_once()
   build_crc32c_once()
-  build_nlohmann_json_once()
 
   # Curl is required on all platforms, but building it internally might also trip over S3's copy.
   # For now, force its inclusion from the underlying system or fail.
@@ -3827,7 +3836,7 @@ macro(build_google_cloud_cpp_storage)
 
   add_dependencies(google_cloud_cpp_dependencies absl_ep)
   add_dependencies(google_cloud_cpp_dependencies crc32c_ep)
-  add_dependencies(google_cloud_cpp_dependencies nlohmann_json_ep)
+  add_dependencies(google_cloud_cpp_dependencies nlohmann_json::nlohmann_json)
 
   set(GOOGLE_CLOUD_CPP_STATIC_LIBRARY_STORAGE
       "${GOOGLE_CLOUD_CPP_INSTALL_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}google_cloud_cpp_storage${CMAKE_STATIC_LIBRARY_SUFFIX}"
@@ -4020,9 +4029,6 @@ endif()
 macro(build_opentelemetry)
   message(STATUS "Building OpenTelemetry from source")
 
-  build_nlohmann_json_once()
-  find_curl()
-
   set(OPENTELEMETRY_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/opentelemetry_ep-install")
   set(OPENTELEMETRY_INCLUDE_DIR "${OPENTELEMETRY_PREFIX}/include")
   set(OPENTELEMETRY_STATIC_LIB
@@ -4130,8 +4136,8 @@ macro(build_opentelemetry)
                       INSTALL_COMMAND ""
                       EXCLUDE_FROM_ALL OFF)
 
-  add_dependencies(opentelemetry_dependencies nlohmann_json_ep opentelemetry_proto_ep
-                   ${ARROW_PROTOBUF_LIBPROTOBUF})
+  add_dependencies(opentelemetry_dependencies nlohmann_json::nlohmann_json
+                   opentelemetry_proto_ep ${ARROW_PROTOBUF_LIBPROTOBUF})
 
   set(OPENTELEMETRY_PREFIX_PATH_LIST_SEP_CHAR "|")
   # JOIN is CMake >=3.12 only
@@ -4222,6 +4228,9 @@ macro(build_opentelemetry)
 endmacro()
 
 if(ARROW_WITH_OPENTELEMETRY)
+  # cURL is required whether we build from source or use an existing installation
+  # (OTel's cmake files do not call find_curl for you)
+  find_curl()
   set(opentelemetry-cpp_SOURCE "AUTO")
   resolve_dependency(opentelemetry-cpp)
   get_target_property(OPENTELEMETRY_INCLUDE_DIR opentelemetry-cpp::api