apache · cyb70289 · Apr 10, 2022 · Apr 18, 2022 · Apr 18, 2022 · May 12, 2022
diff --git a/LICENSE.txt b/LICENSE.txt
@@ -2280,3 +2280,46 @@ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 SOFTWARE.
+
+--------------------------------------------------------------------------------
+
+3rdparty dependency UCX is redistributed as a dynamically linked shared
+library in certain binary distributions. UCX has the following license:
+
+Copyright (c) 2014-2015      UT-Battelle, LLC. All rights reserved.
+Copyright (C) 2014-2020      Mellanox Technologies Ltd. All rights reserved.
+Copyright (C) 2014-2015      The University of Houston System. All rights reserved.
+Copyright (C) 2015           The University of Tennessee and The University
+                             of Tennessee Research Foundation. All rights reserved.
+Copyright (C) 2016-2020      ARM Ltd. All rights reserved.
+Copyright (c) 2016           Los Alamos National Security, LLC. All rights reserved.
+Copyright (C) 2016-2020      Advanced Micro Devices, Inc.  All rights reserved.
+Copyright (C) 2019           UChicago Argonne, LLC.  All rights reserved.
+Copyright (c) 2018-2020      NVIDIA CORPORATION. All rights reserved.
+Copyright (C) 2020           Huawei Technologies Co., Ltd. All rights reserved.
+Copyright (C) 2016-2020      Stony Brook University. All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
+TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
@@ -827,6 +827,11 @@ if(ARROW_USE_XSIMD)
   list(APPEND ARROW_STATIC_LINK_LIBS xsimd)
 endif()
 
+if(ARROW_WITH_UCX)
+  list(APPEND ARROW_LINK_LIBS ucx::ucx)
+  list(APPEND ARROW_STATIC_LINK_LIBS ucx::ucx)
+endif()
+
 add_custom_target(arrow_dependencies)
 add_custom_target(arrow_benchmark_dependencies)
 add_custom_target(arrow_test_dependencies)

diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake
@@ -72,6 +72,7 @@ set(ARROW_THIRDPARTY_DEPENDENCIES
     Snappy
     Substrait
     Thrift
+    ucx
     utf8proc
     xsimd
     ZLIB
@@ -173,6 +174,8 @@ macro(build_dependency DEPENDENCY_NAME)
     build_substrait()
   elseif("${DEPENDENCY_NAME}" STREQUAL "Thrift")
     build_thrift()
+  elseif("${DEPENDENCY_NAME}" STREQUAL "ucx")
+    build_ucx()
   elseif("${DEPENDENCY_NAME}" STREQUAL "utf8proc")
     build_utf8proc()
   elseif("${DEPENDENCY_NAME}" STREQUAL "xsimd")
@@ -653,6 +656,13 @@ else()
            "${THIRDPARTY_MIRROR_URL}/thrift-${ARROW_THRIFT_BUILD_VERSION}.tar.gz")
 endif()
 
+if(DEFINED ENV{ARROW_UCX_URL})
+  set(ARROW_UCX_SOURCE_URL "$ENV{ARROW_UCX_URL}")
+else()
+  set_urls(ARROW_UCX_SOURCE_URL
+           "https://github.com/openucx/ucx/archive/v${ARROW_UCX_BUILD_VERSION}.tar.gz")
+endif()
+
 if(DEFINED ENV{ARROW_UTF8PROC_URL})
   set(ARROW_UTF8PROC_SOURCE_URL "$ENV{ARROW_UTF8PROC_URL}")
 else()
@@ -4545,6 +4555,82 @@ if(ARROW_S3)
   endif()
 endif()
 
+# ----------------------------------------------------------------------
+# ucx - communication framework for modern, high-bandwidth and low-latency networks
+
+macro(build_ucx)
+  message(STATUS "Building UCX from source")
+
+  set(UCX_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/ucx_ep-install")
+
+  # link with static ucx libraries leads to test failures, use shared libs instead
+  set(UCX_SHARED_LIB_UCP "${UCX_PREFIX}/lib/libucp${CMAKE_SHARED_LIBRARY_SUFFIX}")
+  set(UCX_SHARED_LIB_UCT "${UCX_PREFIX}/lib/libuct${CMAKE_SHARED_LIBRARY_SUFFIX}")
+  set(UCX_SHARED_LIB_UCS "${UCX_PREFIX}/lib/libucs${CMAKE_SHARED_LIBRARY_SUFFIX}")
+  set(UCX_SHARED_LIB_UCM "${UCX_PREFIX}/lib/libucm${CMAKE_SHARED_LIBRARY_SUFFIX}")
+
+  set(UCX_CONFIGURE_COMMAND ./autogen.sh COMMAND ./configure)
+  list(APPEND
+       UCX_CONFIGURE_COMMAND
+       "CC=${CMAKE_C_COMPILER}"
+       "CXX=${CMAKE_CXX_COMPILER}"
+       "CFLAGS=${EP_C_FLAGS}"
+       "CXXFLAGS=${EP_CXX_FLAGS}"
+       "--prefix=${UCX_PREFIX}"
+       "--enable-mt"
+       "--enable-shared")
+  if(${UPPERCASE_BUILD_TYPE} STREQUAL "DEBUG")
+    list(APPEND
+         UCX_CONFIGURE_COMMAND
+         "--enable-profiling"
+         "--enable-frame-pointer"
+         "--enable-stats"
+         "--enable-fault-injection"
+         "--enable-debug-data")
+  else()
+    list(APPEND
+         UCX_CONFIGURE_COMMAND
+         "--disable-logging"
+         "--disable-debug"
+         "--disable-assertions"
+         "--disable-params-check")
+  endif()
+  set(UCX_BUILD_COMMAND ${MAKE} ${MAKE_BUILD_ARGS})
+  externalproject_add(ucx_ep
+                      ${EP_LOG_OPTIONS}
+                      URL ${ARROW_UCX_SOURCE_URL}
+                      URL_HASH "SHA256=${ARROW_UCX_BUILD_SHA256_CHECKSUM}"
+                      CONFIGURE_COMMAND ${UCX_CONFIGURE_COMMAND}
+                      BUILD_IN_SOURCE 1
+                      BUILD_COMMAND ${UCX_BUILD_COMMAND}
+                      BUILD_BYPRODUCTS "${UCX_SHARED_LIB_UCP}" "${UCX_SHARED_LIB_UCT}"
+                                       "${UCX_SHARED_LIB_UCS}" "${UCX_SHARED_LIB_UCM}"
+                      INSTALL_COMMAND ${MAKE} install)
+
+  # ucx cmake module sets UCX_INCLUDE_DIRS
+  set(UCX_INCLUDE_DIRS "${UCX_PREFIX}/include")
+  file(MAKE_DIRECTORY "${UCX_INCLUDE_DIRS}")
+
+  add_library(ucx::ucp SHARED IMPORTED)
+  set_target_properties(ucx::ucp PROPERTIES IMPORTED_LOCATION "${UCX_SHARED_LIB_UCP}")
+  add_library(ucx::uct SHARED IMPORTED)
+  set_target_properties(ucx::uct PROPERTIES IMPORTED_LOCATION "${UCX_SHARED_LIB_UCT}")
+  add_library(ucx::ucs SHARED IMPORTED)
+  set_target_properties(ucx::ucs PROPERTIES IMPORTED_LOCATION "${UCX_SHARED_LIB_UCS}")
+
+  add_dependencies(toolchain ucx_ep)
+  add_dependencies(ucx::ucp ucx_ep)
+  add_dependencies(ucx::uct ucx_ep)
+  add_dependencies(ucx::ucs ucx_ep)
+endmacro()
+
+if(ARROW_WITH_UCX)
+  resolve_dependency(ucx PC_PACKAGE_NAMES ucx)
+  add_library(ucx::ucx INTERFACE IMPORTED)
+  target_include_directories(ucx::ucx INTERFACE "${UCX_INCLUDE_DIRS}")
+  target_link_libraries(ucx::ucx INTERFACE ucx::ucp ucx::uct ucx::ucs)
+endif()
+
 message(STATUS "All bundled static libraries: ${ARROW_BUNDLED_STATIC_LIBS}")
 
 # Write out the package configurations.

diff --git a/cpp/src/arrow/flight/transport/ucx/CMakeLists.txt b/cpp/src/arrow/flight/transport/ucx/CMakeLists.txt
@@ -18,18 +18,12 @@
 add_custom_target(arrow_flight_transport_ucx)
 arrow_install_all_headers("arrow/flight/transport/ucx")
 
-find_package(PkgConfig REQUIRED)
-pkg_check_modules(UCX REQUIRED IMPORTED_TARGET ucx)
-
 set(ARROW_FLIGHT_TRANSPORT_UCX_SRCS
     ucx_client.cc
     ucx_server.cc
     ucx.cc
     ucx_internal.cc
     util_internal.cc)
-set(ARROW_FLIGHT_TRANSPORT_UCX_LINK_LIBS)
-
-list(APPEND ARROW_FLIGHT_TRANSPORT_UCX_LINK_LIBS PkgConfig::UCX)
 
 add_arrow_lib(arrow_flight_transport_ucx
               # CMAKE_PACKAGE_NAME
@@ -46,11 +40,11 @@ add_arrow_lib(arrow_flight_transport_ucx
               SHARED_LINK_LIBS
               arrow_shared
               arrow_flight_shared
-              ${ARROW_FLIGHT_TRANSPORT_UCX_LINK_LIBS}
+              ucx::ucx
               STATIC_LINK_LIBS
               arrow_static
               arrow_flight_static
-              ${ARROW_FLIGHT_TRANSPORT_UCX_LINK_LIBS})
+              ucx::ucx)
 
 if(ARROW_BUILD_TESTS)
   if(ARROW_FLIGHT_TEST_LINKAGE STREQUAL "static")

diff --git a/cpp/thirdparty/versions.txt b/cpp/thirdparty/versions.txt
@@ -91,6 +91,8 @@ ARROW_SUBSTRAIT_BUILD_VERSION=e1b4c04a
 ARROW_SUBSTRAIT_BUILD_SHA256_CHECKSUM=65f83e5f5d979ede5fc8ac9f8bbaf793e0c72d9c415f1a162ba522f6d0bb5bbe
 ARROW_THRIFT_BUILD_VERSION=0.13.0
 ARROW_THRIFT_BUILD_SHA256_CHECKSUM=7ad348b88033af46ce49148097afe354d513c1fca7c607b59c33ebb6064b5179
+ARROW_UCX_BUILD_VERSION=1.12.1
+ARROW_UCX_BUILD_SHA256_CHECKSUM=9bef31aed0e28bf1973d28d74d9ac4f8926c43ca3b7010bd22a084e164e31b71
 ARROW_UTF8PROC_BUILD_VERSION=v2.7.0
 ARROW_UTF8PROC_BUILD_SHA256_CHECKSUM=4bb121e297293c0fd55f08f83afab6d35d48f0af4ecc07523ad8ec99aa2b12a1
 ARROW_XSIMD_BUILD_VERSION=7d1778c3b38d63db7cec7145d939f40bc5d859d1
@@ -134,6 +136,7 @@ DEPENDENCIES=(
   "ARROW_RE2_URL re2-${ARROW_RE2_BUILD_VERSION}.tar.gz https://github.com/google/re2/archive/${ARROW_RE2_BUILD_VERSION}.tar.gz"
   "ARROW_SNAPPY_URL snappy-${ARROW_SNAPPY_BUILD_VERSION}.tar.gz https://github.com/google/snappy/archive/${ARROW_SNAPPY_BUILD_VERSION}.tar.gz"
   "ARROW_THRIFT_URL thrift-${ARROW_THRIFT_BUILD_VERSION}.tar.gz https://archive.apache.org/dist/thrift/${ARROW_THRIFT_BUILD_VERSION}/thrift-${ARROW_THRIFT_BUILD_VERSION}.tar.gz"
+  "ARROW_UCX_URL ucx-${ARROW_UCX_BUILD_VERSION}.tar.gz https://github.com/openucx/ucx/archive/v${ARROW_UCX_BUILD_VERSION}.tar.gz"
   "ARROW_UTF8PROC_URL utf8proc-${ARROW_UTF8PROC_BUILD_VERSION}.tar.gz https://github.com/JuliaStrings/utf8proc/archive/${ARROW_UTF8PROC_BUILD_VERSION}.tar.gz"
   "ARROW_XSIMD_URL xsimd-${ARROW_XSIMD_BUILD_VERSION}.tar.gz https://github.com/xtensor-stack/xsimd/archive/${ARROW_XSIMD_BUILD_VERSION}.tar.gz"
   "ARROW_ZLIB_URL zlib-${ARROW_ZLIB_BUILD_VERSION}.tar.gz https://zlib.net/fossils/zlib-${ARROW_ZLIB_BUILD_VERSION}.tar.gz"