From c754b27b79c12e05cf358253e8165829afef66e8 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Mon, 20 Jan 2025 17:28:21 +0900 Subject: [PATCH 01/52] GH-45195: [C++] Update bundled AWS SDK for C++ to 1.11.488 --- cpp/cmake_modules/ThirdpartyToolchain.cmake | 534 ++++---------------- cpp/thirdparty/versions.txt | 60 +-- 2 files changed, 130 insertions(+), 464 deletions(-) diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index 193711a6fbd..c331ee8465e 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -1035,6 +1035,7 @@ macro(prepare_fetchcontent) string(APPEND CMAKE_C_FLAGS_DEBUG " -Wno-error") string(APPEND CMAKE_CXX_FLAGS_DEBUG " -Wno-error") endif() + set(ENABLE_TESTING OFF) endmacro() # ---------------------------------------------------------------------- @@ -5057,437 +5058,112 @@ endif() include(AWSSDKVariables) -macro(build_awssdk) - message(STATUS "Building AWS C++ SDK from source") - set(AWSSDK_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/awssdk_ep-install") - set(AWSSDK_INCLUDE_DIR "${AWSSDK_PREFIX}/include") +function(build_awssdk) + message(STATUS "Building AWS SDK for C++ from source") - # The AWS SDK has a few warnings around shortening lengths - set(AWS_C_FLAGS "${EP_C_FLAGS}") - set(AWS_CXX_FLAGS "${EP_CXX_FLAGS}") - if(CMAKE_CXX_COMPILER_ID STREQUAL "AppleClang" OR CMAKE_CXX_COMPILER_ID STREQUAL - "Clang") - # Negate warnings that AWS SDK cannot build under - string(APPEND AWS_C_FLAGS " -Wno-error=shorten-64-to-32") - string(APPEND AWS_CXX_FLAGS " -Wno-error=shorten-64-to-32") + set(AWSSDK_PRODUCTS aws-c-common aws-checksums) + # aws-lc and s2n-tls only needed on Linux. + # We can use LINUX with CMake 3.25 or later. + if(UNIX AND NOT APPLE) + list(APPEND AWSSDK_PRODUCTS + # aws-lc + s2n-tls) endif() - if(NOT MSVC) - string(APPEND AWS_C_FLAGS " -Wno-deprecated") - string(APPEND AWS_CXX_FLAGS " -Wno-deprecated") - endif() - # GH-44950: This is required to build under Rtools40 and we may be able to - # remove it if/when we no longer need to build under Rtools40 - if(WIN32 AND NOT MSVC) - string(APPEND - AWS_C_FLAGS - " -D_WIN32_WINNT=0x0601 -D__USE_MINGW_ANSI_STDIO=1 -Wno-error -Wno-error=format= -Wno-error=format-extra-args -Wno-unused-local-typedefs -Wno-unused-variable" - ) - string(APPEND - AWS_CXX_FLAGS - " -D_WIN32_WINNT=0x0601 -D__USE_MINGW_ANSI_STDIO=1 -Wno-error -Wno-error=format= -Wno-error=format-extra-args -Wno-unused-local-typedefs -Wno-unused-variable" - ) - endif() - - set(AWSSDK_COMMON_CMAKE_ARGS - ${EP_COMMON_CMAKE_ARGS} - -DCMAKE_C_FLAGS=${AWS_C_FLAGS} - -DCMAKE_CXX_FLAGS=${AWS_CXX_FLAGS} - -DCPP_STANDARD=${CMAKE_CXX_STANDARD} - -DCMAKE_INSTALL_PREFIX=${AWSSDK_PREFIX} - -DCMAKE_PREFIX_PATH=${AWSSDK_PREFIX} - -DENABLE_TESTING=OFF - -DENABLE_UNITY_BUILD=ON - -DOPENSSL_CRYPTO_LIBRARY=${OPENSSL_CRYPTO_LIBRARY} - -DOPENSSL_INCLUDE_DIR=${OPENSSL_INCLUDE_DIR} - -DOPENSSL_SSL_LIBRARY=${OPENSSL_SSL_LIBRARY} - -Dcrypto_INCLUDE_DIR=${OPENSSL_INCLUDE_DIR} - -Dcrypto_LIBRARY=${OPENSSL_CRYPTO_LIBRARY}) - if(ARROW_OPENSSL_USE_SHARED) - list(APPEND AWSSDK_COMMON_CMAKE_ARGS - -Dcrypto_SHARED_LIBRARY=${OPENSSL_CRYPTO_LIBRARY}) - else() - list(APPEND AWSSDK_COMMON_CMAKE_ARGS - -Dcrypto_STATIC_LIBRARY=${OPENSSL_CRYPTO_LIBRARY}) - endif() - set(AWSSDK_CMAKE_ARGS - ${AWSSDK_COMMON_CMAKE_ARGS} - -DBUILD_DEPS=OFF - -DBUILD_ONLY=config\\$s3\\$transfer\\$identity-management\\$sts - -DMINIMIZE_SIZE=ON) - # Remove unused directories to save build directory storage. - # 807MB -> 31MB - set(AWSSDK_PATCH_COMMAND ${CMAKE_COMMAND} -E) - if(CMAKE_VERSION VERSION_LESS 3.17) - list(APPEND AWSSDK_PATCH_COMMAND remove_directory) - else() - list(APPEND AWSSDK_PATCH_COMMAND rm -rf) - endif() - list(APPEND AWSSDK_PATCH_COMMAND ${AWSSDK_UNUSED_DIRECTORIES}) - - # Patch parts of the AWSSDK EP so it builds cleanly under Rtools40 - if(WIN32 AND NOT MSVC) - find_program(PATCH patch REQUIRED) - # Patch aws_c_common to build under Rtools40 - set(AWS_C_COMMON_PATCH_COMMAND ${PATCH} -p1 -i - ${CMAKE_SOURCE_DIR}/../ci/rtools/aws_c_common_ep.patch) - message(STATUS "Hello ${AWS_C_COMMON_PATCH_COMMAND}") - # aws_c_io_ep to build under Rtools40 - set(AWS_C_IO_PATCH_COMMAND ${PATCH} -p1 -i - ${CMAKE_SOURCE_DIR}/../ci/rtools/aws_c_io_ep.patch) - message(STATUS "Hello ${AWS_C_IO_PATCH_COMMAND}") - # awssdk_ep to build under Rtools40 - list(APPEND - AWSSDK_PATCH_COMMAND - && - ${PATCH} - -p1 - -i - ${CMAKE_SOURCE_DIR}/../ci/rtools/awssdk_ep.patch) - message(STATUS "Hello ${AWSSDK_PATCH_COMMAND}") - endif() - - if(UNIX) - # on Linux and macOS curl seems to be required - find_curl() - get_filename_component(CURL_ROOT_HINT "${CURL_INCLUDE_DIRS}" DIRECTORY) - get_filename_component(ZLIB_ROOT_HINT "${ZLIB_INCLUDE_DIRS}" DIRECTORY) - - # provide hint for AWS SDK to link with the already located libcurl and zlib - list(APPEND - AWSSDK_CMAKE_ARGS - -DCURL_INCLUDE_DIR=${CURL_ROOT_HINT}/include - -DCURL_LIBRARY=${CURL_ROOT_HINT}/lib - -DZLIB_INCLUDE_DIR=${ZLIB_ROOT_HINT}/include - -DZLIB_LIBRARY=${ZLIB_ROOT_HINT}/lib) - endif() - - file(MAKE_DIRECTORY ${AWSSDK_INCLUDE_DIR}) - - # AWS C++ SDK related libraries to link statically - set(_AWSSDK_LIBS - aws-cpp-sdk-identity-management - aws-cpp-sdk-sts - aws-cpp-sdk-cognito-identity - aws-cpp-sdk-s3 - aws-cpp-sdk-core - aws-crt-cpp - aws-c-s3 - aws-c-auth - aws-c-mqtt - aws-c-http - aws-c-compression - aws-c-sdkutils - aws-c-event-stream - aws-c-io - aws-c-cal - aws-checksums - aws-c-common) - - # aws-lc needs to be installed on a separate folder to hide from unintended use - set(AWS_LC_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/aws_lc_ep-install") - set(AWS_LC_INCLUDE_DIR "${AWS_LC_PREFIX}/include") - - if(UNIX AND NOT APPLE) # aws-lc and s2n-tls only needed on linux - file(MAKE_DIRECTORY ${AWS_LC_INCLUDE_DIR}) - list(APPEND _AWSSDK_LIBS s2n-tls aws-lc) - endif() - - set(AWSSDK_LIBRARIES) - foreach(_AWSSDK_LIB ${_AWSSDK_LIBS}) - # aws-c-common -> AWS-C-COMMON - string(TOUPPER ${_AWSSDK_LIB} _AWSSDK_LIB_UPPER) - # AWS-C-COMMON -> AWS_C_COMMON - string(REPLACE "-" "_" _AWSSDK_LIB_NAME_PREFIX ${_AWSSDK_LIB_UPPER}) - set(_AWSSDK_STATIC_LIBRARY - "${AWSSDK_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}${_AWSSDK_LIB}${CMAKE_STATIC_LIBRARY_SUFFIX}" - ) - if(${_AWSSDK_LIB} STREQUAL "s2n-tls") # Build output of s2n-tls is libs2n.a - set(_AWSSDK_STATIC_LIBRARY - "${AWSSDK_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}s2n${CMAKE_STATIC_LIBRARY_SUFFIX}" - ) - elseif(${_AWSSDK_LIB} STREQUAL "aws-lc") # We only need libcrypto from aws-lc - set(_AWSSDK_STATIC_LIBRARY - "${AWS_LC_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}crypto${CMAKE_STATIC_LIBRARY_SUFFIX}" - ) - endif() - if(${_AWSSDK_LIB} MATCHES "^aws-cpp-sdk-") - set(_AWSSDK_TARGET_NAME ${_AWSSDK_LIB}) - elseif(${_AWSSDK_LIB} STREQUAL "aws-lc") - set(_AWSSDK_TARGET_NAME AWS::crypto) - else() - set(_AWSSDK_TARGET_NAME AWS::${_AWSSDK_LIB}) - endif() - add_library(${_AWSSDK_TARGET_NAME} STATIC IMPORTED) - set_target_properties(${_AWSSDK_TARGET_NAME} PROPERTIES IMPORTED_LOCATION - ${_AWSSDK_STATIC_LIBRARY}) - target_include_directories(${_AWSSDK_TARGET_NAME} BEFORE - INTERFACE "${AWSSDK_INCLUDE_DIR}") - if(${_AWSSDK_LIB} STREQUAL "aws-lc") - set_target_properties(${_AWSSDK_TARGET_NAME} PROPERTIES IMPORTED_LOCATION - ${_AWSSDK_STATIC_LIBRARY}) - target_include_directories(${_AWSSDK_TARGET_NAME} BEFORE - INTERFACE "${AWS_LC_INCLUDE_DIR}") - endif() - set("${_AWSSDK_LIB_NAME_PREFIX}_STATIC_LIBRARY" ${_AWSSDK_STATIC_LIBRARY}) - - if(NOT ${_AWSSDK_LIB} STREQUAL "aws-lc") - # aws-lc only linked against s2n but not arrow - list(APPEND AWSSDK_LIBRARIES ${_AWSSDK_TARGET_NAME}) + list(APPEND + AWSSDK_PRODUCTS + aws-c-cal + aws-c-io + aws-c-event-stream + aws-c-sdkutils + aws-c-compression + aws-c-http + aws-c-mqtt + aws-c-auth + aws-c-s3 + aws-crt-cpp + aws-sdk-cpp) + set(AWS_SDK_CPP_SOURCE_URL "${AWSSDK_SOURCE_URL}") + set(ARROW_AWS_SDK_CPP_BUILD_SHA256_CHECKSUM "${ARROW_AWSSDK_BUILD_SHA256_CHECKSUM}") + foreach(AWSSDK_PRODUCT ${AWSSDK_PRODUCTS}) + # aws-c-cal -> + # AWS-C-CAL + string(TOUPPER "${AWSSDK_PRODUCT}" BASE_VARIABLE_NAME) + # AWS-C-CAL -> + # AWS_C_CAL + string(REGEX REPLACE "-" "_" BASE_VARIABLE_NAME "${BASE_VARIABLE_NAME}") + set(${BASE_VARIABLE_NAME}_DIFF_FILE + "${CMAKE_CURRENT_LIST_DIR}/${AWSSDK_PRODUCT}.diff") + if(EXISTS "${${BASE_VARIABLE_NAME}_DIFF_FILE}") + if(NOT PATCH) + find_program(PATCH patch REQUIRED) + endif() + set(${BASE_VARIABLE_NAME}_PATCH_COMMAND ${PATCH} -p1 -i + "${${BASE_VARIABLE_NAME}_DIFF_FILE}") endif() + fetchcontent_declare(${AWSSDK_PRODUCT} + ${FC_DECLARE_COMMON_OPTIONS} + PATCH_COMMAND ${${BASE_VARIABLE_NAME}_PATCH_COMMAND} + URL ${${BASE_VARIABLE_NAME}_SOURCE_URL} + URL_HASH "SHA256=${ARROW_${BASE_VARIABLE_NAME}_BUILD_SHA256_CHECKSUM}" + ) endforeach() - externalproject_add(aws_c_common_ep - ${EP_COMMON_OPTIONS} - URL ${AWS_C_COMMON_SOURCE_URL} - URL_HASH "SHA256=${ARROW_AWS_C_COMMON_BUILD_SHA256_CHECKSUM}" - PATCH_COMMAND ${AWS_C_COMMON_PATCH_COMMAND} - CMAKE_ARGS ${AWSSDK_COMMON_CMAKE_ARGS} - BUILD_BYPRODUCTS ${AWS_C_COMMON_STATIC_LIBRARY}) - add_dependencies(AWS::aws-c-common aws_c_common_ep) - - set(AWS_CHECKSUMS_CMAKE_ARGS ${AWSSDK_COMMON_CMAKE_ARGS}) - if(NOT WIN32) - # On non-Windows, always build in release mode. - # Especially with gcc, debug builds can fail with "asm constraint" errors: - # https://github.com/TileDB-Inc/TileDB/issues/1351 - list(APPEND AWS_CHECKSUMS_CMAKE_ARGS -DCMAKE_BUILD_TYPE=Release) - endif() - externalproject_add(aws_checksums_ep - ${EP_COMMON_OPTIONS} - URL ${AWS_CHECKSUMS_SOURCE_URL} - URL_HASH "SHA256=${ARROW_AWS_CHECKSUMS_BUILD_SHA256_CHECKSUM}" - CMAKE_ARGS ${AWS_CHECKSUMS_CMAKE_ARGS} - BUILD_BYPRODUCTS ${AWS_CHECKSUMS_STATIC_LIBRARY} - DEPENDS aws_c_common_ep) - add_dependencies(AWS::aws-checksums aws_checksums_ep) - - if("s2n-tls" IN_LIST _AWSSDK_LIBS) - # Remove unused directories to save build directory storage. - # 169MB -> 105MB - set(AWS_LC_PATCH_COMMAND ${CMAKE_COMMAND} -E) - if(CMAKE_VERSION VERSION_LESS 3.17) - list(APPEND AWS_LC_PATCH_COMMAND remove_directory) - else() - list(APPEND AWS_LC_PATCH_COMMAND rm -rf) - endif() - list(APPEND AWS_LC_PATCH_COMMAND fuzz) - - set(AWS_LC_C_FLAGS ${EP_C_FLAGS}) - string(APPEND AWS_LC_C_FLAGS " -Wno-error=overlength-strings -Wno-error=pedantic") - # Link time optimization is causing trouble like #34349 - string(REPLACE "-flto=auto" "" AWS_LC_C_FLAGS "${AWS_LC_C_FLAGS}") - string(REPLACE "-ffat-lto-objects" "" AWS_LC_C_FLAGS "${AWS_LC_C_FLAGS}") - - set(AWS_LC_CMAKE_ARGS ${AWSSDK_COMMON_CMAKE_ARGS}) - list(APPEND AWS_LC_CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${AWS_LC_PREFIX} - -DCMAKE_C_FLAGS=${AWS_LC_C_FLAGS}) - - externalproject_add(aws_lc_ep - ${EP_COMMON_OPTIONS} - URL ${AWS_LC_SOURCE_URL} - URL_HASH "SHA256=${ARROW_AWS_LC_BUILD_SHA256_CHECKSUM}" - PATCH_COMMAND ${AWS_LC_PATCH_COMMAND} - CMAKE_ARGS ${AWS_LC_CMAKE_ARGS} - BUILD_BYPRODUCTS ${AWS_LC_STATIC_LIBRARY}) - add_dependencies(AWS::crypto aws_lc_ep) - - set(S2N_TLS_C_FLAGS ${EP_C_FLAGS}) - # Link time optimization is causing trouble like #34349 - string(REPLACE "-flto=auto" "" S2N_TLS_C_FLAGS "${S2N_TLS_C_FLAGS}") - string(REPLACE "-ffat-lto-objects" "" S2N_TLS_C_FLAGS "${S2N_TLS_C_FLAGS}") - - set(S2N_TLS_CMAKE_ARGS ${AWSSDK_COMMON_CMAKE_ARGS}) - list(APPEND - S2N_TLS_CMAKE_ARGS - # internalize libcrypto to avoid name conflict with OpenSSL - -DS2N_INTERN_LIBCRYPTO=ON - # path to find crypto provided by aws-lc - -DCMAKE_PREFIX_PATH=${AWS_LC_PREFIX} - -DCMAKE_C_FLAGS=${S2N_TLS_C_FLAGS} - # paths to find crypto provided by aws-lc - -Dcrypto_INCLUDE_DIR=${AWS_LC_PREFIX}/include - -Dcrypto_LIBRARY=${AWS_LC_STATIC_LIBRARY} - -Dcrypto_STATIC_LIBRARY=${AWS_LC_STATIC_LIBRARY}) - - externalproject_add(s2n_tls_ep - ${EP_COMMON_OPTIONS} - URL ${S2N_TLS_SOURCE_URL} - URL_HASH "SHA256=${ARROW_S2N_TLS_BUILD_SHA256_CHECKSUM}" - CMAKE_ARGS ${S2N_TLS_CMAKE_ARGS} - BUILD_BYPRODUCTS ${S2N_TLS_STATIC_LIBRARY} - DEPENDS aws_lc_ep) - add_dependencies(AWS::s2n-tls s2n_tls_ep) - endif() - - externalproject_add(aws_c_cal_ep - ${EP_COMMON_OPTIONS} - URL ${AWS_C_CAL_SOURCE_URL} - URL_HASH "SHA256=${ARROW_AWS_C_CAL_BUILD_SHA256_CHECKSUM}" - CMAKE_ARGS ${AWSSDK_COMMON_CMAKE_ARGS} - BUILD_BYPRODUCTS ${AWS_C_CAL_STATIC_LIBRARY} - DEPENDS aws_c_common_ep) - add_dependencies(AWS::aws-c-cal aws_c_cal_ep) - - set(AWS_C_IO_DEPENDS aws_c_common_ep aws_c_cal_ep) - if(TARGET s2n_tls_ep) - list(APPEND AWS_C_IO_DEPENDS s2n_tls_ep) - endif() - externalproject_add(aws_c_io_ep - ${EP_COMMON_OPTIONS} - URL ${AWS_C_IO_SOURCE_URL} - URL_HASH "SHA256=${ARROW_AWS_C_IO_BUILD_SHA256_CHECKSUM}" - PATCH_COMMAND ${AWS_C_IO_PATCH_COMMAND} - CMAKE_ARGS ${AWSSDK_COMMON_CMAKE_ARGS} - BUILD_BYPRODUCTS ${AWS_C_IO_STATIC_LIBRARY} - DEPENDS ${AWS_C_IO_DEPENDS}) - add_dependencies(AWS::aws-c-io aws_c_io_ep) - - externalproject_add(aws_c_event_stream_ep - ${EP_COMMON_OPTIONS} - URL ${AWS_C_EVENT_STREAM_SOURCE_URL} - URL_HASH "SHA256=${ARROW_AWS_C_EVENT_STREAM_BUILD_SHA256_CHECKSUM}" - CMAKE_ARGS ${AWSSDK_COMMON_CMAKE_ARGS} - BUILD_BYPRODUCTS ${AWS_C_EVENT_STREAM_STATIC_LIBRARY} - DEPENDS aws_checksums_ep aws_c_io_ep) - add_dependencies(AWS::aws-c-event-stream aws_c_event_stream_ep) - - externalproject_add(aws_c_sdkutils_ep - ${EP_COMMON_OPTIONS} - URL ${AWS_C_SDKUTILS_SOURCE_URL} - URL_HASH "SHA256=${ARROW_AWS_C_SDKUTILS_BUILD_SHA256_CHECKSUM}" - CMAKE_ARGS ${AWSSDK_COMMON_CMAKE_ARGS} - BUILD_BYPRODUCTS ${AWS_C_SDKUTILS_STATIC_LIBRARY} - DEPENDS aws_c_common_ep) - add_dependencies(AWS::aws-c-sdkutils aws_c_sdkutils_ep) - - externalproject_add(aws_c_compression_ep - ${EP_COMMON_OPTIONS} - URL ${AWS_C_COMPRESSION_SOURCE_URL} - URL_HASH "SHA256=${ARROW_AWS_C_COMPRESSION_BUILD_SHA256_CHECKSUM}" - CMAKE_ARGS ${AWSSDK_COMMON_CMAKE_ARGS} - BUILD_BYPRODUCTS ${AWS_C_COMPRESSION_STATIC_LIBRARY} - DEPENDS aws_c_common_ep) - add_dependencies(AWS::aws-c-compression aws_c_compression_ep) - - externalproject_add(aws_c_http_ep - ${EP_COMMON_OPTIONS} - URL ${AWS_C_HTTP_SOURCE_URL} - URL_HASH "SHA256=${ARROW_AWS_C_HTTP_BUILD_SHA256_CHECKSUM}" - CMAKE_ARGS ${AWSSDK_COMMON_CMAKE_ARGS} - BUILD_BYPRODUCTS ${AWS_C_HTTP_STATIC_LIBRARY} - DEPENDS aws_c_io_ep aws_c_compression_ep) - add_dependencies(AWS::aws-c-http aws_c_http_ep) - - externalproject_add(aws_c_mqtt_ep - ${EP_COMMON_OPTIONS} - URL ${AWS_C_MQTT_SOURCE_URL} - URL_HASH "SHA256=${ARROW_AWS_C_MQTT_BUILD_SHA256_CHECKSUM}" - CMAKE_ARGS ${AWSSDK_COMMON_CMAKE_ARGS} - BUILD_BYPRODUCTS ${AWS_C_MQTT_STATIC_LIBRARY} - DEPENDS aws_c_http_ep) - add_dependencies(AWS::aws-c-mqtt aws_c_mqtt_ep) - - externalproject_add(aws_c_auth_ep - ${EP_COMMON_OPTIONS} - URL ${AWS_C_AUTH_SOURCE_URL} - URL_HASH "SHA256=${ARROW_AWS_C_AUTH_BUILD_SHA256_CHECKSUM}" - CMAKE_ARGS ${AWSSDK_COMMON_CMAKE_ARGS} - BUILD_BYPRODUCTS ${AWS_C_AUTH_STATIC_LIBRARY} - DEPENDS aws_c_sdkutils_ep aws_c_cal_ep aws_c_http_ep) - add_dependencies(AWS::aws-c-auth aws_c_auth_ep) - - externalproject_add(aws_c_s3_ep - ${EP_COMMON_OPTIONS} - URL ${AWS_C_S3_SOURCE_URL} - URL_HASH "SHA256=${ARROW_AWS_C_S3_BUILD_SHA256_CHECKSUM}" - CMAKE_ARGS ${AWSSDK_COMMON_CMAKE_ARGS} - BUILD_BYPRODUCTS ${AWS_C_S3_STATIC_LIBRARY} - DEPENDS aws_checksums_ep aws_c_auth_ep) - add_dependencies(AWS::aws-c-s3 aws_c_s3_ep) - - externalproject_add(aws_crt_cpp_ep - ${EP_COMMON_OPTIONS} - URL ${AWS_CRT_CPP_SOURCE_URL} - URL_HASH "SHA256=${ARROW_AWS_CRT_CPP_BUILD_SHA256_CHECKSUM}" - CMAKE_ARGS ${AWSSDK_CMAKE_ARGS} - BUILD_BYPRODUCTS ${AWS_CRT_CPP_STATIC_LIBRARY} - DEPENDS aws_c_auth_ep - aws_c_cal_ep - aws_c_common_ep - aws_c_event_stream_ep - aws_c_http_ep - aws_c_io_ep - aws_c_mqtt_ep - aws_c_s3_ep - aws_checksums_ep) - add_dependencies(AWS::aws-crt-cpp aws_crt_cpp_ep) - - externalproject_add(awssdk_ep - ${EP_COMMON_OPTIONS} - URL ${AWSSDK_SOURCE_URL} - URL_HASH "SHA256=${ARROW_AWSSDK_BUILD_SHA256_CHECKSUM}" - PATCH_COMMAND ${AWSSDK_PATCH_COMMAND} - CMAKE_ARGS ${AWSSDK_CMAKE_ARGS} - BUILD_BYPRODUCTS ${AWS_CPP_SDK_COGNITO_IDENTITY_STATIC_LIBRARY} - ${AWS_CPP_SDK_CORE_STATIC_LIBRARY} - ${AWS_CPP_SDK_IDENTITY_MANAGEMENT_STATIC_LIBRARY} - ${AWS_CPP_SDK_S3_STATIC_LIBRARY} - ${AWS_CPP_SDK_STS_STATIC_LIBRARY} - DEPENDS aws_crt_cpp_ep) - foreach(_AWSSDK_LIB ${_AWSSDK_LIBS}) - if(${_AWSSDK_LIB} MATCHES "^aws-cpp-sdk-") - add_dependencies(${_AWSSDK_LIB} awssdk_ep) + prepare_fetchcontent() + set(BUILD_DEPS + OFF + CACHE BOOL "" FORCE) + set(BUILD_ONLY + "" + CACHE STRING "" FORCE) + list(APPEND + BUILD_ONLY + config + core + identity-management + s3 + sts + transfer) + set(IN_SOURCE_BUILD + ON + CACHE BOOL "" FORCE) + set(MINIMIZE_SIZE + ON + CACHE BOOL "" FORCE) + set(USE_OPENSSL + ON + CACHE BOOL "" FORCE) + + set(AWSSDK_LINK_LIBRARIES) + foreach(AWSSDK_PRODUCT ${AWSSDK_PRODUCTS}) + fetchcontent_makeavailable(${AWSSDK_PRODUCT}) + list(PREPEND CMAKE_MODULE_PATH "${${AWSSDK_PRODUCT}_SOURCE_DIR}/cmake") + if(NOT "${AWSSDK_PRODUCT}" STREQUAL "aws-sdk-cpp") + if("${AWSSDK_PRODUCT}" STREQUAL "s2n-tls") + list(PREPEND AWSSDK_LINK_LIBRARIES s2n) + else() + list(PREPEND AWSSDK_LINK_LIBRARIES ${AWSSDK_PRODUCT}) + endif() endif() endforeach() - - set(AWSSDK_VENDORED TRUE) - list(APPEND ARROW_BUNDLED_STATIC_LIBS ${AWSSDK_LIBRARIES}) - set(AWSSDK_LINK_LIBRARIES ${AWSSDK_LIBRARIES}) - if(UNIX) - # on Linux and macOS curl seems to be required - set_property(TARGET aws-cpp-sdk-core - APPEND - PROPERTY INTERFACE_LINK_LIBRARIES CURL::libcurl) - set_property(TARGET AWS::aws-c-cal - APPEND - PROPERTY INTERFACE_LINK_LIBRARIES OpenSSL::Crypto OpenSSL::SSL) - if(APPLE) - set_property(TARGET AWS::aws-c-cal - APPEND - PROPERTY INTERFACE_LINK_LIBRARIES "-framework Security") - endif() - if(ZLIB_VENDORED) - set_property(TARGET aws-cpp-sdk-core - APPEND - PROPERTY INTERFACE_LINK_LIBRARIES ZLIB::ZLIB) - add_dependencies(awssdk_ep zlib_ep) - endif() - set_property(TARGET AWS::aws-c-io - APPEND - PROPERTY INTERFACE_LINK_LIBRARIES ${CMAKE_DL_LIBS}) - elseif(WIN32) - set_property(TARGET aws-cpp-sdk-core - APPEND - PROPERTY INTERFACE_LINK_LIBRARIES - "winhttp.lib" - "bcrypt.lib" - "wininet.lib" - "userenv.lib" - "version.lib") - set_property(TARGET AWS::aws-c-cal - APPEND - PROPERTY INTERFACE_LINK_LIBRARIES - "bcrypt.lib" - "ncrypt.lib" - "Secur32.lib" - "Shlwapi.lib") - set_property(TARGET AWS::aws-c-io - APPEND - PROPERTY INTERFACE_LINK_LIBRARIES "crypt32.lib") - endif() - - # AWSSDK is static-only build -endmacro() + list(PREPEND + AWSSDK_LINK_LIBRARIES + aws-cpp-sdk-identity-management + aws-cpp-sdk-sts + aws-cpp-sdk-cognito-identity + aws-cpp-sdk-s3 + aws-cpp-sdk-core) + + set(AWSSDK_VENDORED + TRUE + PARENT_SCOPE) + list(APPEND ARROW_BUNDLED_STATIC_LIBS ${AWSSDK_LINK_LIBRARIES}) + set(ARROW_BUNDLED_STATIC_LIBS + ${ARROW_BUNDLED_STATIC_LIBS} + PARENT_SCOPE) + set(AWSSDK_LINK_LIBRARIES + ${AWSSDK_LINK_LIBRARIES} + PARENT_SCOPE) +endfunction() if(ARROW_S3) resolve_dependency(AWSSDK HAVE_ALT TRUE) @@ -5510,16 +5186,6 @@ if(ARROW_S3) endif() endif() endif() - - if(APPLE) - # CoreFoundation's path is hardcoded in the CMake files provided by - # aws-sdk-cpp to use the macOS SDK provided by XCode which makes - # XCode a hard dependency. Command Line Tools is often used instead - # of the full XCode suite, so let the linker to find it. - set_target_properties(AWS::aws-c-common - PROPERTIES INTERFACE_LINK_LIBRARIES - "-pthread;pthread;-framework CoreFoundation") - endif() endif() # ---------------------------------------------------------------------- diff --git a/cpp/thirdparty/versions.txt b/cpp/thirdparty/versions.txt index 1fc53c4d5e6..a1b04bffeb7 100644 --- a/cpp/thirdparty/versions.txt +++ b/cpp/thirdparty/versions.txt @@ -25,34 +25,34 @@ ARROW_ABSL_BUILD_VERSION=20211102.0 ARROW_ABSL_BUILD_SHA256_CHECKSUM=dcf71b9cba8dc0ca9940c4b316a0c796be8fab42b070bb6b7cab62b48f0e66c4 -ARROW_AWS_C_AUTH_BUILD_VERSION=v0.6.22 -ARROW_AWS_C_AUTH_BUILD_SHA256_CHECKSUM=691a6b4418afcd3dc141351b6ad33fccd8e3ff84df0e9e045b42295d284ee14c -ARROW_AWS_C_CAL_BUILD_VERSION=v0.5.20 -ARROW_AWS_C_CAL_BUILD_SHA256_CHECKSUM=acc352359bd06f8597415c366cf4ec4f00d0b0da92d637039a73323dd55b6cd0 -ARROW_AWS_C_COMMON_BUILD_VERSION=v0.8.9 -ARROW_AWS_C_COMMON_BUILD_SHA256_CHECKSUM=2f3fbaf7c38eae5a00e2a816d09b81177f93529ae8ba1b82dc8f31407565327a -ARROW_AWS_C_COMPRESSION_BUILD_VERSION=v0.2.16 -ARROW_AWS_C_COMPRESSION_BUILD_SHA256_CHECKSUM=044b1dbbca431a07bde8255ef9ec443c300fc60d4c9408d4b862f65e496687f4 -ARROW_AWS_C_EVENT_STREAM_BUILD_VERSION=v0.2.18 -ARROW_AWS_C_EVENT_STREAM_BUILD_SHA256_CHECKSUM=310ca617f713bf664e4c7485a3d42c1fb57813abd0107e49790d107def7cde4f -ARROW_AWS_C_HTTP_BUILD_VERSION=v0.7.3 -ARROW_AWS_C_HTTP_BUILD_SHA256_CHECKSUM=07e16c6bf5eba6f0dea96b6f55eae312a7c95b736f4d2e4a210000f45d8265ae -ARROW_AWS_C_IO_BUILD_VERSION=v0.13.14 -ARROW_AWS_C_IO_BUILD_SHA256_CHECKSUM=12b66510c3d9a4f7e9b714e9cfab2a5bf835f8b9ce2f909d20ae2a2128608c71 -ARROW_AWS_C_MQTT_BUILD_VERSION=v0.8.4 -ARROW_AWS_C_MQTT_BUILD_SHA256_CHECKSUM=232eeac63e72883d460c686a09b98cdd811d24579affac47c5c3f696f956773f -ARROW_AWS_C_S3_BUILD_VERSION=v0.2.3 -ARROW_AWS_C_S3_BUILD_SHA256_CHECKSUM=a00b3c9f319cd1c9aa2c3fa15098864df94b066dcba0deaccbb3caa952d902fe -ARROW_AWS_C_SDKUTILS_BUILD_VERSION=v0.1.6 -ARROW_AWS_C_SDKUTILS_BUILD_SHA256_CHECKSUM=8a2951344b2fb541eab1e9ca17c18a7fcbfd2aaff4cdd31d362d1fad96111b91 -ARROW_AWS_CHECKSUMS_BUILD_VERSION=v0.1.13 -ARROW_AWS_CHECKSUMS_BUILD_SHA256_CHECKSUM=0f897686f1963253c5069a0e495b85c31635ba146cd3ac38cc2ea31eaf54694d -ARROW_AWS_CRT_CPP_BUILD_VERSION=v0.18.16 -ARROW_AWS_CRT_CPP_BUILD_SHA256_CHECKSUM=9e69bc1dc4b50871d1038aa9ff6ddeb4c9b28f7d6b5e5b1b69041ccf50a13483 -ARROW_AWS_LC_BUILD_VERSION=v1.3.0 -ARROW_AWS_LC_BUILD_SHA256_CHECKSUM=ae96a3567161552744fc0cae8b4d68ed88b1ec0f3d3c98700070115356da5a37 -ARROW_AWSSDK_BUILD_VERSION=1.10.55 -ARROW_AWSSDK_BUILD_SHA256_CHECKSUM=2d552fb1a84bef4a9b65e34aa7031851ed2aef5319e02cc6e4cb735c48aa30de +ARROW_AWS_C_AUTH_BUILD_VERSION=v0.8.0 +ARROW_AWS_C_AUTH_BUILD_SHA256_CHECKSUM=217a0ebf8d7c5ad7e5f5ae814c2a371042164b64b4b9330c1c4bb2c6db1dbd33 +ARROW_AWS_C_CAL_BUILD_VERSION=v0.8.1 +ARROW_AWS_C_CAL_BUILD_SHA256_CHECKSUM=4d603641758ef350c3e5401184804e8a6bba4aa5294593cc6228b0dca77b22f5 +ARROW_AWS_C_COMMON_BUILD_VERSION=v0.10.6 +ARROW_AWS_C_COMMON_BUILD_SHA256_CHECKSUM=d0acbabc786035d41791c3a2f45dbeda31d9693521ee746dc1375d6380eb912b +ARROW_AWS_C_COMPRESSION_BUILD_VERSION=v0.3.0 +ARROW_AWS_C_COMPRESSION_BUILD_SHA256_CHECKSUM=7e5d7308d1dbb1801eae9356ef65558f707edf33660dd6443c985db9474725eb +ARROW_AWS_C_EVENT_STREAM_BUILD_VERSION=v0.5.0 +ARROW_AWS_C_EVENT_STREAM_BUILD_SHA256_CHECKSUM=3a53a9d05f9e2fd06036a12854a8b4f05a0c4858bb5b8df8a30edba9de8532b5 +ARROW_AWS_C_HTTP_BUILD_VERSION=v0.9.2 +ARROW_AWS_C_HTTP_BUILD_SHA256_CHECKSUM=328013ebc2b5725326cac01941041eec1e1010058c60709da2c23aa8fb967370 +ARROW_AWS_C_IO_BUILD_VERSION=v0.15.3 +ARROW_AWS_C_IO_BUILD_SHA256_CHECKSUM=d8cb4d7d3ec4fb27cbce158d6823a1f2f5d868e116f1d6703db2ab8159343c3f +ARROW_AWS_C_MQTT_BUILD_VERSION=v0.11.0 +ARROW_AWS_C_MQTT_BUILD_SHA256_CHECKSUM=3854664c13896b6de3d56412f928435a4933259cb7fe62b10c1f497e6999333c +ARROW_AWS_C_S3_BUILD_VERSION=v0.7.9 +ARROW_AWS_C_S3_BUILD_SHA256_CHECKSUM=1942161f9b9d656604a0efeb1387ad99d67d1f2d09154e0395c8bf5da2a39b7a +ARROW_AWS_C_SDKUTILS_BUILD_VERSION=v0.2.2 +ARROW_AWS_C_SDKUTILS_BUILD_SHA256_CHECKSUM=75defbfd4d896b8bdc0790bd25d854218acae61b9409d1956d33832924b82045 +ARROW_AWS_CHECKSUMS_BUILD_VERSION=v0.2.2 +ARROW_AWS_CHECKSUMS_BUILD_SHA256_CHECKSUM=96acfea882c06acd5571c845e4968892d6ffc0fae81b31a0b1565100049743b2 +ARROW_AWS_CRT_CPP_BUILD_VERSION=v0.29.9 +ARROW_AWS_CRT_CPP_BUILD_SHA256_CHECKSUM=d445ab7a26c03a0c0cbb9d82203ee32a56c762a3cef1874783783431b8eb015a +ARROW_AWS_LC_BUILD_VERSION=v1.42.0 +ARROW_AWS_LC_BUILD_SHA256_CHECKSUM=8537025ebfd3884830f494166ae5f72f8aaa203d49abe650a0560a7ffedf359e +ARROW_AWSSDK_BUILD_VERSION=1.11.488 +ARROW_AWSSDK_BUILD_SHA256_CHECKSUM=e3729c68c07724566a7e25cc68430b57af18dee421dcbc7cda0e9ad2ba0b486e # Despite the confusing version name this is still the whole Azure SDK for C++ including core, keyvault, storage-common, etc. ARROW_AZURE_SDK_BUILD_VERSION=azure-identity_1.9.0 ARROW_AZURE_SDK_BUILD_SHA256_CHECKSUM=97065bfc971ac8df450853ce805f820f52b59457bd7556510186a1569502e4a1 @@ -105,8 +105,8 @@ ARROW_SNAPPY_BUILD_VERSION=1.2.2 ARROW_SNAPPY_BUILD_SHA256_CHECKSUM=90f74bc1fbf78a6c56b3c4a082a05103b3a56bb17bca1a27e052ea11723292dc ARROW_SUBSTRAIT_BUILD_VERSION=v0.44.0 ARROW_SUBSTRAIT_BUILD_SHA256_CHECKSUM=f989a862f694e7dbb695925ddb7c4ce06aa6c51aca945105c075139aed7e55a2 -ARROW_S2N_TLS_BUILD_VERSION=v1.3.35 -ARROW_S2N_TLS_BUILD_SHA256_CHECKSUM=9d32b26e6bfcc058d98248bf8fc231537e347395dd89cf62bb432b55c5da990d +ARROW_S2N_TLS_BUILD_VERSION=v1.5.11 +ARROW_S2N_TLS_BUILD_SHA256_CHECKSUM=5690f030da35f86e3b5d61d1de150b5b52c84eef383799f7a706bdf21227417e ARROW_THRIFT_BUILD_VERSION=0.20.0 ARROW_THRIFT_BUILD_SHA256_CHECKSUM=b5d8311a779470e1502c027f428a1db542f5c051c8e1280ccd2163fa935ff2d6 ARROW_UTF8PROC_BUILD_VERSION=v2.10.0 From 3addfc799f1b4ef4d484906c8414e6b34ea721c9 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Mon, 20 Jan 2025 17:37:45 +0900 Subject: [PATCH 02/52] Add missing diff --- cpp/cmake_modules/aws-crt-cpp.diff | 28 ++++++++++++++++++++++++++++ cpp/cmake_modules/aws-sdk-cpp.diff | 28 ++++++++++++++++++++++++++++ 2 files changed, 56 insertions(+) create mode 100644 cpp/cmake_modules/aws-crt-cpp.diff create mode 100644 cpp/cmake_modules/aws-sdk-cpp.diff diff --git a/cpp/cmake_modules/aws-crt-cpp.diff b/cpp/cmake_modules/aws-crt-cpp.diff new file mode 100644 index 00000000000..c8fdc612927 --- /dev/null +++ b/cpp/cmake_modules/aws-crt-cpp.diff @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +diff -ru aws-crt-cpp-src.orig/CMakeLists.txt aws-crt-cpp-src/CMakeLists.txt +--- aws-crt-cpp-src.orig/CMakeLists.txt 2025-01-20 15:06:08.602765260 +0900 ++++ aws-crt-cpp-src/CMakeLists.txt 2025-01-20 15:07:30.879403510 +0900 +@@ -133,5 +133,4 @@ + set(BUILD_TESTING ${BUILD_TESTING_PREV}) + else() + include(AwsFindPackage) +- set(IN_SOURCE_BUILD OFF) + endif() + + include(AwsCFlags) diff --git a/cpp/cmake_modules/aws-sdk-cpp.diff b/cpp/cmake_modules/aws-sdk-cpp.diff new file mode 100644 index 00000000000..0ecb248de8d --- /dev/null +++ b/cpp/cmake_modules/aws-sdk-cpp.diff @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +diff -ru aws-sdk-cpp-src.orig/CMakeLists.txt aws-sdk-cpp-src/CMakeLists.txt +--- aws-sdk-cpp-src.orig/CMakeLists.txt 2025-01-20 14:51:21.180004943 +0900 ++++ aws-sdk-cpp-src/CMakeLists.txt 2025-01-20 15:00:20.256077009 +0900 +@@ -245,7 +245,6 @@ + # Append that generated list to the module search path + list(APPEND CMAKE_MODULE_PATH ${AWS_MODULE_PATH}) + include(AwsFindPackage) +- set(IN_SOURCE_BUILD OFF) + endif () + aws_use_package(aws-crt-cpp) + aws_use_package(aws-c-http) From 0de54aafd07f3e44eae7f8c03ebc9306d3af771a Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Mon, 20 Jan 2025 18:16:48 +0900 Subject: [PATCH 03/52] Fix variable name --- cpp/CMakeLists.txt | 5 +++++ cpp/cmake_modules/ThirdpartyToolchain.cmake | 10 ++++------ 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 60a77c815e3..7a1a94a99b5 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -56,6 +56,11 @@ cmake_policy(SET CMP0068 NEW) # find_package() uses _ROOT variables. cmake_policy(SET CMP0074 NEW) +# https://cmake.org/cmake/help/latest/policy/CMP0077.html +# +# option() honors normal variables. +cmake_policy(SET CMP0077 NEW) + # https://cmake.org/cmake/help/latest/policy/CMP0091.html # # MSVC runtime library flags are selected by an abstraction. diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index c331ee8465e..d1754306bf8 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -1022,11 +1022,12 @@ macro(prepare_fetchcontent) set(BUILD_STATIC_LIBS ON) set(CMAKE_COMPILE_WARNING_AS_ERROR FALSE) set(CMAKE_EXPORT_NO_PACKAGE_REGISTRY TRUE) - set(CMAKE_MACOSX_RPATH ${ARROW_INSTALL_NAME_RPATH}) # We set CMAKE_POLICY_VERSION_MINIMUM temporarily due to failures with CMake 4 # We should remove it once we have updated the dependencies: # https://github.com/apache/arrow/issues/45985 set(CMAKE_POLICY_VERSION_MINIMUM 3.5) + set(CMAKE_MACOSX_RPATH ${ARROW_INSTALL_NAME_RPATH}) + set(ENABLE_TESTING OFF) if(MSVC) string(REPLACE "/WX" "" CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG}") @@ -1035,7 +1036,6 @@ macro(prepare_fetchcontent) string(APPEND CMAKE_C_FLAGS_DEBUG " -Wno-error") string(APPEND CMAKE_CXX_FLAGS_DEBUG " -Wno-error") endif() - set(ENABLE_TESTING OFF) endmacro() # ---------------------------------------------------------------------- @@ -5062,12 +5062,10 @@ function(build_awssdk) message(STATUS "Building AWS SDK for C++ from source") set(AWSSDK_PRODUCTS aws-c-common aws-checksums) - # aws-lc and s2n-tls only needed on Linux. + # s2n-tls only needed on Linux. # We can use LINUX with CMake 3.25 or later. if(UNIX AND NOT APPLE) - list(APPEND AWSSDK_PRODUCTS - # aws-lc - s2n-tls) + list(APPEND AWSSDK_PRODUCTS s2n-tls) endif() list(APPEND AWSSDK_PRODUCTS From d222e25b532a8b4258d2ad640dcea3a5fa86eaea Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Wed, 22 Jan 2025 09:44:01 +0900 Subject: [PATCH 04/52] Update versions --- cpp/thirdparty/README.md | 16 +++++ cpp/thirdparty/update.rb | 130 ++++++++++++++++++++++++++++++++++++ cpp/thirdparty/versions.txt | 8 +-- 3 files changed, 150 insertions(+), 4 deletions(-) create mode 100755 cpp/thirdparty/update.rb diff --git a/cpp/thirdparty/README.md b/cpp/thirdparty/README.md index 81806810530..830371bad8a 100644 --- a/cpp/thirdparty/README.md +++ b/cpp/thirdparty/README.md @@ -23,3 +23,19 @@ See the "Build Dependency Management" section in the [C++ Developer Documentation][1]. [1]: https://github.com/apache/arrow/blob/main/docs/source/developers/cpp/building.rst + +## Update versions automatically + +There is a convenient script that update versions in `versions.txt` to +the latest version automatically. You can use it like the following: + +```console +dev/release/update.rb PRODUCT_PATTERN1 PRODUCT_PATTERN2 ... +``` + +For example, you can update AWS SDK for C++ related products' versions +by the following command line: + +```console +dev/release/update.rb "AWS*" "S2N*" +``` diff --git a/cpp/thirdparty/update.rb b/cpp/thirdparty/update.rb new file mode 100755 index 00000000000..5a063bcc83a --- /dev/null +++ b/cpp/thirdparty/update.rb @@ -0,0 +1,130 @@ +#!/usr/bin/env ruby +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +require "digest/sha2" +require "json" +require "open-uri" + +if ARGV.empty? + puts("Usage: #{$0} PRODUCT_PATTERN1 PRODUCT_PATTERN2 ...") + puts(" e.g.: #{$0} 'AWS*' 'S2N*'.") + exit(false) +end + +def parse_versions_txt_content(content) + products = {} + content.each_line(chomp: true) do |line| + case line + when /\AARROW_([A-Za-z0-9_-]+)_BUILD_VERSION=(.+?)\z/ + product = Regexp.last_match[1] + version = Regexp.last_match[2] + products[product] = {version: version} + when /\AARROW_([A-Za-z0-9_-]+)_BUILD_SHA256_CHECKSUM=(.+?)\z/ + product = Regexp.last_match[1] + checksum = Regexp.last_match[2] + products[product][:checksum] = checksum + when /\A "ARROW_([A-Za-z0-9_-]+)_URL (?:\S+) (\S+)"\z/ + product = Regexp.last_match[1] + url_template = Regexp.last_match[2] + url_template.gsub!(/\${.+?}/) do |matched| + if matched.end_with?("//./_}") + "%{version_underscore}" + else + "%{version}" + end + end + products[product][:url_template] = url_template + end + end + products +end + +def update_product_github(product, metadata, repository) + version = metadata[:version] + tags_url = "https://api.github.com/repos/#{repository}/tags" + tags = URI.open(tags_url) do |response| + JSON.parse(response.read) + end + latest_tag_name = tags[0]["name"] + if latest_tag_name.start_with?("v") + if metadata[:version].start_with?("v") + latest_version = latest_tag_name + else + latest_version = latest_tag_name[1..-1] + end + else + latest_version = latest_tag_name + end + return if version == latest_version + + url_template = metadata[:url_template] + url = url_template % { + version: latest_version, + version_underscore: latest_version.gsub(".", "_"), + } + $stderr.puts("Updating #{product}: #{version} -> #{latest_version}") + metadata[:version] = latest_version + URI.open(url, "rb") do |response| + metadata[:checksum] = Digest::SHA256.hexdigest(response.read) + end + $stderr.puts(" Checksum: #{metadata[:checksum]}") +end + +def update_product(product, metadata) + url_template = metadata[:url_template] + if url_template.nil? + $stderr.puts("#{product} isn't supported " + + "because there is no associated URL") + return + end + + case url_template + when /\Ahttps:\/\/github.com\/((?:[^\/]+)\/(?:[^\/]+))\// + github_repository = Regexp.last_match[1] + update_product_github(product, metadata, github_repository) + else + $stderr.puts("TODO: #{product} isn't supported yet: #{url_template}") + end +end + +def update_versions_txt_content!(content, products) + products.each do |product, metadata| + prefix = "ARROW_#{Regexp.escape(product)}" + content.gsub!(/^#{prefix}_BUILD_VERSION=.*$/) do + "ARROW_#{product}_BUILD_VERSION=#{metadata[:version]}" + end + content.gsub!(/^#{prefix}_BUILD_SHA256_CHECKSUM=.*?$/) do + "ARROW_#{product}_BUILD_SHA256_CHECKSUM=#{metadata[:checksum]}" + end + end +end + +versions_txt = File.join(__dir__, "versions.txt") +versions_txt_content = File.read(versions_txt) +products = parse_versions_txt_content(versions_txt_content) +ARGV.each do |pattern| + target_products = products.filter do |product, _| + File.fnmatch?(pattern, product) + end + target_products.each do |product, metadata| + update_product(product, metadata) + end +end +update_versions_txt_content!(versions_txt_content, products) +File.write(versions_txt, versions_txt_content) diff --git a/cpp/thirdparty/versions.txt b/cpp/thirdparty/versions.txt index a1b04bffeb7..1b491a01ea1 100644 --- a/cpp/thirdparty/versions.txt +++ b/cpp/thirdparty/versions.txt @@ -25,8 +25,8 @@ ARROW_ABSL_BUILD_VERSION=20211102.0 ARROW_ABSL_BUILD_SHA256_CHECKSUM=dcf71b9cba8dc0ca9940c4b316a0c796be8fab42b070bb6b7cab62b48f0e66c4 -ARROW_AWS_C_AUTH_BUILD_VERSION=v0.8.0 -ARROW_AWS_C_AUTH_BUILD_SHA256_CHECKSUM=217a0ebf8d7c5ad7e5f5ae814c2a371042164b64b4b9330c1c4bb2c6db1dbd33 +ARROW_AWS_C_AUTH_BUILD_VERSION=v0.8.1 +ARROW_AWS_C_AUTH_BUILD_SHA256_CHECKSUM=15241d7284aa0ac552589b61d04be455413af76fb2e1f13084a784a41f5faee5 ARROW_AWS_C_CAL_BUILD_VERSION=v0.8.1 ARROW_AWS_C_CAL_BUILD_SHA256_CHECKSUM=4d603641758ef350c3e5401184804e8a6bba4aa5294593cc6228b0dca77b22f5 ARROW_AWS_C_COMMON_BUILD_VERSION=v0.10.6 @@ -51,8 +51,8 @@ ARROW_AWS_CRT_CPP_BUILD_VERSION=v0.29.9 ARROW_AWS_CRT_CPP_BUILD_SHA256_CHECKSUM=d445ab7a26c03a0c0cbb9d82203ee32a56c762a3cef1874783783431b8eb015a ARROW_AWS_LC_BUILD_VERSION=v1.42.0 ARROW_AWS_LC_BUILD_SHA256_CHECKSUM=8537025ebfd3884830f494166ae5f72f8aaa203d49abe650a0560a7ffedf359e -ARROW_AWSSDK_BUILD_VERSION=1.11.488 -ARROW_AWSSDK_BUILD_SHA256_CHECKSUM=e3729c68c07724566a7e25cc68430b57af18dee421dcbc7cda0e9ad2ba0b486e +ARROW_AWSSDK_BUILD_VERSION=1.11.489 +ARROW_AWSSDK_BUILD_SHA256_CHECKSUM=89ce525c735243a043c5a31eee5dc082fc976b9b5a4b0b66813eed48c97abbfa # Despite the confusing version name this is still the whole Azure SDK for C++ including core, keyvault, storage-common, etc. ARROW_AZURE_SDK_BUILD_VERSION=azure-identity_1.9.0 ARROW_AZURE_SDK_BUILD_SHA256_CHECKSUM=97065bfc971ac8df450853ce805f820f52b59457bd7556510186a1569502e4a1 From fb53737c332a3a8e32fa0c60ff786dc67d3f5ca0 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Wed, 22 Jan 2025 09:53:38 +0900 Subject: [PATCH 05/52] Use aws-lc to intern symbols in s2n-tls --- cpp/cmake_modules/ThirdpartyToolchain.cmake | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index d1754306bf8..b92c7b1c7c8 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -5062,10 +5062,10 @@ function(build_awssdk) message(STATUS "Building AWS SDK for C++ from source") set(AWSSDK_PRODUCTS aws-c-common aws-checksums) - # s2n-tls only needed on Linux. + # aws-lc and s2n-tls only needed on Linux. # We can use LINUX with CMake 3.25 or later. if(UNIX AND NOT APPLE) - list(APPEND AWSSDK_PRODUCTS s2n-tls) + list(APPEND AWSSDK_PRODUCTS aws-lc s2n-tls) endif() list(APPEND AWSSDK_PRODUCTS @@ -5131,6 +5131,14 @@ function(build_awssdk) ON CACHE BOOL "" FORCE) + # For s2n-tls + set(crypto_STATIC_LIBRARY + "$" + CACHE STRING "" FORCE) + set(S2N_INTERN_LIBCRYPTO + ON + CACHE BOOL "" FORCE) + set(AWSSDK_LINK_LIBRARIES) foreach(AWSSDK_PRODUCT ${AWSSDK_PRODUCTS}) fetchcontent_makeavailable(${AWSSDK_PRODUCT}) From 885f56ecfdd5b9575f8926914353a0a535ab3303 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Wed, 22 Jan 2025 09:58:11 +0900 Subject: [PATCH 06/52] Disable needless components --- cpp/cmake_modules/ThirdpartyToolchain.cmake | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index b92c7b1c7c8..fcbd606023e 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -5121,6 +5121,9 @@ function(build_awssdk) s3 sts transfer) + set(BUILD_TOOL + OFF + CACHE BOOL "" FORCE) set(IN_SOURCE_BUILD ON CACHE BOOL "" FORCE) @@ -5131,6 +5134,14 @@ function(build_awssdk) ON CACHE BOOL "" FORCE) + # For aws-lc + set(DISABLE_PERL + ON + CACHE BOOL "" FORCE) + set(DISABLE_GO + ON + CACHE BOOL "" FORCE) + # For s2n-tls set(crypto_STATIC_LIBRARY "$" From 823279f50ae8dc6ff83e1991de7f2cda3da6bab4 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Wed, 22 Jan 2025 10:38:12 +0900 Subject: [PATCH 07/52] Add workaround for s2n-tls --- cpp/cmake_modules/ThirdpartyToolchain.cmake | 29 +++++++++++++++------ 1 file changed, 21 insertions(+), 8 deletions(-) diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index fcbd606023e..c098f08c243 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -5089,14 +5089,19 @@ function(build_awssdk) # AWS-C-CAL -> # AWS_C_CAL string(REGEX REPLACE "-" "_" BASE_VARIABLE_NAME "${BASE_VARIABLE_NAME}") - set(${BASE_VARIABLE_NAME}_DIFF_FILE - "${CMAKE_CURRENT_LIST_DIR}/${AWSSDK_PRODUCT}.diff") - if(EXISTS "${${BASE_VARIABLE_NAME}_DIFF_FILE}") - if(NOT PATCH) - find_program(PATCH patch REQUIRED) + if(AWSSDK_PRODUCT STREQUAL "s2n-tls") + set(${BASE_VARIABLE_NAME}_PATCH_COMMAND + ${CMAKE_COMMAND} -E rm tests/features/S2N_LIBCRYPTO_SUPPORTS_ENGINE.c) + else() + set(${BASE_VARIABLE_NAME}_DIFF_FILE + "${CMAKE_CURRENT_LIST_DIR}/${AWSSDK_PRODUCT}.diff") + if(EXISTS "${${BASE_VARIABLE_NAME}_DIFF_FILE}") + if(NOT PATCH) + find_program(PATCH patch REQUIRED) + endif() + set(${BASE_VARIABLE_NAME}_PATCH_COMMAND ${PATCH} -p1 -i + "${${BASE_VARIABLE_NAME}_DIFF_FILE}") endif() - set(${BASE_VARIABLE_NAME}_PATCH_COMMAND ${PATCH} -p1 -i - "${${BASE_VARIABLE_NAME}_DIFF_FILE}") endif() fetchcontent_declare(${AWSSDK_PRODUCT} ${FC_DECLARE_COMMON_OPTIONS} @@ -5146,6 +5151,9 @@ function(build_awssdk) set(crypto_STATIC_LIBRARY "$" CACHE STRING "" FORCE) + set(crypto_INCLUDE_DIR + "$" + CACHE STRING "" FORCE) set(S2N_INTERN_LIBCRYPTO ON CACHE BOOL "" FORCE) @@ -5155,8 +5163,13 @@ function(build_awssdk) fetchcontent_makeavailable(${AWSSDK_PRODUCT}) list(PREPEND CMAKE_MODULE_PATH "${${AWSSDK_PRODUCT}_SOURCE_DIR}/cmake") if(NOT "${AWSSDK_PRODUCT}" STREQUAL "aws-sdk-cpp") - if("${AWSSDK_PRODUCT}" STREQUAL "s2n-tls") + if("${AWSSDK_PRODUCT}" STREQUAL "aws-lc") + list(PREPEND AWSSDK_LINK_LIBRARIES ssl) + elseif("${AWSSDK_PRODUCT}" STREQUAL "s2n-tls") list(PREPEND AWSSDK_LINK_LIBRARIES s2n) + # Workaround: S2N_INTERN_LIBCRYPTO doesn't support + # out-of-source build. + file(MAKE_DIRECTORY ${s2n-tls_BINARY_DIR}/lib) else() list(PREPEND AWSSDK_LINK_LIBRARIES ${AWSSDK_PRODUCT}) endif() From d51615b897d49d2a4dfd82153bd4780511f1e99e Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Wed, 22 Jan 2025 10:46:57 +0900 Subject: [PATCH 08/52] Use remove for old CMake --- cpp/cmake_modules/ThirdpartyToolchain.cmake | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index c098f08c243..73a8e7b6685 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -5090,8 +5090,13 @@ function(build_awssdk) # AWS_C_CAL string(REGEX REPLACE "-" "_" BASE_VARIABLE_NAME "${BASE_VARIABLE_NAME}") if(AWSSDK_PRODUCT STREQUAL "s2n-tls") - set(${BASE_VARIABLE_NAME}_PATCH_COMMAND - ${CMAKE_COMMAND} -E rm tests/features/S2N_LIBCRYPTO_SUPPORTS_ENGINE.c) + if(CMAKE_VERSION VERSION_LESS 3.17) + set(${BASE_VARIABLE_NAME}_PATCH_COMMAND + ${CMAKE_COMMAND} -E remove tests/features/S2N_LIBCRYPTO_SUPPORTS_ENGINE.c) + else() + set(${BASE_VARIABLE_NAME}_PATCH_COMMAND + ${CMAKE_COMMAND} -E rm tests/features/S2N_LIBCRYPTO_SUPPORTS_ENGINE.c) + endif() else() set(${BASE_VARIABLE_NAME}_DIFF_FILE "${CMAKE_CURRENT_LIST_DIR}/${AWSSDK_PRODUCT}.diff") From 66978feac8da4abfc47f324ba6fce8700af1efef Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Thu, 23 Jan 2025 18:45:30 +0900 Subject: [PATCH 09/52] Link s2n_libcrypto.a --- cpp/cmake_modules/ThirdpartyToolchain.cmake | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index 73a8e7b6685..2ab9e3c0289 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -5171,6 +5171,12 @@ function(build_awssdk) if("${AWSSDK_PRODUCT}" STREQUAL "aws-lc") list(PREPEND AWSSDK_LINK_LIBRARIES ssl) elseif("${AWSSDK_PRODUCT}" STREQUAL "s2n-tls") + add_library(s2n_libcrypto_static STATIC IMPORTED) + set_target_properties(s2n_libcrypto_static + PROPERTIES IMPORTED_LOCATION + "${s2n-tls_BINARY_DIR}/s2n_libcrypto.a") + add_dependencies(s2n_libcrypto_static s2n_libcrypto) + list(PREPEND AWSSDK_LINK_LIBRARIES s2n_libcrypto_static) list(PREPEND AWSSDK_LINK_LIBRARIES s2n) # Workaround: S2N_INTERN_LIBCRYPTO doesn't support # out-of-source build. From b54e8047816f114f6e4e29e8c9454a8bba59d2ff Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Sun, 26 Jan 2025 23:57:28 +0900 Subject: [PATCH 10/52] Fix s2n-tls --- cpp/cmake_modules/ThirdpartyToolchain.cmake | 36 ++++++--------------- cpp/cmake_modules/s2n-tls.diff | 25 ++++++++++++++ 2 files changed, 35 insertions(+), 26 deletions(-) create mode 100644 cpp/cmake_modules/s2n-tls.diff diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index 2ab9e3c0289..6e6cea970bf 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -5089,24 +5089,14 @@ function(build_awssdk) # AWS-C-CAL -> # AWS_C_CAL string(REGEX REPLACE "-" "_" BASE_VARIABLE_NAME "${BASE_VARIABLE_NAME}") - if(AWSSDK_PRODUCT STREQUAL "s2n-tls") - if(CMAKE_VERSION VERSION_LESS 3.17) - set(${BASE_VARIABLE_NAME}_PATCH_COMMAND - ${CMAKE_COMMAND} -E remove tests/features/S2N_LIBCRYPTO_SUPPORTS_ENGINE.c) - else() - set(${BASE_VARIABLE_NAME}_PATCH_COMMAND - ${CMAKE_COMMAND} -E rm tests/features/S2N_LIBCRYPTO_SUPPORTS_ENGINE.c) - endif() - else() - set(${BASE_VARIABLE_NAME}_DIFF_FILE - "${CMAKE_CURRENT_LIST_DIR}/${AWSSDK_PRODUCT}.diff") - if(EXISTS "${${BASE_VARIABLE_NAME}_DIFF_FILE}") - if(NOT PATCH) - find_program(PATCH patch REQUIRED) - endif() - set(${BASE_VARIABLE_NAME}_PATCH_COMMAND ${PATCH} -p1 -i - "${${BASE_VARIABLE_NAME}_DIFF_FILE}") + set(${BASE_VARIABLE_NAME}_DIFF_FILE + "${CMAKE_CURRENT_LIST_DIR}/${AWSSDK_PRODUCT}.diff") + if(EXISTS "${${BASE_VARIABLE_NAME}_DIFF_FILE}") + if(NOT PATCH) + find_program(PATCH patch REQUIRED) endif() + set(${BASE_VARIABLE_NAME}_PATCH_COMMAND ${PATCH} -p1 -i + "${${BASE_VARIABLE_NAME}_DIFF_FILE}") endif() fetchcontent_declare(${AWSSDK_PRODUCT} ${FC_DECLARE_COMMON_OPTIONS} @@ -5134,6 +5124,9 @@ function(build_awssdk) set(BUILD_TOOL OFF CACHE BOOL "" FORCE) + set(ENABLE_TESTING + OFF + CACHE BOOL "" FORCE) set(IN_SOURCE_BUILD ON CACHE BOOL "" FORCE) @@ -5171,16 +5164,7 @@ function(build_awssdk) if("${AWSSDK_PRODUCT}" STREQUAL "aws-lc") list(PREPEND AWSSDK_LINK_LIBRARIES ssl) elseif("${AWSSDK_PRODUCT}" STREQUAL "s2n-tls") - add_library(s2n_libcrypto_static STATIC IMPORTED) - set_target_properties(s2n_libcrypto_static - PROPERTIES IMPORTED_LOCATION - "${s2n-tls_BINARY_DIR}/s2n_libcrypto.a") - add_dependencies(s2n_libcrypto_static s2n_libcrypto) - list(PREPEND AWSSDK_LINK_LIBRARIES s2n_libcrypto_static) list(PREPEND AWSSDK_LINK_LIBRARIES s2n) - # Workaround: S2N_INTERN_LIBCRYPTO doesn't support - # out-of-source build. - file(MAKE_DIRECTORY ${s2n-tls_BINARY_DIR}/lib) else() list(PREPEND AWSSDK_LINK_LIBRARIES ${AWSSDK_PRODUCT}) endif() diff --git a/cpp/cmake_modules/s2n-tls.diff b/cpp/cmake_modules/s2n-tls.diff new file mode 100644 index 00000000000..9ff0a52cf2f --- /dev/null +++ b/cpp/cmake_modules/s2n-tls.diff @@ -0,0 +1,25 @@ +diff --git a/CMakeLists.txt b/CMakeLists.txt +index 23ef74369..50c1573d9 100644 +--- a/CMakeLists.txt ++++ b/CMakeLists.txt +@@ -352,6 +352,9 @@ FILE(GLOB FEATURE_SRCS "${CMAKE_CURRENT_LIST_DIR}/tests/features/*.c") + list(SORT FEATURE_SRCS) + foreach(file ${FEATURE_SRCS}) + get_filename_component(feature_name ${file} NAME_WE) ++ if(feature_name STREQUAL "S2N_LIBCRYPTO_SUPPORTS_ENGINE") ++ continue() ++ endif() + feature_probe(${feature_name}) + endforeach() + +@@ -445,9 +448,8 @@ if (S2N_INTERN_LIBCRYPTO) + # add all of the prefixed symbols to the archive + add_custom_command( + TARGET ${PROJECT_NAME} POST_BUILD +- DEPENDS libcrypto.symbols + COMMAND +- bash -c "${CMAKE_AR} -r lib/libs2n.a s2n_libcrypto/*.o" ++ bash -c "${CMAKE_AR} -r $ s2n_libcrypto/*.o" + VERBATIM + ) + endif() From 9abe096510c038675dac3c7c138a41daf29ea868 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Mon, 27 Jan 2025 13:25:17 +0900 Subject: [PATCH 11/52] Don't use OpenSSL --- cpp/cmake_modules/ThirdpartyToolchain.cmake | 3 --- 1 file changed, 3 deletions(-) diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index 6e6cea970bf..9911e75d596 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -5133,9 +5133,6 @@ function(build_awssdk) set(MINIMIZE_SIZE ON CACHE BOOL "" FORCE) - set(USE_OPENSSL - ON - CACHE BOOL "" FORCE) # For aws-lc set(DISABLE_PERL From fec40766f8c6b2cabf75ef4ee3e4f34ca87944aa Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Mon, 27 Jan 2025 13:38:49 +0900 Subject: [PATCH 12/52] Add a missing license header --- cpp/cmake_modules/s2n-tls.diff | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/cpp/cmake_modules/s2n-tls.diff b/cpp/cmake_modules/s2n-tls.diff index 9ff0a52cf2f..55f3256e0c5 100644 --- a/cpp/cmake_modules/s2n-tls.diff +++ b/cpp/cmake_modules/s2n-tls.diff @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + diff --git a/CMakeLists.txt b/CMakeLists.txt index 23ef74369..50c1573d9 100644 --- a/CMakeLists.txt From 0563776ec20e7234e434a944487961220eef4e73 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Mon, 27 Jan 2025 15:46:41 +0900 Subject: [PATCH 13/52] Don't use aws-lc --- cpp/CMakeLists.txt | 6 +----- cpp/cmake_modules/ThirdpartyToolchain.cmake | 13 ++++++++++--- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 7a1a94a99b5..18ff93e965f 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -392,14 +392,10 @@ endif() # where to put generated archives (.a files) set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY "${BUILD_OUTPUT_ROOT_DIRECTORY}") -set(ARCHIVE_OUTPUT_DIRECTORY "${BUILD_OUTPUT_ROOT_DIRECTORY}") - # where to put generated libraries (.so files) set(CMAKE_LIBRARY_OUTPUT_DIRECTORY "${BUILD_OUTPUT_ROOT_DIRECTORY}") -set(LIBRARY_OUTPUT_DIRECTORY "${BUILD_OUTPUT_ROOT_DIRECTORY}") - # where to put generated binaries -set(EXECUTABLE_OUTPUT_PATH "${BUILD_OUTPUT_ROOT_DIRECTORY}") +set(CMAKE_RUNTIME_OUTPUT_DIRECTORY "${BUILD_OUTPUT_ROOT_DIRECTORY}") if(CMAKE_GENERATOR STREQUAL Xcode) # Xcode projects support multi-configuration builds. This forces a single output directory diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index 9911e75d596..d155c53893b 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -1020,14 +1020,17 @@ endif() macro(prepare_fetchcontent) set(BUILD_SHARED_LIBS OFF) set(BUILD_STATIC_LIBS ON) + set(BUILD_TESTING OFF) + set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY "") set(CMAKE_COMPILE_WARNING_AS_ERROR FALSE) set(CMAKE_EXPORT_NO_PACKAGE_REGISTRY TRUE) + set(CMAKE_LIBRARY_OUTPUT_DIRECTORY "") + set(CMAKE_MACOSX_RPATH ${ARROW_INSTALL_NAME_RPATH}) # We set CMAKE_POLICY_VERSION_MINIMUM temporarily due to failures with CMake 4 # We should remove it once we have updated the dependencies: # https://github.com/apache/arrow/issues/45985 set(CMAKE_POLICY_VERSION_MINIMUM 3.5) - set(CMAKE_MACOSX_RPATH ${ARROW_INSTALL_NAME_RPATH}) - set(ENABLE_TESTING OFF) + set(CMAKE_RUNTIME_OUTPUT_DIRECTORY "") if(MSVC) string(REPLACE "/WX" "" CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG}") @@ -5133,6 +5136,9 @@ function(build_awssdk) set(MINIMIZE_SIZE ON CACHE BOOL "" FORCE) + set(USE_OPENSSL + ON + CACHE BOOL "" FORCE) # For aws-lc set(DISABLE_PERL @@ -5159,7 +5165,8 @@ function(build_awssdk) list(PREPEND CMAKE_MODULE_PATH "${${AWSSDK_PRODUCT}_SOURCE_DIR}/cmake") if(NOT "${AWSSDK_PRODUCT}" STREQUAL "aws-sdk-cpp") if("${AWSSDK_PRODUCT}" STREQUAL "aws-lc") - list(PREPEND AWSSDK_LINK_LIBRARIES ssl) + # We don't need to link aws-lc. It's used by s2n-tls. + # list(PREPEND AWSSDK_LINK_LIBRARIES ssl) elseif("${AWSSDK_PRODUCT}" STREQUAL "s2n-tls") list(PREPEND AWSSDK_LINK_LIBRARIES s2n) else() From 5c0fab82ee0864d1391b58107f960bb68e29edd8 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Mon, 27 Jan 2025 15:56:12 +0900 Subject: [PATCH 14/52] Fix --- cpp/cmake_modules/BuildUtils.cmake | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/cpp/cmake_modules/BuildUtils.cmake b/cpp/cmake_modules/BuildUtils.cmake index 32d962b0cf2..134f47b12ef 100644 --- a/cpp/cmake_modules/BuildUtils.cmake +++ b/cpp/cmake_modules/BuildUtils.cmake @@ -552,7 +552,7 @@ function(ADD_BENCHMARK REL_BENCHMARK_NAME) if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${REL_BENCHMARK_NAME}.cc) # This benchmark has a corresponding .cc file, set it up as an executable. - set(BENCHMARK_PATH "${EXECUTABLE_OUTPUT_PATH}/${BENCHMARK_NAME}") + set(BENCHMARK_PATH "${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/${BENCHMARK_NAME}") add_executable(${BENCHMARK_NAME} ${SOURCES}) if(ARG_STATIC_LINK_LIBS) @@ -581,7 +581,8 @@ function(ADD_BENCHMARK REL_BENCHMARK_NAME) PROPERTIES BUILD_WITH_INSTALL_RPATH TRUE INSTALL_RPATH_USE_LINK_PATH TRUE INSTALL_RPATH - "$ENV{CONDA_PREFIX}/lib;${EXECUTABLE_OUTPUT_PATH}") + "$ENV{CONDA_PREFIX}/lib;${CMAKE_RUNTIME_OUTPUT_DIRECTORY}" + ) endif() # Add test as dependency of relevant label targets @@ -682,7 +683,7 @@ function(ADD_TEST_CASE REL_TEST_NAME) # Make sure the executable name contains only hyphens, not underscores string(REPLACE "_" "-" TEST_NAME ${TEST_NAME}) - set(TEST_PATH "${EXECUTABLE_OUTPUT_PATH}/${TEST_NAME}") + set(TEST_PATH "${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/${TEST_NAME}") add_executable(${TEST_NAME} ${SOURCES}) # With OSX and conda, we need to set the correct RPATH so that dependencies @@ -695,7 +696,8 @@ function(ADD_TEST_CASE REL_TEST_NAME) PROPERTIES BUILD_WITH_INSTALL_RPATH TRUE INSTALL_RPATH_USE_LINK_PATH TRUE INSTALL_RPATH - "${EXECUTABLE_OUTPUT_PATH};$ENV{CONDA_PREFIX}/lib") + "${CMAKE_RUNTIME_OUTPUT_DIRECTORY};$ENV{CONDA_PREFIX}/lib" + ) endif() # Ensure using bundled GoogleTest when we use bundled GoogleTest. @@ -826,7 +828,7 @@ function(ADD_ARROW_EXAMPLE REL_EXAMPLE_NAME) if(EXISTS ${CMAKE_SOURCE_DIR}/examples/arrow/${REL_EXAMPLE_NAME}.cc) # This example has a corresponding .cc file, set it up as an executable. - set(EXAMPLE_PATH "${EXECUTABLE_OUTPUT_PATH}/${EXAMPLE_NAME}") + set(EXAMPLE_PATH "${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/${EXAMPLE_NAME}") add_executable(${EXAMPLE_NAME} "${REL_EXAMPLE_NAME}.cc" ${ARG_EXTRA_SOURCES}) target_link_libraries(${EXAMPLE_NAME} ${ARROW_EXAMPLE_LINK_LIBS}) add_dependencies(runexample ${EXAMPLE_NAME}) From 6f758c0b4bcc17fec953713eea9ab4c77310c98e Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Mon, 27 Jan 2025 17:07:04 +0900 Subject: [PATCH 15/52] Disable LTO --- cpp/cmake_modules/ThirdpartyToolchain.cmake | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index d155c53893b..fa45dc4a572 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -5159,14 +5159,19 @@ function(build_awssdk) ON CACHE BOOL "" FORCE) + # For aws-lc and s2n-tls + # + # Link time optimization is causing trouble like GH-34349 + string(REPLACE "-flto=auto" "" CMAKE_C_FLAGS "${CMAKE_C_FLAGS}") + string(REPLACE "-ffat-lto-objects" "" CMAKE_C_FLAGS "${CMAKE_C_FLAGS}") + set(AWSSDK_LINK_LIBRARIES) foreach(AWSSDK_PRODUCT ${AWSSDK_PRODUCTS}) fetchcontent_makeavailable(${AWSSDK_PRODUCT}) list(PREPEND CMAKE_MODULE_PATH "${${AWSSDK_PRODUCT}_SOURCE_DIR}/cmake") if(NOT "${AWSSDK_PRODUCT}" STREQUAL "aws-sdk-cpp") if("${AWSSDK_PRODUCT}" STREQUAL "aws-lc") - # We don't need to link aws-lc. It's used by s2n-tls. - # list(PREPEND AWSSDK_LINK_LIBRARIES ssl) + # We don't need to link aws-lc. It's used only by s2n-tls. elseif("${AWSSDK_PRODUCT}" STREQUAL "s2n-tls") list(PREPEND AWSSDK_LINK_LIBRARIES s2n) else() From 21111630d942afdb2b53c777e4c1f6e2b553d992 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Tue, 28 Jan 2025 10:01:38 +0900 Subject: [PATCH 16/52] Add support for -h --- cpp/thirdparty/update.rb | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/cpp/thirdparty/update.rb b/cpp/thirdparty/update.rb index 5a063bcc83a..98571374c09 100755 --- a/cpp/thirdparty/update.rb +++ b/cpp/thirdparty/update.rb @@ -20,10 +20,14 @@ require "digest/sha2" require "json" require "open-uri" +require "optparse" -if ARGV.empty? - puts("Usage: #{$0} PRODUCT_PATTERN1 PRODUCT_PATTERN2 ...") - puts(" e.g.: #{$0} 'AWS*' 'S2N*'.") +option_parser = OptionParser.new +option_parser.banner = + "Usage: #{$0} [options] PRODUCT_PATTERN1 PRODUCT_PATTERN2 ..." +patterns = option_parser.parse!(ARGV) +if patterns.empty? + puts(option_parser) exit(false) end @@ -118,7 +122,7 @@ def update_versions_txt_content!(content, products) versions_txt = File.join(__dir__, "versions.txt") versions_txt_content = File.read(versions_txt) products = parse_versions_txt_content(versions_txt_content) -ARGV.each do |pattern| +patterns.each do |pattern| target_products = products.filter do |product, _| File.fnmatch?(pattern, product) end From af13d016c6c118581269b0d02a7efea767fb9af1 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Tue, 28 Jan 2025 10:01:48 +0900 Subject: [PATCH 17/52] Fix path --- cpp/thirdparty/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cpp/thirdparty/README.md b/cpp/thirdparty/README.md index 830371bad8a..32b216861f1 100644 --- a/cpp/thirdparty/README.md +++ b/cpp/thirdparty/README.md @@ -30,12 +30,12 @@ There is a convenient script that update versions in `versions.txt` to the latest version automatically. You can use it like the following: ```console -dev/release/update.rb PRODUCT_PATTERN1 PRODUCT_PATTERN2 ... +cpp/thirdparty/update.rb PRODUCT_PATTERN1 PRODUCT_PATTERN2 ... ``` For example, you can update AWS SDK for C++ related products' versions by the following command line: ```console -dev/release/update.rb "AWS*" "S2N*" +cpp/thirdparty/update.rb "AWS*" "S2N*" ``` From cb890391ec10ecceade3a93d3741a260ae9e6adb Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Thu, 30 Jan 2025 15:29:11 +0900 Subject: [PATCH 18/52] Use SYSTEM --- cpp/cmake_modules/ThirdpartyToolchain.cmake | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index fa45dc4a572..490376cde40 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -1016,6 +1016,9 @@ set(FC_DECLARE_COMMON_OPTIONS) if(CMAKE_VERSION VERSION_GREATER_EQUAL 3.28) list(APPEND FC_DECLARE_COMMON_OPTIONS EXCLUDE_FROM_ALL TRUE) endif() +if(CMAKE_VERSION VERSION_GREATER_EQUAL 3.25) + list(APPEND FC_DECLARE_COMMON_OPTIONS SYSTEM) +endif() macro(prepare_fetchcontent) set(BUILD_SHARED_LIBS OFF) From f25eb594ddb835d0f93a949b1e1ad4154a9eaccc Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Fri, 31 Jan 2025 10:59:52 +0900 Subject: [PATCH 19/52] Use OVERRIDE_FIND_PACKAGE --- cpp/cmake_modules/ThirdpartyToolchain.cmake | 3 +++ cpp/cmake_modules/aws-crt-cpp.diff | 28 --------------------- cpp/cmake_modules/aws-sdk-cpp.diff | 28 --------------------- 3 files changed, 3 insertions(+), 56 deletions(-) delete mode 100644 cpp/cmake_modules/aws-crt-cpp.diff delete mode 100644 cpp/cmake_modules/aws-sdk-cpp.diff diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index 490376cde40..1d94141a498 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -5106,6 +5106,7 @@ function(build_awssdk) endif() fetchcontent_declare(${AWSSDK_PRODUCT} ${FC_DECLARE_COMMON_OPTIONS} + OVERRIDE_FIND_PACKAGE PATCH_COMMAND ${${BASE_VARIABLE_NAME}_PATCH_COMMAND} URL ${${BASE_VARIABLE_NAME}_SOURCE_URL} URL_HASH "SHA256=${ARROW_${BASE_VARIABLE_NAME}_BUILD_SHA256_CHECKSUM}" @@ -5179,6 +5180,8 @@ function(build_awssdk) list(PREPEND AWSSDK_LINK_LIBRARIES s2n) else() list(PREPEND AWSSDK_LINK_LIBRARIES ${AWSSDK_PRODUCT}) + # This is for find_package(aws-*) in aws-crt-cpp and aws-sdk-cpp. + add_library(AWS::${AWSSDK_PRODUCT} ALIAS ${AWSSDK_PRODUCT}) endif() endif() endforeach() diff --git a/cpp/cmake_modules/aws-crt-cpp.diff b/cpp/cmake_modules/aws-crt-cpp.diff deleted file mode 100644 index c8fdc612927..00000000000 --- a/cpp/cmake_modules/aws-crt-cpp.diff +++ /dev/null @@ -1,28 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -diff -ru aws-crt-cpp-src.orig/CMakeLists.txt aws-crt-cpp-src/CMakeLists.txt ---- aws-crt-cpp-src.orig/CMakeLists.txt 2025-01-20 15:06:08.602765260 +0900 -+++ aws-crt-cpp-src/CMakeLists.txt 2025-01-20 15:07:30.879403510 +0900 -@@ -133,5 +133,4 @@ - set(BUILD_TESTING ${BUILD_TESTING_PREV}) - else() - include(AwsFindPackage) -- set(IN_SOURCE_BUILD OFF) - endif() - - include(AwsCFlags) diff --git a/cpp/cmake_modules/aws-sdk-cpp.diff b/cpp/cmake_modules/aws-sdk-cpp.diff deleted file mode 100644 index 0ecb248de8d..00000000000 --- a/cpp/cmake_modules/aws-sdk-cpp.diff +++ /dev/null @@ -1,28 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -diff -ru aws-sdk-cpp-src.orig/CMakeLists.txt aws-sdk-cpp-src/CMakeLists.txt ---- aws-sdk-cpp-src.orig/CMakeLists.txt 2025-01-20 14:51:21.180004943 +0900 -+++ aws-sdk-cpp-src/CMakeLists.txt 2025-01-20 15:00:20.256077009 +0900 -@@ -245,7 +245,6 @@ - # Append that generated list to the module search path - list(APPEND CMAKE_MODULE_PATH ${AWS_MODULE_PATH}) - include(AwsFindPackage) -- set(IN_SOURCE_BUILD OFF) - endif () - aws_use_package(aws-crt-cpp) - aws_use_package(aws-c-http) From dc131cfe97b539ceae5779bcd636cbf1ec0dcaa9 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Tue, 4 Feb 2025 08:23:03 +0900 Subject: [PATCH 20/52] Remove s2n-tls.diff --- cpp/cmake_modules/ThirdpartyToolchain.cmake | 22 ++++++----- cpp/cmake_modules/s2n-tls.diff | 42 --------------------- 2 files changed, 13 insertions(+), 51 deletions(-) delete mode 100644 cpp/cmake_modules/s2n-tls.diff diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index 1d94141a498..75ef3fa9808 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -5095,18 +5095,22 @@ function(build_awssdk) # AWS-C-CAL -> # AWS_C_CAL string(REGEX REPLACE "-" "_" BASE_VARIABLE_NAME "${BASE_VARIABLE_NAME}") - set(${BASE_VARIABLE_NAME}_DIFF_FILE - "${CMAKE_CURRENT_LIST_DIR}/${AWSSDK_PRODUCT}.diff") - if(EXISTS "${${BASE_VARIABLE_NAME}_DIFF_FILE}") - if(NOT PATCH) - find_program(PATCH patch REQUIRED) + if(AWSSDK_PRODUCT STREQUAL "s2n-tls") + # S2N_LIBCRYPTO_SUPPORTS_ENGINE.c doesn't refer + # S2N_INTERN_LIBCRYPTO. We need to use aws-lc with + # S2N_INTERN_LIBCRYPTO but S2N_LIBCRYPTO_SUPPORTS_ENGINE.c may + # refer system OpenSSL. So s2n-tls may mix aws-lc and OpenSSL + # configurations. + if(CMAKE_VERSION VERSION_LESS 3.17) + set(${BASE_VARIABLE_NAME}_PATCH_COMMAND + ${CMAKE_COMMAND} -E remove tests/features/S2N_LIBCRYPTO_SUPPORTS_ENGINE.c) + else() + set(${BASE_VARIABLE_NAME}_PATCH_COMMAND + ${CMAKE_COMMAND} -E rm tests/features/S2N_LIBCRYPTO_SUPPORTS_ENGINE.c) endif() - set(${BASE_VARIABLE_NAME}_PATCH_COMMAND ${PATCH} -p1 -i - "${${BASE_VARIABLE_NAME}_DIFF_FILE}") endif() fetchcontent_declare(${AWSSDK_PRODUCT} - ${FC_DECLARE_COMMON_OPTIONS} - OVERRIDE_FIND_PACKAGE + ${FC_DECLARE_COMMON_OPTIONS} OVERRIDE_FIND_PACKAGE PATCH_COMMAND ${${BASE_VARIABLE_NAME}_PATCH_COMMAND} URL ${${BASE_VARIABLE_NAME}_SOURCE_URL} URL_HASH "SHA256=${ARROW_${BASE_VARIABLE_NAME}_BUILD_SHA256_CHECKSUM}" diff --git a/cpp/cmake_modules/s2n-tls.diff b/cpp/cmake_modules/s2n-tls.diff deleted file mode 100644 index 55f3256e0c5..00000000000 --- a/cpp/cmake_modules/s2n-tls.diff +++ /dev/null @@ -1,42 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -diff --git a/CMakeLists.txt b/CMakeLists.txt -index 23ef74369..50c1573d9 100644 ---- a/CMakeLists.txt -+++ b/CMakeLists.txt -@@ -352,6 +352,9 @@ FILE(GLOB FEATURE_SRCS "${CMAKE_CURRENT_LIST_DIR}/tests/features/*.c") - list(SORT FEATURE_SRCS) - foreach(file ${FEATURE_SRCS}) - get_filename_component(feature_name ${file} NAME_WE) -+ if(feature_name STREQUAL "S2N_LIBCRYPTO_SUPPORTS_ENGINE") -+ continue() -+ endif() - feature_probe(${feature_name}) - endforeach() - -@@ -445,9 +448,8 @@ if (S2N_INTERN_LIBCRYPTO) - # add all of the prefixed symbols to the archive - add_custom_command( - TARGET ${PROJECT_NAME} POST_BUILD -- DEPENDS libcrypto.symbols - COMMAND -- bash -c "${CMAKE_AR} -r lib/libs2n.a s2n_libcrypto/*.o" -+ bash -c "${CMAKE_AR} -r $ s2n_libcrypto/*.o" - VERBATIM - ) - endif() From 6d242d40859dd4291333df7cc6ff44a59b9e9e8a Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Thu, 6 Feb 2025 12:08:54 +0900 Subject: [PATCH 21/52] Use archive --- cpp/cmake_modules/ThirdpartyToolchain.cmake | 27 +++++++++++++++------ 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index 75ef3fa9808..8dd7bf4bfa9 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -5067,14 +5067,20 @@ include(AWSSDKVariables) function(build_awssdk) message(STATUS "Building AWS SDK for C++ from source") - set(AWSSDK_PRODUCTS aws-c-common aws-checksums) + # aws-c-common must be the first product because others depend on + # this. + set(AWSSDK_PRODUCTS aws-c-common) # aws-lc and s2n-tls only needed on Linux. # We can use LINUX with CMake 3.25 or later. if(UNIX AND NOT APPLE) list(APPEND AWSSDK_PRODUCTS aws-lc s2n-tls) endif() + list(APPEND AWSSDK_PRODUCTS aws-checksums) list(APPEND AWSSDK_PRODUCTS + # We can't sort this in alphabetical order because some + # products depend on other products. + aws-checksums aws-c-cal aws-c-io aws-c-event-stream @@ -5108,13 +5114,20 @@ function(build_awssdk) set(${BASE_VARIABLE_NAME}_PATCH_COMMAND ${CMAKE_COMMAND} -E rm tests/features/S2N_LIBCRYPTO_SUPPORTS_ENGINE.c) endif() + # We can use released archive when v1.15.12 is released. + fetchcontent_declare(${AWSSDK_PRODUCT} + ${FC_DECLARE_COMMON_OPTIONS} OVERRIDE_FIND_PACKAGE + PATCH_COMMAND ${${BASE_VARIABLE_NAME}_PATCH_COMMAND} + URL https://github.com/aws/s2n-tls/archive/7c0291809ad58b3b818b128d97c19c71ff3e10e1.zip + URL_HASH "SHA256=1ca1fb0a82642a93ab7c95c5a6c9ff80e3388e387fba03153186426b98d8b9e0" + ) + else() + fetchcontent_declare(${AWSSDK_PRODUCT} + ${FC_DECLARE_COMMON_OPTIONS} OVERRIDE_FIND_PACKAGE + URL ${${BASE_VARIABLE_NAME}_SOURCE_URL} + URL_HASH "SHA256=${ARROW_${BASE_VARIABLE_NAME}_BUILD_SHA256_CHECKSUM}" + ) endif() - fetchcontent_declare(${AWSSDK_PRODUCT} - ${FC_DECLARE_COMMON_OPTIONS} OVERRIDE_FIND_PACKAGE - PATCH_COMMAND ${${BASE_VARIABLE_NAME}_PATCH_COMMAND} - URL ${${BASE_VARIABLE_NAME}_SOURCE_URL} - URL_HASH "SHA256=${ARROW_${BASE_VARIABLE_NAME}_BUILD_SHA256_CHECKSUM}" - ) endforeach() prepare_fetchcontent() From ce707dc6c2427728376bd345fe4c6c0d7d432baa Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Fri, 7 Feb 2025 10:58:29 +0900 Subject: [PATCH 22/52] Remove duplicated entry --- cpp/cmake_modules/ThirdpartyToolchain.cmake | 1 - 1 file changed, 1 deletion(-) diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index 8dd7bf4bfa9..ebbe3e7e58d 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -5075,7 +5075,6 @@ function(build_awssdk) if(UNIX AND NOT APPLE) list(APPEND AWSSDK_PRODUCTS aws-lc s2n-tls) endif() - list(APPEND AWSSDK_PRODUCTS aws-checksums) list(APPEND AWSSDK_PRODUCTS # We can't sort this in alphabetical order because some From bcce3631e1477b95059647bc8965c842a1946abc Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Tue, 1 Apr 2025 16:20:40 +0900 Subject: [PATCH 23/52] Add patch --- cpp/cmake_modules/ThirdpartyToolchain.cmake | 30 +-- cpp/cmake_modules/aws-sdk-cpp-pr-1333.patch | 218 ++++++++++++++++++++ 2 files changed, 233 insertions(+), 15 deletions(-) create mode 100644 cpp/cmake_modules/aws-sdk-cpp-pr-1333.patch diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index ebbe3e7e58d..7aae4268dcc 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -5106,27 +5106,27 @@ function(build_awssdk) # S2N_INTERN_LIBCRYPTO but S2N_LIBCRYPTO_SUPPORTS_ENGINE.c may # refer system OpenSSL. So s2n-tls may mix aws-lc and OpenSSL # configurations. - if(CMAKE_VERSION VERSION_LESS 3.17) - set(${BASE_VARIABLE_NAME}_PATCH_COMMAND - ${CMAKE_COMMAND} -E remove tests/features/S2N_LIBCRYPTO_SUPPORTS_ENGINE.c) - else() - set(${BASE_VARIABLE_NAME}_PATCH_COMMAND - ${CMAKE_COMMAND} -E rm tests/features/S2N_LIBCRYPTO_SUPPORTS_ENGINE.c) - endif() - # We can use released archive when v1.15.12 is released. + set(${BASE_VARIABLE_NAME}_PATCH_COMMAND + ${CMAKE_COMMAND} -E rm tests/features/S2N_LIBCRYPTO_SUPPORTS_ENGINE.c) fetchcontent_declare(${AWSSDK_PRODUCT} ${FC_DECLARE_COMMON_OPTIONS} OVERRIDE_FIND_PACKAGE PATCH_COMMAND ${${BASE_VARIABLE_NAME}_PATCH_COMMAND} - URL https://github.com/aws/s2n-tls/archive/7c0291809ad58b3b818b128d97c19c71ff3e10e1.zip + URL ${${BASE_VARIABLE_NAME}_SOURCE_URL URL_HASH "SHA256=1ca1fb0a82642a93ab7c95c5a6c9ff80e3388e387fba03153186426b98d8b9e0" ) - else() - fetchcontent_declare(${AWSSDK_PRODUCT} - ${FC_DECLARE_COMMON_OPTIONS} OVERRIDE_FIND_PACKAGE - URL ${${BASE_VARIABLE_NAME}_SOURCE_URL} - URL_HASH "SHA256=${ARROW_${BASE_VARIABLE_NAME}_BUILD_SHA256_CHECKSUM}" - ) + elseif(AWSSDK_PRODUCT_NAME STREQUAL "aws-sdk-cpp" AND WINDOWS AND NOT MSVC) + # We can remove this once + # https://github.com/aws/aws-sdk-cpp/issues/3315 is resolved. + find_program(PATCH patch REQUIRED) + set(${BASE_VARIABLE_NAME}_PATCH_COMMAND + patch --input=${CMAKE_CURRENT_SOURCE_DIR}/aws-sdk-cpp-pr-1333.patch --strip=1) endif() + fetchcontent_declare(${AWSSDK_PRODUCT} + ${FC_DECLARE_COMMON_OPTIONS} OVERRIDE_FIND_PACKAGE + PATCH_COMMAND ${${BASE_VARIABLE_NAME}_PATCH_COMMAND} + URL ${${BASE_VARIABLE_NAME}_SOURCE_URL} + URL_HASH "SHA256=${ARROW_${BASE_VARIABLE_NAME}_BUILD_SHA256_CHECKSUM}" + ) endforeach() prepare_fetchcontent() diff --git a/cpp/cmake_modules/aws-sdk-cpp-pr-1333.patch b/cpp/cmake_modules/aws-sdk-cpp-pr-1333.patch new file mode 100644 index 00000000000..f47b2f51583 --- /dev/null +++ b/cpp/cmake_modules/aws-sdk-cpp-pr-1333.patch @@ -0,0 +1,218 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# This is a copy of +# https://github.com/msys2/MINGW-packages/blob/master/mingw-w64-aws-sdk-cpp/aws-sdk-cpp-pr-1333.patch +# . This is based on https://github.com/aws/aws-sdk-cpp/pull/1333 . + +From f9b4499ccb63ef02d8c9de462fbbcf301eb1907a Mon Sep 17 00:00:00 2001 +From: Daniel Schulte +Date: Tue, 3 Mar 2020 11:09:15 +0100 +Subject: [PATCH] Add support for building with MinGW + +--- + .../include/aws/core/utils/crypto/bcrypt/CryptoImpl.h | 7 +++++++ + .../include/aws/core/utils/event/EventHeader.h | 10 ++++++++++ + .../source/http/windows/WinHttpSyncHttpClient.cpp | 4 ++-- + .../source/http/windows/WinINetSyncHttpClient.cpp | 2 +- + .../source/http/windows/WinSyncHttpClient.cpp | 2 +- + .../source/platform/windows/Environment.cpp | 5 +++++ + .../source/platform/windows/FileSystem.cpp | 7 ++++++- + .../source/platform/windows/OSVersionInfo.cpp | 2 ++ + .../source/utils/crypto/factory/Factories.cpp | 4 ++-- + .../BucketAndObjectOperationTest.cpp | 4 ++-- + cmake/compiler_settings.cmake | 6 ++++++ + .../source/platform/windows/PlatformTesting.cpp | 2 ++ + 12 files changed, 46 insertions(+), 9 deletions(-) + +diff --git a/src/aws-cpp-sdk-core/include/aws/core/utils/event/EventHeader.h b/src/aws-cpp-sdk-core/include/aws/core/utils/event/EventHeader.h +index 2041b208da..6fb333bff7 100644 +--- a/src/aws-cpp-sdk-core/include/aws/core/utils/event/EventHeader.h ++++ b/src/aws-cpp-sdk-core/include/aws/core/utils/event/EventHeader.h +@@ -14,6 +14,12 @@ + #include + #include + ++#ifdef __MINGW32__ ++#pragma GCC diagnostic push ++#pragma GCC diagnostic ignored "-Wmaybe-uninitialized" ++#pragma GCC diagnostic ignored "-Wuninitialized" ++#endif ++ + namespace Aws + { + namespace Utils +@@ -318,3 +324,7 @@ namespace Aws + } + } + } ++ ++#ifdef __MINGW32__ ++#pragma GCC diagnostic pop ++#endif +diff --git a/src/aws-cpp-sdk-core/source/http/windows/WinHttpSyncHttpClient.cpp b/src/aws-cpp-sdk-core/source/http/windows/WinHttpSyncHttpClient.cpp +index 4a0f80a161..a98635f706 100644 +--- a/src/aws-cpp-sdk-core/source/http/windows/WinHttpSyncHttpClient.cpp ++++ b/src/aws-cpp-sdk-core/source/http/windows/WinHttpSyncHttpClient.cpp +@@ -17,7 +17,7 @@ + #include + #include + +-#include ++#include + #include + #include // for tcp_keepalive + #include +@@ -686,7 +686,7 @@ + wmemset(contentTypeStr, 0, static_cast(dwSize / sizeof(wchar_t))); + + WinHttpQueryHeaders(hHttpRequest, WINHTTP_QUERY_CONTENT_TYPE, nullptr, &contentTypeStr, &dwSize, 0); +- if (contentTypeStr[0] != NULL) ++ if (contentTypeStr[0]) + { + Aws::String contentStr = StringUtils::FromWString(contentTypeStr); + response->SetContentType(contentStr); +@@ -717,7 +717,7 @@ + + bool WinHttpSyncHttpClient::DoSendRequest(void* hHttpRequest) const + { +- bool success = WinHttpSendRequest(hHttpRequest, NULL, NULL, 0, 0, 0, NULL) != 0; ++ bool success = WinHttpSendRequest(hHttpRequest, NULL, 0, NULL, 0, 0, 0) != 0; + if (!success) + { + AzWinHttpLogLastError("WinHttpSendRequest"); +diff --git a/src/aws-cpp-sdk-core/source/http/windows/WinINetSyncHttpClient.cpp b/src/aws-cpp-sdk-core/source/http/windows/WinINetSyncHttpClient.cpp +index 8b42c64dab..ef1fd99975 100644 +--- a/src/aws-cpp-sdk-core/source/http/windows/WinINetSyncHttpClient.cpp ++++ b/src/aws-cpp-sdk-core/source/http/windows/WinINetSyncHttpClient.cpp +@@ -223,7 +223,7 @@ bool WinINetSyncHttpClient::DoQueryHeaders(void* hHttpRequest, std::shared_ptrSetContentType(contentTypeStr); + AWS_LOGSTREAM_DEBUG(GetLogTag(), "Received content type " << contentTypeStr); +diff --git a/src/aws-cpp-sdk-core/source/platform/windows/FileSystem.cpp b/src/aws-cpp-sdk-core/source/platform/windows/FileSystem.cpp +index 49af3c1f79..f427fa9233 100644 +--- a/src/aws-cpp-sdk-core/source/platform/windows/FileSystem.cpp ++++ b/src/aws-cpp-sdk-core/source/platform/windows/FileSystem.cpp +@@ -11,7 +11,9 @@ + #include + #include + +-#pragma warning( disable : 4996) ++#ifdef _MSC_VER ++# pragma warning( disable : 4996) ++#endif + + using namespace Aws::Utils; + namespace Aws +@@ -304,6 +306,9 @@ Aws::String CreateTempFilePath() + { + #ifdef _MSC_VER + #pragma warning(disable: 4996) // _CRT_SECURE_NO_WARNINGS ++#elif !defined(L_tmpnam_s) ++ // Definition from the MSVC stdio.h ++ #define L_tmpnam_s (sizeof("\\") + 16) + #endif + char s_tempName[L_tmpnam_s+1]; + +diff --git a/src/aws-cpp-sdk-core/source/platform/windows/OSVersionInfo.cpp b/src/aws-cpp-sdk-core/source/platform/windows/OSVersionInfo.cpp +index 23f395f6bb..18bd5836da 100644 +--- a/src/aws-cpp-sdk-core/source/platform/windows/OSVersionInfo.cpp ++++ b/src/aws-cpp-sdk-core/source/platform/windows/OSVersionInfo.cpp +@@ -9,7 +9,9 @@ + + #include + ++#ifdef _MSC_VER + #pragma warning(disable: 4996) ++#endif + #include + #include + namespace Aws +diff --git a/src/aws-cpp-sdk-core/source/utils/crypto/factory/Factories.cpp b/src/aws-cpp-sdk-core/source/utils/crypto/factory/Factories.cpp +index d7cb481d5c..3b34d7fc62 100644 +--- a/src/aws-cpp-sdk-core/source/utils/crypto/factory/Factories.cpp ++++ b/src/aws-cpp-sdk-core/source/utils/crypto/factory/Factories.cpp +@@ -939,7 +939,7 @@ std::shared_ptr Aws::Utils::Crypto::CreateSha256HMACIm + return s_Sha256HMACFactory->CreateImplementation(); + } + +-#ifdef _WIN32 ++#ifdef _MSC_VER + #pragma warning( push ) + #pragma warning( disable : 4702 ) + #endif +@@ -1032,7 +1032,7 @@ std::shared_ptr Aws::Utils::Crypto::CreateAES_KeyWrapImplementa + return s_AES_KeyWrapFactory->CreateImplementation(key); + } + +-#ifdef _WIN32 ++#ifdef _MSC_VER + #pragma warning(pop) + #endif + +diff --git a/tests/aws-cpp-sdk-s3-integration-tests/BucketAndObjectOperationTest.cpp b/tests/aws-cpp-sdk-s3-integration-tests/BucketAndObjectOperationTest.cpp +index 0f31fcd061..586303e044 100644 +--- a/tests/aws-cpp-sdk-s3-integration-tests/BucketAndObjectOperationTest.cpp ++++ b/tests/aws-cpp-sdk-s3-integration-tests/BucketAndObjectOperationTest.cpp +@@ -49,8 +49,9 @@ + #include + #include + +-#ifdef _WIN32 ++#ifdef _MSC_VER + #pragma warning(disable: 4127) ++#endif + #ifdef GetObject + #undef GetObject + #endif +diff --git a/cmake/compiler_settings.cmake b/cmake/compiler_settings.cmake +index db054b2c1b..e1d6dce55e 100644 +--- a/cmake/compiler_settings.cmake ++++ b/cmake/compiler_settings.cmake +@@ -11,6 +11,9 @@ else() + set(COMPILER_CLANG 1) + else() + set(COMPILER_GCC 1) ++ if(MINGW) ++ set(COMPILER_MINGW 1) ++ endif() + endif() + set(USE_GCC_FLAGS 1) + endif() +@@ -34,6 +37,9 @@ endfunction() + + macro(set_gcc_flags) + list(APPEND AWS_COMPILER_FLAGS "-fno-exceptions" "-std=c++${CPP_STANDARD}") ++ if(COMPILER_IS_MINGW) ++ list(APPEND AWS_COMPILER_FLAGS -D__USE_MINGW_ANSI_STDIO=1) ++ endif() + + if(NOT BUILD_SHARED_LIBS) + list(APPEND AWS_COMPILER_FLAGS "-fPIC") +diff --git a/tests/testing-resources/source/platform/windows/PlatformTesting.cpp b/tests/testing-resources/source/platform/windows/PlatformTesting.cpp +index 2b0a04c0b4..2a27710557 100644 +--- a/tests/testing-resources/source/platform/windows/PlatformTesting.cpp ++++ b/tests/testing-resources/source/platform/windows/PlatformTesting.cpp +@@ -5,7 +5,9 @@ + + #include + ++#ifdef _MSC_VER + #pragma warning(disable: 4996) ++#endif + #include + #include From 6f0fcd401044d30321c2dfb1e13c34e24382395f Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Tue, 1 Apr 2025 16:22:45 +0900 Subject: [PATCH 24/52] Update versions --- cpp/thirdparty/versions.txt | 60 ++++++++++++++++++------------------- 1 file changed, 30 insertions(+), 30 deletions(-) diff --git a/cpp/thirdparty/versions.txt b/cpp/thirdparty/versions.txt index 1b491a01ea1..01c28911394 100644 --- a/cpp/thirdparty/versions.txt +++ b/cpp/thirdparty/versions.txt @@ -25,34 +25,34 @@ ARROW_ABSL_BUILD_VERSION=20211102.0 ARROW_ABSL_BUILD_SHA256_CHECKSUM=dcf71b9cba8dc0ca9940c4b316a0c796be8fab42b070bb6b7cab62b48f0e66c4 -ARROW_AWS_C_AUTH_BUILD_VERSION=v0.8.1 -ARROW_AWS_C_AUTH_BUILD_SHA256_CHECKSUM=15241d7284aa0ac552589b61d04be455413af76fb2e1f13084a784a41f5faee5 -ARROW_AWS_C_CAL_BUILD_VERSION=v0.8.1 -ARROW_AWS_C_CAL_BUILD_SHA256_CHECKSUM=4d603641758ef350c3e5401184804e8a6bba4aa5294593cc6228b0dca77b22f5 -ARROW_AWS_C_COMMON_BUILD_VERSION=v0.10.6 -ARROW_AWS_C_COMMON_BUILD_SHA256_CHECKSUM=d0acbabc786035d41791c3a2f45dbeda31d9693521ee746dc1375d6380eb912b -ARROW_AWS_C_COMPRESSION_BUILD_VERSION=v0.3.0 -ARROW_AWS_C_COMPRESSION_BUILD_SHA256_CHECKSUM=7e5d7308d1dbb1801eae9356ef65558f707edf33660dd6443c985db9474725eb -ARROW_AWS_C_EVENT_STREAM_BUILD_VERSION=v0.5.0 -ARROW_AWS_C_EVENT_STREAM_BUILD_SHA256_CHECKSUM=3a53a9d05f9e2fd06036a12854a8b4f05a0c4858bb5b8df8a30edba9de8532b5 -ARROW_AWS_C_HTTP_BUILD_VERSION=v0.9.2 -ARROW_AWS_C_HTTP_BUILD_SHA256_CHECKSUM=328013ebc2b5725326cac01941041eec1e1010058c60709da2c23aa8fb967370 -ARROW_AWS_C_IO_BUILD_VERSION=v0.15.3 -ARROW_AWS_C_IO_BUILD_SHA256_CHECKSUM=d8cb4d7d3ec4fb27cbce158d6823a1f2f5d868e116f1d6703db2ab8159343c3f -ARROW_AWS_C_MQTT_BUILD_VERSION=v0.11.0 -ARROW_AWS_C_MQTT_BUILD_SHA256_CHECKSUM=3854664c13896b6de3d56412f928435a4933259cb7fe62b10c1f497e6999333c -ARROW_AWS_C_S3_BUILD_VERSION=v0.7.9 -ARROW_AWS_C_S3_BUILD_SHA256_CHECKSUM=1942161f9b9d656604a0efeb1387ad99d67d1f2d09154e0395c8bf5da2a39b7a -ARROW_AWS_C_SDKUTILS_BUILD_VERSION=v0.2.2 -ARROW_AWS_C_SDKUTILS_BUILD_SHA256_CHECKSUM=75defbfd4d896b8bdc0790bd25d854218acae61b9409d1956d33832924b82045 -ARROW_AWS_CHECKSUMS_BUILD_VERSION=v0.2.2 -ARROW_AWS_CHECKSUMS_BUILD_SHA256_CHECKSUM=96acfea882c06acd5571c845e4968892d6ffc0fae81b31a0b1565100049743b2 -ARROW_AWS_CRT_CPP_BUILD_VERSION=v0.29.9 -ARROW_AWS_CRT_CPP_BUILD_SHA256_CHECKSUM=d445ab7a26c03a0c0cbb9d82203ee32a56c762a3cef1874783783431b8eb015a -ARROW_AWS_LC_BUILD_VERSION=v1.42.0 -ARROW_AWS_LC_BUILD_SHA256_CHECKSUM=8537025ebfd3884830f494166ae5f72f8aaa203d49abe650a0560a7ffedf359e -ARROW_AWSSDK_BUILD_VERSION=1.11.489 -ARROW_AWSSDK_BUILD_SHA256_CHECKSUM=89ce525c735243a043c5a31eee5dc082fc976b9b5a4b0b66813eed48c97abbfa +ARROW_AWS_C_AUTH_BUILD_VERSION=v0.9.0 +ARROW_AWS_C_AUTH_BUILD_SHA256_CHECKSUM=aa6e98864fefb95c249c100da4ae7aed36ba13a8a91415791ec6fad20bec0427 +ARROW_AWS_C_CAL_BUILD_VERSION=v0.8.8 +ARROW_AWS_C_CAL_BUILD_SHA256_CHECKSUM=45a5e5e4b9070b02c8a847ff7531068f882622a4e8ac4fed4776b7729f018ea9 +ARROW_AWS_C_COMMON_BUILD_VERSION=v0.12.2 +ARROW_AWS_C_COMMON_BUILD_SHA256_CHECKSUM=ecea168ea974f2da73b5a0adc19d9c5ebca73ca4b9f733de7c37fc453ee7d1c2 +ARROW_AWS_C_COMPRESSION_BUILD_VERSION=v0.3.1 +ARROW_AWS_C_COMPRESSION_BUILD_SHA256_CHECKSUM=d89fca17a37de762dc34f332d2da402343078da8dbd2224c46a11a88adddf754 +ARROW_AWS_C_EVENT_STREAM_BUILD_VERSION=v0.5.4 +ARROW_AWS_C_EVENT_STREAM_BUILD_SHA256_CHECKSUM=cef8b78e362836d15514110fb43a0a0c7a86b0a210d5fe25fd248a82027a7272 +ARROW_AWS_C_HTTP_BUILD_VERSION=v0.9.6 +ARROW_AWS_C_HTTP_BUILD_SHA256_CHECKSUM=39381e7b66d73b5dcf8b3afe533f3206349a7e8f0fb78c8bac469bee0f05b957 +ARROW_AWS_C_IO_BUILD_VERSION=v0.18.0 +ARROW_AWS_C_IO_BUILD_SHA256_CHECKSUM=c65a9f059dfe3208dbc92b7fc11f6d846d15e1a14cd0dabf98041ce9627cadda +ARROW_AWS_C_MQTT_BUILD_VERSION=v0.12.3 +ARROW_AWS_C_MQTT_BUILD_SHA256_CHECKSUM=c2ea5d3b34692c5b71ec4ff3efd8277af01f16706970e8851373c361abaf1d72 +ARROW_AWS_C_S3_BUILD_VERSION=v0.7.14 +ARROW_AWS_C_S3_BUILD_SHA256_CHECKSUM=1a8cd98612f5d08ac12f1c0ab7235e1750faf8fb0e7680662101626b81963a66 +ARROW_AWS_C_SDKUTILS_BUILD_VERSION=v0.2.3 +ARROW_AWS_C_SDKUTILS_BUILD_SHA256_CHECKSUM=5a0489d508341b84eea556e351717bc33524d3dfd6207ee3aba6068994ea6018 +ARROW_AWS_CHECKSUMS_BUILD_VERSION=v0.2.5 +ARROW_AWS_CHECKSUMS_BUILD_SHA256_CHECKSUM=c75f1697720d1f3bd5ac5e5a9613e0120337ef48c3c6bf1e6be3c802799ad8e4 +ARROW_AWS_CRT_CPP_BUILD_VERSION=v0.32.2 +ARROW_AWS_CRT_CPP_BUILD_SHA256_CHECKSUM=0969801f13344deaf6e4dc8d458798e0692d4a67dde5ccac3eda9f261ad771bd +ARROW_AWS_LC_BUILD_VERSION=v1.49.1 +ARROW_AWS_LC_BUILD_SHA256_CHECKSUM=2fa2e31efab7220b2e0aac581fc6d4f2a6e0e16a26b9e6037f5f137d5e57b4df +ARROW_AWSSDK_BUILD_VERSION=1.11.537 +ARROW_AWSSDK_BUILD_SHA256_CHECKSUM=4339edd9cd23248c75368a52ac69aa2b6f72f06a5f1bf6f9747f8e3957edcdae # Despite the confusing version name this is still the whole Azure SDK for C++ including core, keyvault, storage-common, etc. ARROW_AZURE_SDK_BUILD_VERSION=azure-identity_1.9.0 ARROW_AZURE_SDK_BUILD_SHA256_CHECKSUM=97065bfc971ac8df450853ce805f820f52b59457bd7556510186a1569502e4a1 @@ -105,8 +105,8 @@ ARROW_SNAPPY_BUILD_VERSION=1.2.2 ARROW_SNAPPY_BUILD_SHA256_CHECKSUM=90f74bc1fbf78a6c56b3c4a082a05103b3a56bb17bca1a27e052ea11723292dc ARROW_SUBSTRAIT_BUILD_VERSION=v0.44.0 ARROW_SUBSTRAIT_BUILD_SHA256_CHECKSUM=f989a862f694e7dbb695925ddb7c4ce06aa6c51aca945105c075139aed7e55a2 -ARROW_S2N_TLS_BUILD_VERSION=v1.5.11 -ARROW_S2N_TLS_BUILD_SHA256_CHECKSUM=5690f030da35f86e3b5d61d1de150b5b52c84eef383799f7a706bdf21227417e +ARROW_S2N_TLS_BUILD_VERSION=v1.5.15 +ARROW_S2N_TLS_BUILD_SHA256_CHECKSUM=103f9361c736fea7278038891b0566ff975c40ac59cef5ac5b9225a476c8abc6 ARROW_THRIFT_BUILD_VERSION=0.20.0 ARROW_THRIFT_BUILD_SHA256_CHECKSUM=b5d8311a779470e1502c027f428a1db542f5c051c8e1280ccd2163fa935ff2d6 ARROW_UTF8PROC_BUILD_VERSION=v2.10.0 From 9982fd687fdd2d9c6fea29e31348e04e81e489da Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Tue, 1 Apr 2025 16:28:40 +0900 Subject: [PATCH 25/52] Remove a needless code --- cpp/cmake_modules/ThirdpartyToolchain.cmake | 6 ------ 1 file changed, 6 deletions(-) diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index 7aae4268dcc..a72742e6a51 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -5108,12 +5108,6 @@ function(build_awssdk) # configurations. set(${BASE_VARIABLE_NAME}_PATCH_COMMAND ${CMAKE_COMMAND} -E rm tests/features/S2N_LIBCRYPTO_SUPPORTS_ENGINE.c) - fetchcontent_declare(${AWSSDK_PRODUCT} - ${FC_DECLARE_COMMON_OPTIONS} OVERRIDE_FIND_PACKAGE - PATCH_COMMAND ${${BASE_VARIABLE_NAME}_PATCH_COMMAND} - URL ${${BASE_VARIABLE_NAME}_SOURCE_URL - URL_HASH "SHA256=1ca1fb0a82642a93ab7c95c5a6c9ff80e3388e387fba03153186426b98d8b9e0" - ) elseif(AWSSDK_PRODUCT_NAME STREQUAL "aws-sdk-cpp" AND WINDOWS AND NOT MSVC) # We can remove this once # https://github.com/aws/aws-sdk-cpp/issues/3315 is resolved. From 697a4a9e8859fff62414e6abf7f5ee71fc7d304f Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Tue, 1 Apr 2025 16:29:15 +0900 Subject: [PATCH 26/52] Fix style --- cpp/cmake_modules/ThirdpartyToolchain.cmake | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index a72742e6a51..1cfa31eab1f 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -5108,7 +5108,9 @@ function(build_awssdk) # configurations. set(${BASE_VARIABLE_NAME}_PATCH_COMMAND ${CMAKE_COMMAND} -E rm tests/features/S2N_LIBCRYPTO_SUPPORTS_ENGINE.c) - elseif(AWSSDK_PRODUCT_NAME STREQUAL "aws-sdk-cpp" AND WINDOWS AND NOT MSVC) + elseif(AWSSDK_PRODUCT_NAME STREQUAL "aws-sdk-cpp" + AND WINDOWS + AND NOT MSVC) # We can remove this once # https://github.com/aws/aws-sdk-cpp/issues/3315 is resolved. find_program(PATCH patch REQUIRED) From e6f23d135bc0fdef10b0059038a72a7fb960986d Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Sat, 7 Jun 2025 09:50:16 +0900 Subject: [PATCH 27/52] Update --- cpp/cmake_modules/ThirdpartyToolchain.cmake | 25 +- cpp/cmake_modules/aws-sdk-cpp-pr-1333.patch | 218 ------------------ .../aws_sdk_cpp_generate_variables.sh | 72 ------ cpp/thirdparty/versions.txt | 48 ++-- 4 files changed, 29 insertions(+), 334 deletions(-) delete mode 100644 cpp/cmake_modules/aws-sdk-cpp-pr-1333.patch delete mode 100755 cpp/cmake_modules/aws_sdk_cpp_generate_variables.sh diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index 1cfa31eab1f..8c67e550816 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -5070,9 +5070,7 @@ function(build_awssdk) # aws-c-common must be the first product because others depend on # this. set(AWSSDK_PRODUCTS aws-c-common) - # aws-lc and s2n-tls only needed on Linux. - # We can use LINUX with CMake 3.25 or later. - if(UNIX AND NOT APPLE) + if(LINUX) list(APPEND AWSSDK_PRODUCTS aws-lc s2n-tls) endif() list(APPEND @@ -5100,23 +5098,6 @@ function(build_awssdk) # AWS-C-CAL -> # AWS_C_CAL string(REGEX REPLACE "-" "_" BASE_VARIABLE_NAME "${BASE_VARIABLE_NAME}") - if(AWSSDK_PRODUCT STREQUAL "s2n-tls") - # S2N_LIBCRYPTO_SUPPORTS_ENGINE.c doesn't refer - # S2N_INTERN_LIBCRYPTO. We need to use aws-lc with - # S2N_INTERN_LIBCRYPTO but S2N_LIBCRYPTO_SUPPORTS_ENGINE.c may - # refer system OpenSSL. So s2n-tls may mix aws-lc and OpenSSL - # configurations. - set(${BASE_VARIABLE_NAME}_PATCH_COMMAND - ${CMAKE_COMMAND} -E rm tests/features/S2N_LIBCRYPTO_SUPPORTS_ENGINE.c) - elseif(AWSSDK_PRODUCT_NAME STREQUAL "aws-sdk-cpp" - AND WINDOWS - AND NOT MSVC) - # We can remove this once - # https://github.com/aws/aws-sdk-cpp/issues/3315 is resolved. - find_program(PATCH patch REQUIRED) - set(${BASE_VARIABLE_NAME}_PATCH_COMMAND - patch --input=${CMAKE_CURRENT_SOURCE_DIR}/aws-sdk-cpp-pr-1333.patch --strip=1) - endif() fetchcontent_declare(${AWSSDK_PRODUCT} ${FC_DECLARE_COMMON_OPTIONS} OVERRIDE_FIND_PACKAGE PATCH_COMMAND ${${BASE_VARIABLE_NAME}_PATCH_COMMAND} @@ -5183,6 +5164,10 @@ function(build_awssdk) set(AWSSDK_LINK_LIBRARIES) foreach(AWSSDK_PRODUCT ${AWSSDK_PRODUCTS}) + if("${AWSSDK_PRODUCT}" STREQUAL "s2n-tls") + # Use aws-lc's openssl/*.h not openssl/*.h in system. + set(ADDITIONAL_FLAGS "-DCOMPILE_DEFINITIONS=-I${aws-lc_SOURCE_DIR}/include") + endif() fetchcontent_makeavailable(${AWSSDK_PRODUCT}) list(PREPEND CMAKE_MODULE_PATH "${${AWSSDK_PRODUCT}_SOURCE_DIR}/cmake") if(NOT "${AWSSDK_PRODUCT}" STREQUAL "aws-sdk-cpp") diff --git a/cpp/cmake_modules/aws-sdk-cpp-pr-1333.patch b/cpp/cmake_modules/aws-sdk-cpp-pr-1333.patch deleted file mode 100644 index f47b2f51583..00000000000 --- a/cpp/cmake_modules/aws-sdk-cpp-pr-1333.patch +++ /dev/null @@ -1,218 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# This is a copy of -# https://github.com/msys2/MINGW-packages/blob/master/mingw-w64-aws-sdk-cpp/aws-sdk-cpp-pr-1333.patch -# . This is based on https://github.com/aws/aws-sdk-cpp/pull/1333 . - -From f9b4499ccb63ef02d8c9de462fbbcf301eb1907a Mon Sep 17 00:00:00 2001 -From: Daniel Schulte -Date: Tue, 3 Mar 2020 11:09:15 +0100 -Subject: [PATCH] Add support for building with MinGW - ---- - .../include/aws/core/utils/crypto/bcrypt/CryptoImpl.h | 7 +++++++ - .../include/aws/core/utils/event/EventHeader.h | 10 ++++++++++ - .../source/http/windows/WinHttpSyncHttpClient.cpp | 4 ++-- - .../source/http/windows/WinINetSyncHttpClient.cpp | 2 +- - .../source/http/windows/WinSyncHttpClient.cpp | 2 +- - .../source/platform/windows/Environment.cpp | 5 +++++ - .../source/platform/windows/FileSystem.cpp | 7 ++++++- - .../source/platform/windows/OSVersionInfo.cpp | 2 ++ - .../source/utils/crypto/factory/Factories.cpp | 4 ++-- - .../BucketAndObjectOperationTest.cpp | 4 ++-- - cmake/compiler_settings.cmake | 6 ++++++ - .../source/platform/windows/PlatformTesting.cpp | 2 ++ - 12 files changed, 46 insertions(+), 9 deletions(-) - -diff --git a/src/aws-cpp-sdk-core/include/aws/core/utils/event/EventHeader.h b/src/aws-cpp-sdk-core/include/aws/core/utils/event/EventHeader.h -index 2041b208da..6fb333bff7 100644 ---- a/src/aws-cpp-sdk-core/include/aws/core/utils/event/EventHeader.h -+++ b/src/aws-cpp-sdk-core/include/aws/core/utils/event/EventHeader.h -@@ -14,6 +14,12 @@ - #include - #include - -+#ifdef __MINGW32__ -+#pragma GCC diagnostic push -+#pragma GCC diagnostic ignored "-Wmaybe-uninitialized" -+#pragma GCC diagnostic ignored "-Wuninitialized" -+#endif -+ - namespace Aws - { - namespace Utils -@@ -318,3 +324,7 @@ namespace Aws - } - } - } -+ -+#ifdef __MINGW32__ -+#pragma GCC diagnostic pop -+#endif -diff --git a/src/aws-cpp-sdk-core/source/http/windows/WinHttpSyncHttpClient.cpp b/src/aws-cpp-sdk-core/source/http/windows/WinHttpSyncHttpClient.cpp -index 4a0f80a161..a98635f706 100644 ---- a/src/aws-cpp-sdk-core/source/http/windows/WinHttpSyncHttpClient.cpp -+++ b/src/aws-cpp-sdk-core/source/http/windows/WinHttpSyncHttpClient.cpp -@@ -17,7 +17,7 @@ - #include - #include - --#include -+#include - #include - #include // for tcp_keepalive - #include -@@ -686,7 +686,7 @@ - wmemset(contentTypeStr, 0, static_cast(dwSize / sizeof(wchar_t))); - - WinHttpQueryHeaders(hHttpRequest, WINHTTP_QUERY_CONTENT_TYPE, nullptr, &contentTypeStr, &dwSize, 0); -- if (contentTypeStr[0] != NULL) -+ if (contentTypeStr[0]) - { - Aws::String contentStr = StringUtils::FromWString(contentTypeStr); - response->SetContentType(contentStr); -@@ -717,7 +717,7 @@ - - bool WinHttpSyncHttpClient::DoSendRequest(void* hHttpRequest) const - { -- bool success = WinHttpSendRequest(hHttpRequest, NULL, NULL, 0, 0, 0, NULL) != 0; -+ bool success = WinHttpSendRequest(hHttpRequest, NULL, 0, NULL, 0, 0, 0) != 0; - if (!success) - { - AzWinHttpLogLastError("WinHttpSendRequest"); -diff --git a/src/aws-cpp-sdk-core/source/http/windows/WinINetSyncHttpClient.cpp b/src/aws-cpp-sdk-core/source/http/windows/WinINetSyncHttpClient.cpp -index 8b42c64dab..ef1fd99975 100644 ---- a/src/aws-cpp-sdk-core/source/http/windows/WinINetSyncHttpClient.cpp -+++ b/src/aws-cpp-sdk-core/source/http/windows/WinINetSyncHttpClient.cpp -@@ -223,7 +223,7 @@ bool WinINetSyncHttpClient::DoQueryHeaders(void* hHttpRequest, std::shared_ptrSetContentType(contentTypeStr); - AWS_LOGSTREAM_DEBUG(GetLogTag(), "Received content type " << contentTypeStr); -diff --git a/src/aws-cpp-sdk-core/source/platform/windows/FileSystem.cpp b/src/aws-cpp-sdk-core/source/platform/windows/FileSystem.cpp -index 49af3c1f79..f427fa9233 100644 ---- a/src/aws-cpp-sdk-core/source/platform/windows/FileSystem.cpp -+++ b/src/aws-cpp-sdk-core/source/platform/windows/FileSystem.cpp -@@ -11,7 +11,9 @@ - #include - #include - --#pragma warning( disable : 4996) -+#ifdef _MSC_VER -+# pragma warning( disable : 4996) -+#endif - - using namespace Aws::Utils; - namespace Aws -@@ -304,6 +306,9 @@ Aws::String CreateTempFilePath() - { - #ifdef _MSC_VER - #pragma warning(disable: 4996) // _CRT_SECURE_NO_WARNINGS -+#elif !defined(L_tmpnam_s) -+ // Definition from the MSVC stdio.h -+ #define L_tmpnam_s (sizeof("\\") + 16) - #endif - char s_tempName[L_tmpnam_s+1]; - -diff --git a/src/aws-cpp-sdk-core/source/platform/windows/OSVersionInfo.cpp b/src/aws-cpp-sdk-core/source/platform/windows/OSVersionInfo.cpp -index 23f395f6bb..18bd5836da 100644 ---- a/src/aws-cpp-sdk-core/source/platform/windows/OSVersionInfo.cpp -+++ b/src/aws-cpp-sdk-core/source/platform/windows/OSVersionInfo.cpp -@@ -9,7 +9,9 @@ - - #include - -+#ifdef _MSC_VER - #pragma warning(disable: 4996) -+#endif - #include - #include - namespace Aws -diff --git a/src/aws-cpp-sdk-core/source/utils/crypto/factory/Factories.cpp b/src/aws-cpp-sdk-core/source/utils/crypto/factory/Factories.cpp -index d7cb481d5c..3b34d7fc62 100644 ---- a/src/aws-cpp-sdk-core/source/utils/crypto/factory/Factories.cpp -+++ b/src/aws-cpp-sdk-core/source/utils/crypto/factory/Factories.cpp -@@ -939,7 +939,7 @@ std::shared_ptr Aws::Utils::Crypto::CreateSha256HMACIm - return s_Sha256HMACFactory->CreateImplementation(); - } - --#ifdef _WIN32 -+#ifdef _MSC_VER - #pragma warning( push ) - #pragma warning( disable : 4702 ) - #endif -@@ -1032,7 +1032,7 @@ std::shared_ptr Aws::Utils::Crypto::CreateAES_KeyWrapImplementa - return s_AES_KeyWrapFactory->CreateImplementation(key); - } - --#ifdef _WIN32 -+#ifdef _MSC_VER - #pragma warning(pop) - #endif - -diff --git a/tests/aws-cpp-sdk-s3-integration-tests/BucketAndObjectOperationTest.cpp b/tests/aws-cpp-sdk-s3-integration-tests/BucketAndObjectOperationTest.cpp -index 0f31fcd061..586303e044 100644 ---- a/tests/aws-cpp-sdk-s3-integration-tests/BucketAndObjectOperationTest.cpp -+++ b/tests/aws-cpp-sdk-s3-integration-tests/BucketAndObjectOperationTest.cpp -@@ -49,8 +49,9 @@ - #include - #include - --#ifdef _WIN32 -+#ifdef _MSC_VER - #pragma warning(disable: 4127) -+#endif - #ifdef GetObject - #undef GetObject - #endif -diff --git a/cmake/compiler_settings.cmake b/cmake/compiler_settings.cmake -index db054b2c1b..e1d6dce55e 100644 ---- a/cmake/compiler_settings.cmake -+++ b/cmake/compiler_settings.cmake -@@ -11,6 +11,9 @@ else() - set(COMPILER_CLANG 1) - else() - set(COMPILER_GCC 1) -+ if(MINGW) -+ set(COMPILER_MINGW 1) -+ endif() - endif() - set(USE_GCC_FLAGS 1) - endif() -@@ -34,6 +37,9 @@ endfunction() - - macro(set_gcc_flags) - list(APPEND AWS_COMPILER_FLAGS "-fno-exceptions" "-std=c++${CPP_STANDARD}") -+ if(COMPILER_IS_MINGW) -+ list(APPEND AWS_COMPILER_FLAGS -D__USE_MINGW_ANSI_STDIO=1) -+ endif() - - if(NOT BUILD_SHARED_LIBS) - list(APPEND AWS_COMPILER_FLAGS "-fPIC") -diff --git a/tests/testing-resources/source/platform/windows/PlatformTesting.cpp b/tests/testing-resources/source/platform/windows/PlatformTesting.cpp -index 2b0a04c0b4..2a27710557 100644 ---- a/tests/testing-resources/source/platform/windows/PlatformTesting.cpp -+++ b/tests/testing-resources/source/platform/windows/PlatformTesting.cpp -@@ -5,7 +5,9 @@ - - #include - -+#ifdef _MSC_VER - #pragma warning(disable: 4996) -+#endif - #include - #include diff --git a/cpp/cmake_modules/aws_sdk_cpp_generate_variables.sh b/cpp/cmake_modules/aws_sdk_cpp_generate_variables.sh deleted file mode 100755 index 79b560a4a14..00000000000 --- a/cpp/cmake_modules/aws_sdk_cpp_generate_variables.sh +++ /dev/null @@ -1,72 +0,0 @@ -#!/bin/bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -eu - -version=$1 - -base_dir="$(dirname "$0")" -output="${base_dir}/AWSSDKVariables.cmake" - -cat <
${output} -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# Generated by: -# $ cpp/cmake_modules/aws_sdk_cpp_generate_variables.sh ${version} - -HEADER - -rm -f ${version}.tar.gz -wget https://github.com/aws/aws-sdk-cpp/archive/${version}.tar.gz -base_name=aws-sdk-cpp-${version} -rm -rf ${base_name} -tar xf ${version}.tar.gz - -echo "set(AWSSDK_UNUSED_DIRECTORIES" >> ${output} -find ${base_name} -mindepth 1 -maxdepth 1 -type d | \ - sort | \ - grep -v cmake | \ - grep -v toolchains | \ - grep -v aws-cpp-sdk-cognito-identity | \ - grep -v aws-cpp-sdk-core | \ - grep -v aws-cpp-sdk-config | \ - grep -v aws-cpp-sdk-s3 | \ - grep -v aws-cpp-sdk-transfer | \ - grep -v aws-cpp-sdk-identity-management | \ - grep -v aws-cpp-sdk-sts | \ - sed -E -e "s,^${base_name}/, ,g" >> ${output} -echo ")" >> ${output} - -rm -rf ${base_name} -rm -f ${version}.tar.gz diff --git a/cpp/thirdparty/versions.txt b/cpp/thirdparty/versions.txt index 01c28911394..a1d5a3bd1bc 100644 --- a/cpp/thirdparty/versions.txt +++ b/cpp/thirdparty/versions.txt @@ -27,32 +27,32 @@ ARROW_ABSL_BUILD_VERSION=20211102.0 ARROW_ABSL_BUILD_SHA256_CHECKSUM=dcf71b9cba8dc0ca9940c4b316a0c796be8fab42b070bb6b7cab62b48f0e66c4 ARROW_AWS_C_AUTH_BUILD_VERSION=v0.9.0 ARROW_AWS_C_AUTH_BUILD_SHA256_CHECKSUM=aa6e98864fefb95c249c100da4ae7aed36ba13a8a91415791ec6fad20bec0427 -ARROW_AWS_C_CAL_BUILD_VERSION=v0.8.8 -ARROW_AWS_C_CAL_BUILD_SHA256_CHECKSUM=45a5e5e4b9070b02c8a847ff7531068f882622a4e8ac4fed4776b7729f018ea9 -ARROW_AWS_C_COMMON_BUILD_VERSION=v0.12.2 -ARROW_AWS_C_COMMON_BUILD_SHA256_CHECKSUM=ecea168ea974f2da73b5a0adc19d9c5ebca73ca4b9f733de7c37fc453ee7d1c2 +ARROW_AWS_C_CAL_BUILD_VERSION=v0.9.1 +ARROW_AWS_C_CAL_BUILD_SHA256_CHECKSUM=1245f007e83a66805f7afe80ce4825f910dad0068028dd8efc3b6172e2679be5 +ARROW_AWS_C_COMMON_BUILD_VERSION=v0.12.3 +ARROW_AWS_C_COMMON_BUILD_SHA256_CHECKSUM=a4e7ac6c6f840cb6ab56b8ee0bcd94a61c59d68ca42570bca518432da4c94273 ARROW_AWS_C_COMPRESSION_BUILD_VERSION=v0.3.1 ARROW_AWS_C_COMPRESSION_BUILD_SHA256_CHECKSUM=d89fca17a37de762dc34f332d2da402343078da8dbd2224c46a11a88adddf754 ARROW_AWS_C_EVENT_STREAM_BUILD_VERSION=v0.5.4 ARROW_AWS_C_EVENT_STREAM_BUILD_SHA256_CHECKSUM=cef8b78e362836d15514110fb43a0a0c7a86b0a210d5fe25fd248a82027a7272 -ARROW_AWS_C_HTTP_BUILD_VERSION=v0.9.6 -ARROW_AWS_C_HTTP_BUILD_SHA256_CHECKSUM=39381e7b66d73b5dcf8b3afe533f3206349a7e8f0fb78c8bac469bee0f05b957 -ARROW_AWS_C_IO_BUILD_VERSION=v0.18.0 -ARROW_AWS_C_IO_BUILD_SHA256_CHECKSUM=c65a9f059dfe3208dbc92b7fc11f6d846d15e1a14cd0dabf98041ce9627cadda -ARROW_AWS_C_MQTT_BUILD_VERSION=v0.12.3 -ARROW_AWS_C_MQTT_BUILD_SHA256_CHECKSUM=c2ea5d3b34692c5b71ec4ff3efd8277af01f16706970e8851373c361abaf1d72 -ARROW_AWS_C_S3_BUILD_VERSION=v0.7.14 -ARROW_AWS_C_S3_BUILD_SHA256_CHECKSUM=1a8cd98612f5d08ac12f1c0ab7235e1750faf8fb0e7680662101626b81963a66 -ARROW_AWS_C_SDKUTILS_BUILD_VERSION=v0.2.3 -ARROW_AWS_C_SDKUTILS_BUILD_SHA256_CHECKSUM=5a0489d508341b84eea556e351717bc33524d3dfd6207ee3aba6068994ea6018 -ARROW_AWS_CHECKSUMS_BUILD_VERSION=v0.2.5 -ARROW_AWS_CHECKSUMS_BUILD_SHA256_CHECKSUM=c75f1697720d1f3bd5ac5e5a9613e0120337ef48c3c6bf1e6be3c802799ad8e4 -ARROW_AWS_CRT_CPP_BUILD_VERSION=v0.32.2 -ARROW_AWS_CRT_CPP_BUILD_SHA256_CHECKSUM=0969801f13344deaf6e4dc8d458798e0692d4a67dde5ccac3eda9f261ad771bd -ARROW_AWS_LC_BUILD_VERSION=v1.49.1 -ARROW_AWS_LC_BUILD_SHA256_CHECKSUM=2fa2e31efab7220b2e0aac581fc6d4f2a6e0e16a26b9e6037f5f137d5e57b4df -ARROW_AWSSDK_BUILD_VERSION=1.11.537 -ARROW_AWSSDK_BUILD_SHA256_CHECKSUM=4339edd9cd23248c75368a52ac69aa2b6f72f06a5f1bf6f9747f8e3957edcdae +ARROW_AWS_C_HTTP_BUILD_VERSION=v0.10.1 +ARROW_AWS_C_HTTP_BUILD_SHA256_CHECKSUM=1550f7bf9666bb8f86514db9e623f07249e3c53e868d2f36ff69b83bd3eadfec +ARROW_AWS_C_IO_BUILD_VERSION=v0.19.1 +ARROW_AWS_C_IO_BUILD_SHA256_CHECKSUM=f2fea0c066924f7fe3c2b1c7b2fa9be640f5b16a6514854226330e63a1faacd0 +ARROW_AWS_C_MQTT_BUILD_VERSION=v0.13.1 +ARROW_AWS_C_MQTT_BUILD_SHA256_CHECKSUM=c54d02c1e46f55bae8d5e6f9c4b0d78d84c1c9d9ac16ba8d78c3361edcd8b5bb +ARROW_AWS_C_S3_BUILD_VERSION=v0.8.0 +ARROW_AWS_C_S3_BUILD_SHA256_CHECKSUM=0b2f2a6d3b17c6d0684b80cc6581dd1b99ced39bfbb633fc9a1b16bf3f8eaa66 +ARROW_AWS_C_SDKUTILS_BUILD_VERSION=v0.2.4 +ARROW_AWS_C_SDKUTILS_BUILD_SHA256_CHECKSUM=493cbed4fa57e0d4622fcff044e11305eb4fc12445f32c8861025597939175fc +ARROW_AWS_CHECKSUMS_BUILD_VERSION=v0.2.7 +ARROW_AWS_CHECKSUMS_BUILD_SHA256_CHECKSUM=178e8398d98111f29150f7813a70c20ad97ab30be0de02525440355fe84ccb1d +ARROW_AWS_CRT_CPP_BUILD_VERSION=v0.32.8 +ARROW_AWS_CRT_CPP_BUILD_SHA256_CHECKSUM=db44260452a0296341fb8e7b987e4c328f08f7829b9f1c740fed9c963e081e93 +ARROW_AWS_LC_BUILD_VERSION=v1.52.1 +ARROW_AWS_LC_BUILD_SHA256_CHECKSUM=fe552e3c3522f73afc3c30011745c431c633f7b4e25dcd7b38325f194a7b3b75 +ARROW_AWSSDK_BUILD_VERSION=1.11.581 +ARROW_AWSSDK_BUILD_SHA256_CHECKSUM=7c7bed16047f3092e7c1cf8de1e0e4d8b7ce008c68734d2cb7215f3c29609230 # Despite the confusing version name this is still the whole Azure SDK for C++ including core, keyvault, storage-common, etc. ARROW_AZURE_SDK_BUILD_VERSION=azure-identity_1.9.0 ARROW_AZURE_SDK_BUILD_SHA256_CHECKSUM=97065bfc971ac8df450853ce805f820f52b59457bd7556510186a1569502e4a1 @@ -105,8 +105,8 @@ ARROW_SNAPPY_BUILD_VERSION=1.2.2 ARROW_SNAPPY_BUILD_SHA256_CHECKSUM=90f74bc1fbf78a6c56b3c4a082a05103b3a56bb17bca1a27e052ea11723292dc ARROW_SUBSTRAIT_BUILD_VERSION=v0.44.0 ARROW_SUBSTRAIT_BUILD_SHA256_CHECKSUM=f989a862f694e7dbb695925ddb7c4ce06aa6c51aca945105c075139aed7e55a2 -ARROW_S2N_TLS_BUILD_VERSION=v1.5.15 -ARROW_S2N_TLS_BUILD_SHA256_CHECKSUM=103f9361c736fea7278038891b0566ff975c40ac59cef5ac5b9225a476c8abc6 +ARROW_S2N_TLS_BUILD_VERSION=v1.5.20 +ARROW_S2N_TLS_BUILD_SHA256_CHECKSUM=1d05303ba8383f62273c51b50147391c23375e918d525b7c827f7aeb69e6b102 ARROW_THRIFT_BUILD_VERSION=0.20.0 ARROW_THRIFT_BUILD_SHA256_CHECKSUM=b5d8311a779470e1502c027f428a1db542f5c051c8e1280ccd2163fa935ff2d6 ARROW_UTF8PROC_BUILD_VERSION=v2.10.0 From 9f5e214317ed66b3ae63cf864ace1a2843c019ac Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Mon, 9 Jun 2025 15:31:05 +0900 Subject: [PATCH 28/52] Use cache variable --- cpp/cmake_modules/ThirdpartyToolchain.cmake | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index 8c67e550816..e1fc6267894 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -1021,19 +1021,19 @@ if(CMAKE_VERSION VERSION_GREATER_EQUAL 3.25) endif() macro(prepare_fetchcontent) - set(BUILD_SHARED_LIBS OFF) - set(BUILD_STATIC_LIBS ON) - set(BUILD_TESTING OFF) - set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY "") - set(CMAKE_COMPILE_WARNING_AS_ERROR FALSE) - set(CMAKE_EXPORT_NO_PACKAGE_REGISTRY TRUE) - set(CMAKE_LIBRARY_OUTPUT_DIRECTORY "") - set(CMAKE_MACOSX_RPATH ${ARROW_INSTALL_NAME_RPATH}) + set(BUILD_SHARED_LIBS OFF CACHE BOOL "" FORCE) + set(BUILD_STATIC_LIBS ON CACHE BOOL "" FORCE) + set(BUILD_TESTING OFF CACHE BOOL "" FORCE) + set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY "" CACHE PATH "" FORCE) + set(CMAKE_COMPILE_WARNING_AS_ERROR FALSE CACHE BOOL "" FORCE) + set(CMAKE_EXPORT_NO_PACKAGE_REGISTRY TRUE CACHE BOOL "" FORCE) + set(CMAKE_LIBRARY_OUTPUT_DIRECTORY "" CACHE PATH "" FORCE) + set(CMAKE_MACOSX_RPATH ${ARROW_INSTALL_NAME_RPATH} CACHE BOOL "" FORCE) # We set CMAKE_POLICY_VERSION_MINIMUM temporarily due to failures with CMake 4 # We should remove it once we have updated the dependencies: # https://github.com/apache/arrow/issues/45985 - set(CMAKE_POLICY_VERSION_MINIMUM 3.5) - set(CMAKE_RUNTIME_OUTPUT_DIRECTORY "") + set(CMAKE_POLICY_VERSION_MINIMUM 3.5 CACHE STRING "" FORCE) + set(CMAKE_RUNTIME_OUTPUT_DIRECTORY "" CACHE PATH "" FORCE) if(MSVC) string(REPLACE "/WX" "" CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG}") From 28a73d04fc3d9b6065e5b9dd544ad6131f1ab164 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Mon, 9 Jun 2025 15:33:08 +0900 Subject: [PATCH 29/52] Fix style --- cpp/cmake_modules/ThirdpartyToolchain.cmake | 40 +++++++++++++++------ 1 file changed, 30 insertions(+), 10 deletions(-) diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index e1fc6267894..16166c6dbc8 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -1021,19 +1021,39 @@ if(CMAKE_VERSION VERSION_GREATER_EQUAL 3.25) endif() macro(prepare_fetchcontent) - set(BUILD_SHARED_LIBS OFF CACHE BOOL "" FORCE) - set(BUILD_STATIC_LIBS ON CACHE BOOL "" FORCE) - set(BUILD_TESTING OFF CACHE BOOL "" FORCE) - set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY "" CACHE PATH "" FORCE) - set(CMAKE_COMPILE_WARNING_AS_ERROR FALSE CACHE BOOL "" FORCE) - set(CMAKE_EXPORT_NO_PACKAGE_REGISTRY TRUE CACHE BOOL "" FORCE) - set(CMAKE_LIBRARY_OUTPUT_DIRECTORY "" CACHE PATH "" FORCE) - set(CMAKE_MACOSX_RPATH ${ARROW_INSTALL_NAME_RPATH} CACHE BOOL "" FORCE) + set(BUILD_SHARED_LIBS + OFF + CACHE BOOL "" FORCE) + set(BUILD_STATIC_LIBS + ON + CACHE BOOL "" FORCE) + set(BUILD_TESTING + OFF + CACHE BOOL "" FORCE) + set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY + "" + CACHE PATH "" FORCE) + set(CMAKE_COMPILE_WARNING_AS_ERROR + FALSE + CACHE BOOL "" FORCE) + set(CMAKE_EXPORT_NO_PACKAGE_REGISTRY + TRUE + CACHE BOOL "" FORCE) + set(CMAKE_LIBRARY_OUTPUT_DIRECTORY + "" + CACHE PATH "" FORCE) + set(CMAKE_MACOSX_RPATH + ${ARROW_INSTALL_NAME_RPATH} + CACHE BOOL "" FORCE) # We set CMAKE_POLICY_VERSION_MINIMUM temporarily due to failures with CMake 4 # We should remove it once we have updated the dependencies: # https://github.com/apache/arrow/issues/45985 - set(CMAKE_POLICY_VERSION_MINIMUM 3.5 CACHE STRING "" FORCE) - set(CMAKE_RUNTIME_OUTPUT_DIRECTORY "" CACHE PATH "" FORCE) + set(CMAKE_POLICY_VERSION_MINIMUM + 3.5 + CACHE STRING "" FORCE) + set(CMAKE_RUNTIME_OUTPUT_DIRECTORY + "" + CACHE PATH "" FORCE) if(MSVC) string(REPLACE "/WX" "" CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG}") From 9f4644c14b39f8192a866e9e73ddf69cb209cabe Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Mon, 9 Jun 2025 16:11:10 +0900 Subject: [PATCH 30/52] Add support for Apache projects --- cpp/thirdparty/update.rb | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/cpp/thirdparty/update.rb b/cpp/thirdparty/update.rb index 98571374c09..477d4c5a14d 100755 --- a/cpp/thirdparty/update.rb +++ b/cpp/thirdparty/update.rb @@ -90,6 +90,30 @@ def update_product_github(product, metadata, repository) $stderr.puts(" Checksum: #{metadata[:checksum]}") end +def update_product_apache(product, metadata, project) + version = metadata[:version] + version_directory_pattern = metadata[:version_directory_template] % { + version: "(\\d+(?:\\.\\d+)+)", + } + versions = URI.open("https://downloads.apache.org/#{project}/") do |response| + response.read.scan(//).flatten + end + latest_version = versions.last + return if version == latest_version + + url_template = metadata[:url_template] + url = url_template % { + version: latest_version, + version_underscore: latest_version.gsub(".", "_"), + } + $stderr.puts("Updating #{product}: #{version} -> #{latest_version}") + metadata[:version] = latest_version + URI.open(url, "rb") do |response| + metadata[:checksum] = Digest::SHA256.hexdigest(response.read) + end + $stderr.puts(" Checksum: #{metadata[:checksum]}") +end + def update_product(product, metadata) url_template = metadata[:url_template] if url_template.nil? @@ -99,9 +123,14 @@ def update_product(product, metadata) end case url_template - when /\Ahttps:\/\/github.com\/((?:[^\/]+)\/(?:[^\/]+))\// + when /\Ahttps:\/\/github\.com\/((?:[^\/]+)\/(?:[^\/]+))\// github_repository = Regexp.last_match[1] update_product_github(product, metadata, github_repository) + when /\Ahttps:\/\/www\.apache\.org\/dyn\/closer\.lua\/ + ((?:[^\/]+))\/((?:[^\/]+))\//x + apache_project = Regexp.last_match[1] + metadata[:version_directory_template] = Regexp.last_match[2] + update_product_apache(product, metadata, apache_project) else $stderr.puts("TODO: #{product} isn't supported yet: #{url_template}") end From 13ee0604c255ee12792ad1da89ad156bb7a90718 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Tue, 10 Jun 2025 05:55:53 +0900 Subject: [PATCH 31/52] Remove unused file --- cpp/cmake_modules/AWSSDKVariables.cmake | 388 ------------------------ 1 file changed, 388 deletions(-) delete mode 100644 cpp/cmake_modules/AWSSDKVariables.cmake diff --git a/cpp/cmake_modules/AWSSDKVariables.cmake b/cpp/cmake_modules/AWSSDKVariables.cmake deleted file mode 100644 index 729790dd0f8..00000000000 --- a/cpp/cmake_modules/AWSSDKVariables.cmake +++ /dev/null @@ -1,388 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# Generated by: -# $ cpp/cmake_modules/aws_sdk_cpp_generate_variables.sh 1.10.55 - -set(AWSSDK_UNUSED_DIRECTORIES - .github - AndroidSDKTesting - CI - Docs - android-build - android-unified-tests - aws-cpp-sdk-AWSMigrationHub - aws-cpp-sdk-access-management - aws-cpp-sdk-accessanalyzer - aws-cpp-sdk-account - aws-cpp-sdk-acm - aws-cpp-sdk-acm-pca - aws-cpp-sdk-alexaforbusiness - aws-cpp-sdk-amp - aws-cpp-sdk-amplify - aws-cpp-sdk-amplifybackend - aws-cpp-sdk-amplifyuibuilder - aws-cpp-sdk-apigateway - aws-cpp-sdk-apigatewaymanagementapi - aws-cpp-sdk-apigatewayv2 - aws-cpp-sdk-appconfig - aws-cpp-sdk-appconfigdata - aws-cpp-sdk-appflow - aws-cpp-sdk-appintegrations - aws-cpp-sdk-application-autoscaling - aws-cpp-sdk-application-insights - aws-cpp-sdk-applicationcostprofiler - aws-cpp-sdk-appmesh - aws-cpp-sdk-apprunner - aws-cpp-sdk-appstream - aws-cpp-sdk-appsync - aws-cpp-sdk-arc-zonal-shift - aws-cpp-sdk-athena - aws-cpp-sdk-auditmanager - aws-cpp-sdk-autoscaling - aws-cpp-sdk-autoscaling-plans - aws-cpp-sdk-awstransfer - aws-cpp-sdk-backup - aws-cpp-sdk-backup-gateway - aws-cpp-sdk-backupstorage - aws-cpp-sdk-batch - aws-cpp-sdk-billingconductor - aws-cpp-sdk-braket - aws-cpp-sdk-budgets - aws-cpp-sdk-ce - aws-cpp-sdk-chime - aws-cpp-sdk-chime-sdk-identity - aws-cpp-sdk-chime-sdk-media-pipelines - aws-cpp-sdk-chime-sdk-meetings - aws-cpp-sdk-chime-sdk-messaging - aws-cpp-sdk-chime-sdk-voice - aws-cpp-sdk-cleanrooms - aws-cpp-sdk-cloud9 - aws-cpp-sdk-cloudcontrol - aws-cpp-sdk-clouddirectory - aws-cpp-sdk-cloudformation - aws-cpp-sdk-cloudfront - aws-cpp-sdk-cloudfront-integration-tests - aws-cpp-sdk-cloudhsm - aws-cpp-sdk-cloudhsmv2 - aws-cpp-sdk-cloudsearch - aws-cpp-sdk-cloudsearchdomain - aws-cpp-sdk-cloudtrail - aws-cpp-sdk-codeartifact - aws-cpp-sdk-codebuild - aws-cpp-sdk-codecatalyst - aws-cpp-sdk-codecommit - aws-cpp-sdk-codedeploy - aws-cpp-sdk-codeguru-reviewer - aws-cpp-sdk-codeguruprofiler - aws-cpp-sdk-codepipeline - aws-cpp-sdk-codestar - aws-cpp-sdk-codestar-connections - aws-cpp-sdk-codestar-notifications - aws-cpp-sdk-cognito-idp - aws-cpp-sdk-cognito-sync - aws-cpp-sdk-cognitoidentity-integration-tests - aws-cpp-sdk-comprehend - aws-cpp-sdk-comprehendmedical - aws-cpp-sdk-compute-optimizer - aws-cpp-sdk-connect - aws-cpp-sdk-connect-contact-lens - aws-cpp-sdk-connectcampaigns - aws-cpp-sdk-connectcases - aws-cpp-sdk-connectparticipant - aws-cpp-sdk-controltower - aws-cpp-sdk-cur - aws-cpp-sdk-custom-service-integration-tests - aws-cpp-sdk-customer-profiles - aws-cpp-sdk-databrew - aws-cpp-sdk-dataexchange - aws-cpp-sdk-datapipeline - aws-cpp-sdk-datasync - aws-cpp-sdk-dax - aws-cpp-sdk-detective - aws-cpp-sdk-devicefarm - aws-cpp-sdk-devops-guru - aws-cpp-sdk-directconnect - aws-cpp-sdk-discovery - aws-cpp-sdk-dlm - aws-cpp-sdk-dms - aws-cpp-sdk-docdb - aws-cpp-sdk-docdb-elastic - aws-cpp-sdk-drs - aws-cpp-sdk-ds - aws-cpp-sdk-dynamodb - aws-cpp-sdk-dynamodb-integration-tests - aws-cpp-sdk-dynamodbstreams - aws-cpp-sdk-ebs - aws-cpp-sdk-ec2 - aws-cpp-sdk-ec2-instance-connect - aws-cpp-sdk-ec2-integration-tests - aws-cpp-sdk-ecr - aws-cpp-sdk-ecr-public - aws-cpp-sdk-ecs - aws-cpp-sdk-eks - aws-cpp-sdk-elastic-inference - aws-cpp-sdk-elasticache - aws-cpp-sdk-elasticbeanstalk - aws-cpp-sdk-elasticfilesystem - aws-cpp-sdk-elasticfilesystem-integration-tests - aws-cpp-sdk-elasticloadbalancing - aws-cpp-sdk-elasticloadbalancingv2 - aws-cpp-sdk-elasticmapreduce - aws-cpp-sdk-elastictranscoder - aws-cpp-sdk-email - aws-cpp-sdk-emr-containers - aws-cpp-sdk-emr-serverless - aws-cpp-sdk-es - aws-cpp-sdk-eventbridge - aws-cpp-sdk-eventbridge-tests - aws-cpp-sdk-events - aws-cpp-sdk-evidently - aws-cpp-sdk-finspace - aws-cpp-sdk-finspace-data - aws-cpp-sdk-firehose - aws-cpp-sdk-fis - aws-cpp-sdk-fms - aws-cpp-sdk-forecast - aws-cpp-sdk-forecastquery - aws-cpp-sdk-frauddetector - aws-cpp-sdk-fsx - aws-cpp-sdk-gamelift - aws-cpp-sdk-gamesparks - aws-cpp-sdk-glacier - aws-cpp-sdk-globalaccelerator - aws-cpp-sdk-glue - aws-cpp-sdk-grafana - aws-cpp-sdk-greengrass - aws-cpp-sdk-greengrassv2 - aws-cpp-sdk-groundstation - aws-cpp-sdk-guardduty - aws-cpp-sdk-health - aws-cpp-sdk-healthlake - aws-cpp-sdk-honeycode - aws-cpp-sdk-iam - aws-cpp-sdk-identitystore - aws-cpp-sdk-imagebuilder - aws-cpp-sdk-importexport - aws-cpp-sdk-inspector - aws-cpp-sdk-inspector2 - aws-cpp-sdk-iot - aws-cpp-sdk-iot-data - aws-cpp-sdk-iot-jobs-data - aws-cpp-sdk-iot-roborunner - aws-cpp-sdk-iot1click-devices - aws-cpp-sdk-iot1click-projects - aws-cpp-sdk-iotanalytics - aws-cpp-sdk-iotdeviceadvisor - aws-cpp-sdk-iotevents - aws-cpp-sdk-iotevents-data - aws-cpp-sdk-iotfleethub - aws-cpp-sdk-iotfleetwise - aws-cpp-sdk-iotsecuretunneling - aws-cpp-sdk-iotsitewise - aws-cpp-sdk-iotthingsgraph - aws-cpp-sdk-iottwinmaker - aws-cpp-sdk-iotwireless - aws-cpp-sdk-ivs - aws-cpp-sdk-ivschat - aws-cpp-sdk-kafka - aws-cpp-sdk-kafkaconnect - aws-cpp-sdk-kendra - aws-cpp-sdk-kendra-ranking - aws-cpp-sdk-keyspaces - aws-cpp-sdk-kinesis - aws-cpp-sdk-kinesis-integration-tests - aws-cpp-sdk-kinesis-video-archived-media - aws-cpp-sdk-kinesis-video-media - aws-cpp-sdk-kinesis-video-signaling - aws-cpp-sdk-kinesis-video-webrtc-storage - aws-cpp-sdk-kinesisanalytics - aws-cpp-sdk-kinesisanalyticsv2 - aws-cpp-sdk-kinesisvideo - aws-cpp-sdk-kms - aws-cpp-sdk-lakeformation - aws-cpp-sdk-lambda - aws-cpp-sdk-lambda-integration-tests - aws-cpp-sdk-lex - aws-cpp-sdk-lex-models - aws-cpp-sdk-lexv2-models - aws-cpp-sdk-lexv2-runtime - aws-cpp-sdk-license-manager - aws-cpp-sdk-license-manager-linux-subscriptions - aws-cpp-sdk-license-manager-user-subscriptions - aws-cpp-sdk-lightsail - aws-cpp-sdk-location - aws-cpp-sdk-logs - aws-cpp-sdk-logs-integration-tests - aws-cpp-sdk-lookoutequipment - aws-cpp-sdk-lookoutmetrics - aws-cpp-sdk-lookoutvision - aws-cpp-sdk-m2 - aws-cpp-sdk-machinelearning - aws-cpp-sdk-macie - aws-cpp-sdk-macie2 - aws-cpp-sdk-managedblockchain - aws-cpp-sdk-marketplace-catalog - aws-cpp-sdk-marketplace-entitlement - aws-cpp-sdk-marketplacecommerceanalytics - aws-cpp-sdk-mediaconnect - aws-cpp-sdk-mediaconvert - aws-cpp-sdk-medialive - aws-cpp-sdk-mediapackage - aws-cpp-sdk-mediapackage-vod - aws-cpp-sdk-mediastore - aws-cpp-sdk-mediastore-data - aws-cpp-sdk-mediastore-data-integration-tests - aws-cpp-sdk-mediatailor - aws-cpp-sdk-memorydb - aws-cpp-sdk-meteringmarketplace - aws-cpp-sdk-mgn - aws-cpp-sdk-migration-hub-refactor-spaces - aws-cpp-sdk-migrationhub-config - aws-cpp-sdk-migrationhuborchestrator - aws-cpp-sdk-migrationhubstrategy - aws-cpp-sdk-mobile - aws-cpp-sdk-monitoring - aws-cpp-sdk-mq - aws-cpp-sdk-mturk-requester - aws-cpp-sdk-mwaa - aws-cpp-sdk-neptune - aws-cpp-sdk-network-firewall - aws-cpp-sdk-networkmanager - aws-cpp-sdk-nimble - aws-cpp-sdk-oam - aws-cpp-sdk-omics - aws-cpp-sdk-opensearch - aws-cpp-sdk-opensearchserverless - aws-cpp-sdk-opsworks - aws-cpp-sdk-opsworkscm - aws-cpp-sdk-organizations - aws-cpp-sdk-outposts - aws-cpp-sdk-panorama - aws-cpp-sdk-personalize - aws-cpp-sdk-personalize-events - aws-cpp-sdk-personalize-runtime - aws-cpp-sdk-pi - aws-cpp-sdk-pinpoint - aws-cpp-sdk-pinpoint-email - aws-cpp-sdk-pinpoint-sms-voice-v2 - aws-cpp-sdk-pipes - aws-cpp-sdk-polly - aws-cpp-sdk-polly-sample - aws-cpp-sdk-pricing - aws-cpp-sdk-privatenetworks - aws-cpp-sdk-proton - aws-cpp-sdk-qldb - aws-cpp-sdk-qldb-session - aws-cpp-sdk-queues - aws-cpp-sdk-quicksight - aws-cpp-sdk-ram - aws-cpp-sdk-rbin - aws-cpp-sdk-rds - aws-cpp-sdk-rds-data - aws-cpp-sdk-rds-integration-tests - aws-cpp-sdk-redshift - aws-cpp-sdk-redshift-data - aws-cpp-sdk-redshift-integration-tests - aws-cpp-sdk-redshift-serverless - aws-cpp-sdk-rekognition - aws-cpp-sdk-resiliencehub - aws-cpp-sdk-resource-explorer-2 - aws-cpp-sdk-resource-groups - aws-cpp-sdk-resourcegroupstaggingapi - aws-cpp-sdk-robomaker - aws-cpp-sdk-rolesanywhere - aws-cpp-sdk-route53 - aws-cpp-sdk-route53-recovery-cluster - aws-cpp-sdk-route53-recovery-control-config - aws-cpp-sdk-route53-recovery-readiness - aws-cpp-sdk-route53domains - aws-cpp-sdk-route53resolver - aws-cpp-sdk-rum - aws-cpp-sdk-sagemaker - aws-cpp-sdk-sagemaker-a2i-runtime - aws-cpp-sdk-sagemaker-edge - aws-cpp-sdk-sagemaker-featurestore-runtime - aws-cpp-sdk-sagemaker-geospatial - aws-cpp-sdk-sagemaker-metrics - aws-cpp-sdk-sagemaker-runtime - aws-cpp-sdk-savingsplans - aws-cpp-sdk-scheduler - aws-cpp-sdk-schemas - aws-cpp-sdk-sdb - aws-cpp-sdk-secretsmanager - aws-cpp-sdk-securityhub - aws-cpp-sdk-securitylake - aws-cpp-sdk-serverlessrepo - aws-cpp-sdk-service-quotas - aws-cpp-sdk-servicecatalog - aws-cpp-sdk-servicecatalog-appregistry - aws-cpp-sdk-servicediscovery - aws-cpp-sdk-sesv2 - aws-cpp-sdk-shield - aws-cpp-sdk-signer - aws-cpp-sdk-simspaceweaver - aws-cpp-sdk-sms - aws-cpp-sdk-sms-voice - aws-cpp-sdk-snow-device-management - aws-cpp-sdk-snowball - aws-cpp-sdk-sns - aws-cpp-sdk-sqs - aws-cpp-sdk-sqs-integration-tests - aws-cpp-sdk-ssm - aws-cpp-sdk-ssm-contacts - aws-cpp-sdk-ssm-incidents - aws-cpp-sdk-ssm-sap - aws-cpp-sdk-sso - aws-cpp-sdk-sso-admin - aws-cpp-sdk-sso-oidc - aws-cpp-sdk-states - aws-cpp-sdk-storagegateway - aws-cpp-sdk-support - aws-cpp-sdk-support-app - aws-cpp-sdk-swf - aws-cpp-sdk-synthetics - aws-cpp-sdk-text-to-speech - aws-cpp-sdk-text-to-speech-tests - aws-cpp-sdk-textract - aws-cpp-sdk-timestream-query - aws-cpp-sdk-timestream-write - aws-cpp-sdk-transcribe - aws-cpp-sdk-transcribestreaming - aws-cpp-sdk-transcribestreaming-integration-tests - aws-cpp-sdk-translate - aws-cpp-sdk-voice-id - aws-cpp-sdk-waf - aws-cpp-sdk-waf-regional - aws-cpp-sdk-wafv2 - aws-cpp-sdk-wellarchitected - aws-cpp-sdk-wisdom - aws-cpp-sdk-workdocs - aws-cpp-sdk-worklink - aws-cpp-sdk-workmail - aws-cpp-sdk-workmailmessageflow - aws-cpp-sdk-workspaces - aws-cpp-sdk-workspaces-web - aws-cpp-sdk-xray - code-generation - crt - doc_crosslinks - doc_crosslinks_new - doxygen - generated - scripts - testing-resources) From 3346b8d09619de0e9d5831813aca8ee1e74236ff Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Tue, 10 Jun 2025 06:11:17 +0900 Subject: [PATCH 32/52] Add docstring --- cpp/thirdparty/update.rb | 70 +++++++++++++++++++++++++++------------- 1 file changed, 48 insertions(+), 22 deletions(-) diff --git a/cpp/thirdparty/update.rb b/cpp/thirdparty/update.rb index 477d4c5a14d..77f22de04fb 100755 --- a/cpp/thirdparty/update.rb +++ b/cpp/thirdparty/update.rb @@ -31,6 +31,24 @@ exit(false) end +# Extract product information from the `cpp/thirdparty/versions.txt` +# content. +# +# Output: +# +# { +# "ABSL" => { +# version: "20211102.0", +# checksum: "dcf71b9cba8dc0ca9940c4b316a0c796be8fab42b070bb6b7cab62b48f0e66c4", +# url_template: "https://github.com/abseil/abseil-cpp/archive/%{version}.tar.gz" +# }, +# "AWS_C_AUTH" => { +# version: "v0.9.0", +# checksum: "aa6e98864fefb95c249c100da4ae7aed36ba13a8a91415791ec6fad20bec0427", +# url_template: "https://github.com/awslabs/aws-c-auth/archive/%{version}.tar.gz", +# }, +# ... +# } def parse_versions_txt_content(content) products = {} content.each_line(chomp: true) do |line| @@ -59,6 +77,28 @@ def parse_versions_txt_content(content) products end +# Update `metadata[:version]` and `metadata[:checksum]` for +# `latest_version`. +# +# This is used by product specific `#update_product_*` such as +# `#update_product_github` and `#update_product_apache`. +def update_product_generic(product, metadata, latest_version) + version = metadata[:version] + url_template = metadata[:url_template] + url = url_template % { + version: latest_version, + version_underscore: latest_version.gsub(".", "_"), + } + $stderr.puts("Updating #{product}: #{version} -> #{latest_version}") + metadata[:version] = latest_version + URI.open(url, "rb") do |response| + metadata[:checksum] = Digest::SHA256.hexdigest(response.read) + end + $stderr.puts(" Checksum: #{metadata[:checksum]}") +end + +# Update metadata to the latest version. This is for products hosted +# on GitHub. def update_product_github(product, metadata, repository) version = metadata[:version] tags_url = "https://api.github.com/repos/#{repository}/tags" @@ -77,19 +117,11 @@ def update_product_github(product, metadata, repository) end return if version == latest_version - url_template = metadata[:url_template] - url = url_template % { - version: latest_version, - version_underscore: latest_version.gsub(".", "_"), - } - $stderr.puts("Updating #{product}: #{version} -> #{latest_version}") - metadata[:version] = latest_version - URI.open(url, "rb") do |response| - metadata[:checksum] = Digest::SHA256.hexdigest(response.read) - end - $stderr.puts(" Checksum: #{metadata[:checksum]}") + update_product_generic(product, metadata, latest_version) end +# Update metadata to the latest version. This is for products +# developed by Apache Software Foundation. def update_product_apache(product, metadata, project) version = metadata[:version] version_directory_pattern = metadata[:version_directory_template] % { @@ -101,19 +133,10 @@ def update_product_apache(product, metadata, project) latest_version = versions.last return if version == latest_version - url_template = metadata[:url_template] - url = url_template % { - version: latest_version, - version_underscore: latest_version.gsub(".", "_"), - } - $stderr.puts("Updating #{product}: #{version} -> #{latest_version}") - metadata[:version] = latest_version - URI.open(url, "rb") do |response| - metadata[:checksum] = Digest::SHA256.hexdigest(response.read) - end - $stderr.puts(" Checksum: #{metadata[:checksum]}") + update_product_generic(product, metadata, latest_version) end +# Update one product to the latest version. def update_product(product, metadata) url_template = metadata[:url_template] if url_template.nil? @@ -136,6 +159,9 @@ def update_product(product, metadata) end end +# Update `versions.txt` content with `products`. `products` must be +# the same structure as `Hash` returned by +# `#parse_versions_txt_content`. def update_versions_txt_content!(content, products) products.each do |product, metadata| prefix = "ARROW_#{Regexp.escape(product)}" From 5b02f7d89f0f7bb7273d606a072c26d534cbb44c Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Tue, 10 Jun 2025 06:16:59 +0900 Subject: [PATCH 33/52] Add a patch --- cpp/cmake_modules/ThirdpartyToolchain.cmake | 5 ++ cpp/cmake_modules/aws-c-common-1208.patch | 66 +++++++++++++++++++++ 2 files changed, 71 insertions(+) create mode 100644 cpp/cmake_modules/aws-c-common-1208.patch diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index 16166c6dbc8..252c6690412 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -5118,6 +5118,11 @@ function(build_awssdk) # AWS-C-CAL -> # AWS_C_CAL string(REGEX REPLACE "-" "_" BASE_VARIABLE_NAME "${BASE_VARIABLE_NAME}") + if(MINGW AND AWSSDK_PRODUCT STREQUAL "aws-c-common") + find_program(PATCH patch REQUIRED) + set(${BASE_VARIABLE_NAME}_PATCH_COMMAND + ${PATCH} -p1 -i ${CMAKE_CURRENT_LIST_DIR}/aws-c-common-1208.patch) + endif() fetchcontent_declare(${AWSSDK_PRODUCT} ${FC_DECLARE_COMMON_OPTIONS} OVERRIDE_FIND_PACKAGE PATCH_COMMAND ${${BASE_VARIABLE_NAME}_PATCH_COMMAND} diff --git a/cpp/cmake_modules/aws-c-common-1208.patch b/cpp/cmake_modules/aws-c-common-1208.patch new file mode 100644 index 00000000000..644f03ea492 --- /dev/null +++ b/cpp/cmake_modules/aws-c-common-1208.patch @@ -0,0 +1,66 @@ +From 06c435c6765833734c62ac45909b5ab1fc5276a7 Mon Sep 17 00:00:00 2001 +From: Sutou Kouhei +Date: Sat, 7 Jun 2025 20:40:36 +0900 +Subject: [PATCH] Use _WIN32 not _MSC_VER for MinGW + +This was missed in https://github.com/awslabs/aws-c-common/pull/801 +and https://github.com/awslabs/aws-c-common/pull/822 . + +MSYS2 also includes this change: +https://github.com/msys2/MINGW-packages/blob/master/mingw-w64-aws-c-common/001-fix-build-on-mingw-aarch64.patch + +Error message without this: + +https://github.com/ursacomputing/crossbow/actions/runs/15502494926/job/43652596580#step:7:5884 + + cd /D/a/crossbow/crossbow/src/build-i686-cpp/_deps/aws-c-common-build && /D/a/crossbow/crossbow/sccache/sccache.exe /C/rtools40/mingw32/bin/cc.exe -DARROW_HAVE_RUNTIME_SSE4_2 -DARROW_HAVE_SSE4_2 -DARROW_WITH_TIMING_TESTS -DAWS_AFFINITY_METHOD=AWS_AFFINITY_METHOD_NONE -DCJSON_HIDE_SYMBOLS -DINTEL_NO_ITTNOTIFY_API -DPSAPI_VERSION=1 -DWINDOWS_KERNEL_LIB=kernel32 -D_CRT_SECURE_NO_WARNINGS @CMakeFiles/aws-c-common.dir/includes_C.rsp -O3 -DNDEBUG -O2 -ftree-vectorize -std=gnu99 -fvisibility=hidden -Wall -Wstrict-prototypes -pedantic -Wno-long-long -fPIC -D_FILE_OFFSET_BITS=64 -MD -MT _deps/aws-c-common-build/CMakeFiles/aws-c-common.dir/source/allocator.c.obj -MF CMakeFiles/aws-c-common.dir/source/allocator.c.obj.d -o CMakeFiles/aws-c-common.dir/source/allocator.c.obj -c /D/a/crossbow/crossbow/src/build-i686-cpp/_deps/aws-c-common-src/source/allocator.c + In file included from D:/a/crossbow/crossbow/src/build-i686-cpp/_deps/aws-c-common-src/include/aws/common/byte_order.h:71, + from D:/a/crossbow/crossbow/src/build-i686-cpp/_deps/aws-c-common-src/include/aws/common/byte_buf.h:9, + from D:/a/crossbow/crossbow/src/build-i686-cpp/_deps/aws-c-common-src/include/aws/common/thread.h:8, + from D:/a/crossbow/crossbow/src/build-i686-cpp/_deps/aws-c-common-src/include/aws/common/logging.h:11, + from D:/a/crossbow/crossbow/src/build-i686-cpp/_deps/aws-c-common-src/source/allocator.c:8: + D:/a/crossbow/crossbow/src/build-i686-cpp/_deps/aws-c-common-src/include/aws/common/byte_order.inl: In function 'aws_hton64': + D:/a/crossbow/crossbow/src/build-i686-cpp/_deps/aws-c-common-src/include/aws/common/byte_order.inl:47:23: warning: implicit declaration of function 'htonl'; did you mean 'wtoll'? [-Wimplicit-function-declaration] + return ((uint64_t)htonl(low)) << 32 | htonl(high); + ^~~~~ + wtoll + In file included from C:/rtools40/mingw32/i686-w64-mingw32/include/windows.h:92, + from D:/a/crossbow/crossbow/src/build-i686-cpp/_deps/aws-c-common-src/source/allocator.c:15: + C:/rtools40/mingw32/i686-w64-mingw32/include/winsock.h: At top level: + C:/rtools40/mingw32/i686-w64-mingw32/include/winsock.h:286:37: error: conflicting types for 'htonl' + WINSOCK_API_LINKAGE u_long WSAAPI htonl(u_long hostlong); + ^~~~~ + In file included from D:/a/crossbow/crossbow/src/build-i686-cpp/_deps/aws-c-common-src/include/aws/common/byte_order.h:71, + from D:/a/crossbow/crossbow/src/build-i686-cpp/_deps/aws-c-common-src/include/aws/common/byte_buf.h:9, + from D:/a/crossbow/crossbow/src/build-i686-cpp/_deps/aws-c-common-src/include/aws/common/thread.h:8, + from D:/a/crossbow/crossbow/src/build-i686-cpp/_deps/aws-c-common-src/include/aws/common/logging.h:11, + from D:/a/crossbow/crossbow/src/build-i686-cpp/_deps/aws-c-common-src/source/allocator.c:8: + D:/a/crossbow/crossbow/src/build-i686-cpp/_deps/aws-c-common-src/include/aws/common/byte_order.inl:47:23: note: previous implicit declaration of 'htonl' was here + return ((uint64_t)htonl(low)) << 32 | htonl(high); + ^~~~~ +--- + include/aws/common/byte_order.inl | 4 ++-- + 1 file changed, 2 insertions(+), 2 deletions(-) + +diff --git a/include/aws/common/byte_order.inl b/include/aws/common/byte_order.inl +index 1204be06a1..0abd9cb8cd 100644 +--- a/include/aws/common/byte_order.inl ++++ b/include/aws/common/byte_order.inl +@@ -13,7 +13,7 @@ + # include + #else + # include +-#endif /* _MSC_VER */ ++#endif /* _WIN32 */ + + AWS_EXTERN_C_BEGIN + +@@ -39,7 +39,7 @@ AWS_STATIC_IMPL uint64_t aws_hton64(uint64_t x) { + uint64_t v; + __asm__("bswap %q0" : "=r"(v) : "0"(x)); + return v; +-#elif defined(_MSC_VER) ++#elif defined(_WIN32) + return _byteswap_uint64(x); + #else + uint32_t low = x & UINT32_MAX; From 1a40333f3d0b9d4ac6201ec95d7e7d5b6bf73c9f Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Tue, 10 Jun 2025 06:45:12 +0900 Subject: [PATCH 34/52] Remove needless include --- cpp/cmake_modules/ThirdpartyToolchain.cmake | 2 -- 1 file changed, 2 deletions(-) diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index 252c6690412..20d4b22cea8 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -5082,8 +5082,6 @@ endif() # ---------------------------------------------------------------------- # AWS SDK for C++ -include(AWSSDKVariables) - function(build_awssdk) message(STATUS "Building AWS SDK for C++ from source") From 4c3e94bf478d207c47347d4ee37d1ca7280919fa Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Tue, 10 Jun 2025 10:45:36 +0900 Subject: [PATCH 35/52] Add _WIN32_WINNT=0x0601 --- cpp/cmake_modules/ThirdpartyToolchain.cmake | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index 20d4b22cea8..f5864523409 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -5160,6 +5160,15 @@ function(build_awssdk) ON CACHE BOOL "" FORCE) + # For aws-c-common + if(MINGW) + # PPROCESSOR_NUMBER requires Windows 7 or later. + # + # 0x0601 == _WIN32_WINNT_WIN7 + string(APPEND CMAKE_C_FLAGS " -D_WIN32_WINNT=0x0601") + string(APPEND CMAKE_CXX_FLAGS " -D_WIN32_WINNT=0x0601") + endif() + # For aws-lc set(DISABLE_PERL ON From d9ec5bf615a350b909afc55b51801c5ae95a863d Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Tue, 10 Jun 2025 11:20:34 +0900 Subject: [PATCH 36/52] Add workaround for aws-c-io and RTools 40 --- cpp/cmake_modules/ThirdpartyToolchain.cmake | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index f5864523409..dbc7a071659 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -5194,6 +5194,23 @@ function(build_awssdk) string(REPLACE "-flto=auto" "" CMAKE_C_FLAGS "${CMAKE_C_FLAGS}") string(REPLACE "-ffat-lto-objects" "" CMAKE_C_FLAGS "${CMAKE_C_FLAGS}") + # For aws-c-io + if(MINGW AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS "9") + # This is for RTools 40. We can remove this after we dropped + # support for R < 4.2. schannel.h in RTools 40 is old. + + # See also: https://learn.microsoft.com/en-us/windows/win32/api/schannel/ns-schannel-schannel_cred + string(APPEND CMAKE_C_FLAGS " -DSP_PROT_TLS1_0_SERVER=0x00000040") + string(APPEND CMAKE_C_FLAGS " -DSP_PROT_TLS1_0_CLIENT=0x00000080") + string(APPEND CMAKE_C_FLAGS " -DSP_PROT_TLS1_1_SERVER=0x00000100") + string(APPEND CMAKE_C_FLAGS " -DSP_PROT_TLS1_1_CLIENT=0x00000200") + string(APPEND CMAKE_C_FLAGS " -DSP_PROT_TLS1_2_SERVER=0x00000400") + string(APPEND CMAKE_C_FLAGS " -DSP_PROT_TLS1_2_CLIENT=0x00000800") + string(APPEND CMAKE_C_FLAGS " -DSP_PROT_TLS1_3_SERVER=0x00001000") + string(APPEND CMAKE_C_FLAGS " -DSP_PROT_TLS1_3_CLIENT=0x00002000") + string(APPEND CMAKE_C_FLAGS " -DSCH_USE_STRONG_CRYPTO=0x00400000") + endif() + set(AWSSDK_LINK_LIBRARIES) foreach(AWSSDK_PRODUCT ${AWSSDK_PRODUCTS}) if("${AWSSDK_PRODUCT}" STREQUAL "s2n-tls") From 9d629b58a79edc6ed1969f4c65acadf46bb2908b Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Tue, 10 Jun 2025 11:54:55 +0900 Subject: [PATCH 37/52] Add more definitions --- cpp/cmake_modules/ThirdpartyToolchain.cmake | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index dbc7a071659..8b23b03a290 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -5199,7 +5199,10 @@ function(build_awssdk) # This is for RTools 40. We can remove this after we dropped # support for R < 4.2. schannel.h in RTools 40 is old. - # See also: https://learn.microsoft.com/en-us/windows/win32/api/schannel/ns-schannel-schannel_cred + # For schannel.h. + # + # See also: + # https://learn.microsoft.com/en-us/windows/win32/api/schannel/ns-schannel-schannel_cred string(APPEND CMAKE_C_FLAGS " -DSP_PROT_TLS1_0_SERVER=0x00000040") string(APPEND CMAKE_C_FLAGS " -DSP_PROT_TLS1_0_CLIENT=0x00000080") string(APPEND CMAKE_C_FLAGS " -DSP_PROT_TLS1_1_SERVER=0x00000100") @@ -5209,6 +5212,15 @@ function(build_awssdk) string(APPEND CMAKE_C_FLAGS " -DSP_PROT_TLS1_3_SERVER=0x00001000") string(APPEND CMAKE_C_FLAGS " -DSP_PROT_TLS1_3_CLIENT=0x00002000") string(APPEND CMAKE_C_FLAGS " -DSCH_USE_STRONG_CRYPTO=0x00400000") + + # For sspi.h. + # + # See also: + # https://learn.microsoft.com/en-us/windows/win32/api/sspi/ne-sspi-sec_application_protocol_negotiation_ext + string(APPEND CMAKE_C_FLAGS " -DSecApplicationProtocolNegotiationExt_ALPN=2") + # See also: + # https://learn.microsoft.com/en-us/windows/win32/api/sspi/ns-sspi-secbuffer + string(APPEND CMAKE_C_FLAGS " -DSECBUFFER_ALERT=17") endif() set(AWSSDK_LINK_LIBRARIES) From 3b9dd7939fbe148b66b7b11acce5af5aa890c4f5 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Tue, 10 Jun 2025 13:57:29 +0900 Subject: [PATCH 38/52] Don't use -Werror --- cpp/cmake_modules/ThirdpartyToolchain.cmake | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index 8b23b03a290..5329cc0ada5 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -5199,7 +5199,7 @@ function(build_awssdk) # This is for RTools 40. We can remove this after we dropped # support for R < 4.2. schannel.h in RTools 40 is old. - # For schannel.h. + # For schannel.h # # See also: # https://learn.microsoft.com/en-us/windows/win32/api/schannel/ns-schannel-schannel_cred @@ -5213,7 +5213,7 @@ function(build_awssdk) string(APPEND CMAKE_C_FLAGS " -DSP_PROT_TLS1_3_CLIENT=0x00002000") string(APPEND CMAKE_C_FLAGS " -DSCH_USE_STRONG_CRYPTO=0x00400000") - # For sspi.h. + # For sspi.h # # See also: # https://learn.microsoft.com/en-us/windows/win32/api/sspi/ne-sspi-sec_application_protocol_negotiation_ext @@ -5223,6 +5223,11 @@ function(build_awssdk) string(APPEND CMAKE_C_FLAGS " -DSECBUFFER_ALERT=17") endif() + # For aws-sdk-cpp + set(AWS_SDK_WARNINGS_ARE_ERRORS + OFF + CACHE BOOL "" FORCE) + set(AWSSDK_LINK_LIBRARIES) foreach(AWSSDK_PRODUCT ${AWSSDK_PRODUCTS}) if("${AWSSDK_PRODUCT}" STREQUAL "s2n-tls") From f2ec58c7498b876f4d6be5ba371005874e239c96 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Tue, 10 Jun 2025 14:54:21 +0900 Subject: [PATCH 39/52] Add missing macros --- cpp/cmake_modules/ThirdpartyToolchain.cmake | 23 +++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index 5329cc0ada5..06523d2451b 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -5227,6 +5227,29 @@ function(build_awssdk) set(AWS_SDK_WARNINGS_ARE_ERRORS OFF CACHE BOOL "" FORCE) + if(MINGW AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS "9") + # This is for RTools 40. We can remove this after we dropped + # support for R < 4.2. schannel.h in RTools 40 is old. + + # For winhttp.h + # + # See also: + # https://learn.microsoft.com/en-us/windows/win32/winhttp/error-messages + string(APPEND CMAKE_CXX_FLAGS " -DERROR_WINHTTP_UNHANDLED_SCRIPT_TYPE=12176") + string(APPEND CMAKE_CXX_FLAGS " -DERROR_WINHTTP_SCRIPT_EXECUTION_ERROR=12177") + # See also: + # https://learn.microsoft.com/en-us/windows/win32/api/winhttp/ns-winhttp-winhttp_async_result + string(APPEND CMAKE_CXX_FLAGS " -DAPI_GET_PROXY_FOR_URL=6") + # See also: + # https://learn.microsoft.com/en-us/windows/win32/api/winhttp/nc-winhttp-winhttp_status_callback + string(APPEND CMAKE_CXX_FLAGS " -DWINHTTP_CALLBACK_STATUS_CLOSE_COMPLETE=0x02000000") + string(APPEND CMAKE_CXX_FLAGS + " -DWINHTTP_CALLBACK_STATUS_SHUTDOWN_COMPLETE=0x04000000") + # See also: + # https://learn.microsoft.com/en-us/windows/win32/winhttp/option-flags + string(APPEND CMAKE_CXX_FLAGS " -DWINHTTP_FLAG_SECURE_PROTOCOL_TLS1_2=0x00000800") + string(APPEND CMAKE_CXX_FLAGS " -DWINHTTP_NO_CLIENT_CERT_CONTEXT=0") + endif() set(AWSSDK_LINK_LIBRARIES) foreach(AWSSDK_PRODUCT ${AWSSDK_PRODUCTS}) From ec8fb869f01e0d398e8d73b438dc2504dd7bbf25 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Tue, 10 Jun 2025 15:02:38 +0900 Subject: [PATCH 40/52] Add license header --- cpp/cmake_modules/aws-c-common-1208.patch | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/cpp/cmake_modules/aws-c-common-1208.patch b/cpp/cmake_modules/aws-c-common-1208.patch index 644f03ea492..72a5f70776b 100644 --- a/cpp/cmake_modules/aws-c-common-1208.patch +++ b/cpp/cmake_modules/aws-c-common-1208.patch @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + From 06c435c6765833734c62ac45909b5ab1fc5276a7 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Sat, 7 Jun 2025 20:40:36 +0900 From e4585b0cc7dafae795f2380895c815952da2d228 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Wed, 11 Jun 2025 09:21:11 +0900 Subject: [PATCH 41/52] Remove needless patches --- ci/rtools/README.md | 35 ------ ci/rtools/aws_c_common_ep.patch | 39 ------- ci/rtools/aws_c_io_ep.patch | 56 ---------- ci/rtools/awssdk_ep.patch | 181 -------------------------------- 4 files changed, 311 deletions(-) delete mode 100644 ci/rtools/README.md delete mode 100644 ci/rtools/aws_c_common_ep.patch delete mode 100644 ci/rtools/aws_c_io_ep.patch delete mode 100644 ci/rtools/awssdk_ep.patch diff --git a/ci/rtools/README.md b/ci/rtools/README.md deleted file mode 100644 index 08b5ea7f513..00000000000 --- a/ci/rtools/README.md +++ /dev/null @@ -1,35 +0,0 @@ - - -# rtools40 patches for AWS SDK and related libs - -The patches in this directory are solely for the purpose of building Arrow C++ -under [Rtools40](https://cran.r-project.org/bin/windows/Rtools/rtools40.html) -and not used elsewhere. Once we've dropped support for Rtools40, we can consider -removing these patches. - -The larger reason these patches are needed is that Rtools provides their own -packages and their versions of the AWS libraries weren't compatible with CMake -3.25. Our solution was to bundle the AWS libs instead and these patches were -required to get them building under the Rtools40 environment. - -The patches were added while upgrading the minimum required CMake version to -3.25 in [GH-44950](https://github.com/apache/arrow/issues/44950). Please see the -associated PR, [GH-44989](https://github.com/apache/arrow/pull/44989), for more -context. diff --git a/ci/rtools/aws_c_common_ep.patch b/ci/rtools/aws_c_common_ep.patch deleted file mode 100644 index 94c84d0fe1b..00000000000 --- a/ci/rtools/aws_c_common_ep.patch +++ /dev/null @@ -1,39 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -diff --git a/include/aws/common/byte_order.inl b/include/aws/common/byte_order.inl -index 1204be0..0abd9cb 100644 ---- a/include/aws/common/byte_order.inl -+++ b/include/aws/common/byte_order.inl -@@ -13,7 +13,7 @@ - # include - #else - # include --#endif /* _MSC_VER */ -+#endif /* _WIN32 */ - - AWS_EXTERN_C_BEGIN - -@@ -39,7 +39,7 @@ AWS_STATIC_IMPL uint64_t aws_hton64(uint64_t x) { - uint64_t v; - __asm__("bswap %q0" : "=r"(v) : "0"(x)); - return v; --#elif defined(_MSC_VER) -+#elif defined(_WIN32) - return _byteswap_uint64(x); - #else - uint32_t low = x & UINT32_MAX; diff --git a/ci/rtools/aws_c_io_ep.patch b/ci/rtools/aws_c_io_ep.patch deleted file mode 100644 index a15d706ba12..00000000000 --- a/ci/rtools/aws_c_io_ep.patch +++ /dev/null @@ -1,56 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -diff --git a/source/windows/secure_channel_tls_handler.c b/source/windows/secure_channel_tls_handler.c -index 50caf02..29fe850 100644 ---- a/source/windows/secure_channel_tls_handler.c -+++ b/source/windows/secure_channel_tls_handler.c -@@ -35,6 +36,25 @@ - # pragma warning(disable : 4306) /* Identifier is type cast to a larger pointer. */ - #endif - -+#ifndef SP_PROT_TLS1_0_SERVER -+#define SP_PROT_TLS1_0_SERVER SP_PROT_TLS1_SERVER -+#endif -+#ifndef SP_PROT_TLS1_0_CLIENT -+#define SP_PROT_TLS1_0_CLIENT SP_PROT_TLS1_CLIENT -+#endif -+#ifndef SP_PROT_TLS1_1_SERVER -+#define SP_PROT_TLS1_1_SERVER 0x00000100 -+#endif -+#ifndef SP_PROT_TLS1_1_CLIENT -+#define SP_PROT_TLS1_1_CLIENT 0x00000200 -+#endif -+#ifndef SCH_USE_STRONG_CRYPTO -+#define SCH_USE_STRONG_CRYPTO 0x00400000 -+#endif -+#ifndef SECBUFFER_ALERT -+#define SECBUFFER_ALERT 0x11 -+#endif -+ - #define KB_1 1024 - #define READ_OUT_SIZE (16 * KB_1) - #define READ_IN_SIZE READ_OUT_SIZE -@@ -456,7 +476,7 @@ static int s_fillin_alpn_data( - - *extension_length += sizeof(uint32_t) + sizeof(uint16_t); - -- *extension_name = SecApplicationProtocolNegotiationExt_ALPN; -+ *extension_name = 2; - /*now add the protocols*/ - for (size_t i = 0; i < protocols_count; ++i) { - struct aws_byte_cursor *protocol_ptr = NULL; diff --git a/ci/rtools/awssdk_ep.patch b/ci/rtools/awssdk_ep.patch deleted file mode 100644 index bd26f853290..00000000000 --- a/ci/rtools/awssdk_ep.patch +++ /dev/null @@ -1,181 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -diff --git a/aws-cpp-sdk-core/include/aws/core/utils/Array.h b/aws-cpp-sdk-core/include/aws/core/utils/Array.h -index 2b5bbc566..7cb93bdf0 100644 ---- a/aws-cpp-sdk-core/include/aws/core/utils/Array.h -+++ b/aws-cpp-sdk-core/include/aws/core/utils/Array.h -@@ -54,7 +54,7 @@ namespace Aws - { - m_data.reset(Aws::NewArray(m_size, ARRAY_ALLOCATION_TAG)); - --#ifdef _WIN32 -+#ifdef _MSC_VER - std::copy(arrayToCopy, arrayToCopy + arraySize, stdext::checked_array_iterator< T * >(m_data.get(), m_size)); - #else - std::copy(arrayToCopy, arrayToCopy + arraySize, m_data.get()); -@@ -82,7 +82,7 @@ namespace Aws - if(arr->m_size > 0 && arr->m_data) - { - size_t arraySize = arr->m_size; --#ifdef _WIN32 -+#ifdef _MSC_VER - std::copy(arr->m_data.get(), arr->m_data.get() + arraySize, stdext::checked_array_iterator< T * >(m_data.get() + location, m_size)); - #else - std::copy(arr->m_data.get(), arr->m_data.get() + arraySize, m_data.get() + location); -@@ -101,7 +101,7 @@ namespace Aws - { - m_data.reset(Aws::NewArray(m_size, ARRAY_ALLOCATION_TAG)); - --#ifdef _WIN32 -+#ifdef _MSC_VER - std::copy(other.m_data.get(), other.m_data.get() + other.m_size, stdext::checked_array_iterator< T * >(m_data.get(), m_size)); - #else - std::copy(other.m_data.get(), other.m_data.get() + other.m_size, m_data.get()); -@@ -134,7 +134,7 @@ namespace Aws - { - m_data.reset(Aws::NewArray(m_size, ARRAY_ALLOCATION_TAG)); - --#ifdef _WIN32 -+#ifdef _MSC_VER - std::copy(other.m_data.get(), other.m_data.get() + other.m_size, stdext::checked_array_iterator< T * >(m_data.get(), m_size)); - #else - std::copy(other.m_data.get(), other.m_data.get() + other.m_size, m_data.get()); -diff --git a/aws-cpp-sdk-core/source/http/windows/WinHttpSyncHttpClient.cpp b/aws-cpp-sdk-core/source/http/windows/WinHttpSyncHttpClient.cpp -index 4dade6489..a0456cf8e 100644 ---- a/aws-cpp-sdk-core/source/http/windows/WinHttpSyncHttpClient.cpp -+++ b/aws-cpp-sdk-core/source/http/windows/WinHttpSyncHttpClient.cpp -@@ -22,6 +22,16 @@ - #include - #include - -+#ifndef WINHTTP_OPTION_WEB_SOCKET_KEEPALIVE_INTERVAL -+#define WINHTTP_OPTION_WEB_SOCKET_KEEPALIVE_INTERVAL 116 -+#endif -+#ifndef WINHTTP_FLAG_SECURE_PROTOCOL_TLS1_1 -+#define WINHTTP_FLAG_SECURE_PROTOCOL_TLS1_1 0x00000200 -+#endif -+#ifndef WINHTTP_FLAG_SECURE_PROTOCOL_TLS1_2 -+#define WINHTTP_FLAG_SECURE_PROTOCOL_TLS1_2 0x00000800 -+#endif -+ - using namespace Aws::Client; - using namespace Aws::Http; - using namespace Aws::Http::Standard; -@@ -272,7 +282,7 @@ bool WinHttpSyncHttpClient::DoQueryHeaders(void* hHttpRequest, std::shared_ptr(dwSize / sizeof(wchar_t))); - - WinHttpQueryHeaders(hHttpRequest, WINHTTP_QUERY_CONTENT_TYPE, nullptr, &contentTypeStr, &dwSize, 0); -- if (contentTypeStr[0] != NULL) -+ if (contentTypeStr[0]) - { - Aws::String contentStr = StringUtils::FromWString(contentTypeStr); - response->SetContentType(contentStr); -@@ -303,7 +313,7 @@ bool WinHttpSyncHttpClient::DoQueryHeaders(void* hHttpRequest, std::shared_ptrSetContentType(contentTypeStr); - AWS_LOGSTREAM_DEBUG(GetLogTag(), "Received content type " << contentTypeStr); -diff --git a/aws-cpp-sdk-core/source/http/windows/WinSyncHttpClient.cpp b/aws-cpp-sdk-core/source/http/windows/WinSyncHttpClient.cpp -index d7513cc3c..e390a8d4e 100644 ---- a/aws-cpp-sdk-core/source/http/windows/WinSyncHttpClient.cpp -+++ b/aws-cpp-sdk-core/source/http/windows/WinSyncHttpClient.cpp -@@ -349,7 +349,7 @@ std::shared_ptr WinSyncHttpClient::MakeRequest(const std::shared_p - } - } - -- if (!success && !IsRequestProcessingEnabled() || !ContinueRequest(*request)) -+ if ((!success && !IsRequestProcessingEnabled()) || !ContinueRequest(*request)) - { - response->SetClientErrorType(CoreErrors::USER_CANCELLED); - response->SetClientErrorMessage("Request processing disabled or continuation cancelled by user's continuation handler."); -diff --git a/aws-cpp-sdk-core/source/platform/windows/FileSystem.cpp b/aws-cpp-sdk-core/source/platform/windows/FileSystem.cpp -index 2ea82de6f..bc423441e 100644 ---- a/aws-cpp-sdk-core/source/platform/windows/FileSystem.cpp -+++ b/aws-cpp-sdk-core/source/platform/windows/FileSystem.cpp -@@ -11,7 +11,9 @@ - #include - #include - -+#ifdef _MSC_VER - #pragma warning( disable : 4996) -+#endif - - using namespace Aws::Utils; - namespace Aws -@@ -304,6 +306,9 @@ Aws::String CreateTempFilePath() - { - #ifdef _MSC_VER - #pragma warning(disable: 4996) // _CRT_SECURE_NO_WARNINGS -+#elif !defined(L_tmpnam_s) -+ // Definition from the MSVC stdio.h -+ #define L_tmpnam_s (sizeof("\\") + 16) - #endif - char s_tempName[L_tmpnam_s+1]; - -diff --git a/aws-cpp-sdk-core/source/platform/windows/OSVersionInfo.cpp b/aws-cpp-sdk-core/source/platform/windows/OSVersionInfo.cpp -index 0180f7fbf..3adbab313 100644 ---- a/aws-cpp-sdk-core/source/platform/windows/OSVersionInfo.cpp -+++ b/aws-cpp-sdk-core/source/platform/windows/OSVersionInfo.cpp -@@ -9,7 +9,9 @@ - - #include - -+#ifdef _MSC_VER - #pragma warning(disable: 4996) -+#endif - #include - #include - namespace Aws -diff --git a/aws-cpp-sdk-core/source/utils/crypto/factory/Factories.cpp b/aws-cpp-sdk-core/source/utils/crypto/factory/Factories.cpp -index 2ee517b48..3b0dce665 100644 ---- a/aws-cpp-sdk-core/source/utils/crypto/factory/Factories.cpp -+++ b/aws-cpp-sdk-core/source/utils/crypto/factory/Factories.cpp -@@ -939,7 +939,7 @@ std::shared_ptr Aws::Utils::Crypto::CreateSha256HMACIm - return GetSha256HMACFactory()->CreateImplementation(); - } - --#ifdef _WIN32 -+#ifdef _MSC_VER - #pragma warning( push ) - #pragma warning( disable : 4702 ) - #endif -@@ -1032,7 +1032,7 @@ std::shared_ptr Aws::Utils::Crypto::CreateAES_KeyWrapImplementa - return GetAES_KeyWrapFactory()->CreateImplementation(key); - } - --#ifdef _WIN32 -+#ifdef _MSC_VER - #pragma warning(pop) - #endif From d8192df05be98411c2c1f62ec3d1f5fadad379be Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Wed, 11 Jun 2025 09:21:19 +0900 Subject: [PATCH 42/52] Use bundled zlib --- cpp/cmake_modules/ThirdpartyToolchain.cmake | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index 06523d2451b..602710ba9ce 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -5170,20 +5170,20 @@ function(build_awssdk) endif() # For aws-lc - set(DISABLE_PERL + set(DISABLE_GO ON CACHE BOOL "" FORCE) - set(DISABLE_GO + set(DISABLE_PERL ON CACHE BOOL "" FORCE) # For s2n-tls - set(crypto_STATIC_LIBRARY - "$" - CACHE STRING "" FORCE) set(crypto_INCLUDE_DIR "$" CACHE STRING "" FORCE) + set(crypto_STATIC_LIBRARY + "$" + CACHE STRING "" FORCE) set(S2N_INTERN_LIBCRYPTO ON CACHE BOOL "" FORCE) @@ -5227,6 +5227,12 @@ function(build_awssdk) set(AWS_SDK_WARNINGS_ARE_ERRORS OFF CACHE BOOL "" FORCE) + set(ZLIB_INCLUDE_DIR + "$" + CACHE STRING "" FORCE) + set(ZLIB_LIBRARY + "$" + CACHE STRING "" FORCE) if(MINGW AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS "9") # This is for RTools 40. We can remove this after we dropped # support for R < 4.2. schannel.h in RTools 40 is old. From 777f8812195dfb086717145a0ff4202cb0d3ddf3 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Wed, 11 Jun 2025 16:20:52 +0900 Subject: [PATCH 43/52] Add a comment why we need to use CACHE variables --- cpp/cmake_modules/ThirdpartyToolchain.cmake | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index 602710ba9ce..313e0861f01 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -1021,6 +1021,14 @@ if(CMAKE_VERSION VERSION_GREATER_EQUAL 3.25) endif() macro(prepare_fetchcontent) + # We need to use CACHE variables here because there are dependencies + # that don't use CMP0077 NEW policy. If CMP0077 NEW policy isn't + # used, option() ignores normal (not CACHE) variables. We can use + # normal (not CACHE) variables here when all dependencies use + # CMP0077 NEW policy. + # + # See also: + # https://cmake.org/cmake/help/latest/policy/CMP0077.html set(BUILD_SHARED_LIBS OFF CACHE BOOL "" FORCE) From 022543f3026c00d3b79b91f4dabd9f8499b772a7 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Thu, 12 Jun 2025 05:28:47 +0900 Subject: [PATCH 44/52] Don't change CMAKE_BUILD_TYPE It's referred from aws-sdk-cpp. --- cpp/cmake_modules/SetupCxxFlags.cmake | 8 +++----- cpp/cmake_modules/ThirdpartyToolchain.cmake | 2 +- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/cpp/cmake_modules/SetupCxxFlags.cmake b/cpp/cmake_modules/SetupCxxFlags.cmake index fdb28b540e2..9ecca91ecd5 100644 --- a/cpp/cmake_modules/SetupCxxFlags.cmake +++ b/cpp/cmake_modules/SetupCxxFlags.cmake @@ -152,8 +152,6 @@ set(CMAKE_CXX_EXTENSIONS OFF) # shared libraries set(CMAKE_POSITION_INDEPENDENT_CODE ${ARROW_POSITION_INDEPENDENT_CODE}) -string(TOUPPER ${CMAKE_BUILD_TYPE} CMAKE_BUILD_TYPE) - set(UNKNOWN_COMPILER_MESSAGE "Unknown compiler: ${CMAKE_CXX_COMPILER_ID} ${CMAKE_CXX_COMPILER_VERSION}") @@ -265,7 +263,7 @@ endif() # `RELEASE`, then it will default to `PRODUCTION`. The goal of defaulting to # `CHECKIN` is to avoid friction with long response time from CI. if(NOT BUILD_WARNING_LEVEL) - if("${CMAKE_BUILD_TYPE}" STREQUAL "RELEASE") + if("${UPPERCASE_BUILD_TYPE}" STREQUAL "RELEASE") set(BUILD_WARNING_LEVEL PRODUCTION) else() set(BUILD_WARNING_LEVEL CHECKIN) @@ -602,7 +600,7 @@ if(NOT WIN32 AND NOT APPLE) if(MUST_USE_GOLD) message(STATUS "Using hard-wired gold linker (version ${GOLD_VERSION})") if(ARROW_BUGGY_GOLD) - if("${ARROW_LINK}" STREQUAL "d" AND "${CMAKE_BUILD_TYPE}" STREQUAL "RELEASE") + if("${ARROW_LINK}" STREQUAL "d" AND "${UPPERCASE_BUILD_TYPE}" STREQUAL "RELEASE") message(SEND_ERROR "Configured to use buggy gold with dynamic linking " "in a RELEASE build") endif() @@ -808,7 +806,7 @@ if(CMAKE_SYSTEM_NAME STREQUAL "Emscripten") set(CMAKE_SHARED_LINKER_FLAGS "-sSIDE_MODULE=1 ${ARROW_EMSCRIPTEN_LINKER_FLAGS}") if(ARROW_TESTING) # flags for building test executables for use in node - if("${CMAKE_BUILD_TYPE}" STREQUAL "RELEASE") + if("${UPPERCASE_BUILD_TYPE}" STREQUAL "RELEASE") set(CMAKE_EXE_LINKER_FLAGS "${ARROW_EMSCRIPTEN_LINKER_FLAGS} -sALLOW_MEMORY_GROWTH -lnodefs.js -lnoderawfs.js --pre-js ${BUILD_SUPPORT_DIR}/emscripten-test-init.js" ) diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index 313e0861f01..9a4314e4892 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -1093,7 +1093,7 @@ macro(build_boost) if(ARROW_BOOST_REQUIRE_LIBRARY) set(BOOST_LIB_DIR "${BOOST_PREFIX}/stage/lib") set(BOOST_BUILD_LINK "static") - if("${CMAKE_BUILD_TYPE}" STREQUAL "DEBUG") + if("${UPPERCASE_BUILD_TYPE}" STREQUAL "DEBUG") set(BOOST_BUILD_VARIANT "debug") else() set(BOOST_BUILD_VARIANT "release") From 0217dc02e0e5a56528264fbfa7c3291dc2b2ad9c Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Thu, 12 Jun 2025 05:29:39 +0900 Subject: [PATCH 45/52] Don't use cache variable as much as possible --- cpp/cmake_modules/ThirdpartyToolchain.cmake | 127 +++++++------------- 1 file changed, 45 insertions(+), 82 deletions(-) diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index 9a4314e4892..d649422bbe1 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -1021,47 +1021,19 @@ if(CMAKE_VERSION VERSION_GREATER_EQUAL 3.25) endif() macro(prepare_fetchcontent) - # We need to use CACHE variables here because there are dependencies - # that don't use CMP0077 NEW policy. If CMP0077 NEW policy isn't - # used, option() ignores normal (not CACHE) variables. We can use - # normal (not CACHE) variables here when all dependencies use - # CMP0077 NEW policy. - # - # See also: - # https://cmake.org/cmake/help/latest/policy/CMP0077.html - set(BUILD_SHARED_LIBS - OFF - CACHE BOOL "" FORCE) - set(BUILD_STATIC_LIBS - ON - CACHE BOOL "" FORCE) - set(BUILD_TESTING - OFF - CACHE BOOL "" FORCE) - set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY - "" - CACHE PATH "" FORCE) - set(CMAKE_COMPILE_WARNING_AS_ERROR - FALSE - CACHE BOOL "" FORCE) - set(CMAKE_EXPORT_NO_PACKAGE_REGISTRY - TRUE - CACHE BOOL "" FORCE) - set(CMAKE_LIBRARY_OUTPUT_DIRECTORY - "" - CACHE PATH "" FORCE) - set(CMAKE_MACOSX_RPATH - ${ARROW_INSTALL_NAME_RPATH} - CACHE BOOL "" FORCE) + set(BUILD_SHARED_LIBS OFF) + set(BUILD_STATIC_LIBS ON) + set(BUILD_TESTING OFF) + set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY "") + set(CMAKE_COMPILE_WARNING_AS_ERROR OFF) + set(CMAKE_EXPORT_NO_PACKAGE_REGISTRY OFF) + set(CMAKE_LIBRARY_OUTPUT_DIRECTORY "") + set(CMAKE_MACOSX_RPATH ${ARROW_INSTALL_NAME_RPATH}) # We set CMAKE_POLICY_VERSION_MINIMUM temporarily due to failures with CMake 4 # We should remove it once we have updated the dependencies: # https://github.com/apache/arrow/issues/45985 - set(CMAKE_POLICY_VERSION_MINIMUM - 3.5 - CACHE STRING "" FORCE) - set(CMAKE_RUNTIME_OUTPUT_DIRECTORY - "" - CACHE PATH "" FORCE) + set(CMAKE_POLICY_VERSION_MINIMUM 3.5) + set(CMAKE_RUNTIME_OUTPUT_DIRECTORY "") if(MSVC) string(REPLACE "/WX" "" CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG}") @@ -5138,35 +5110,12 @@ function(build_awssdk) endforeach() prepare_fetchcontent() - set(BUILD_DEPS - OFF - CACHE BOOL "" FORCE) - set(BUILD_ONLY - "" - CACHE STRING "" FORCE) - list(APPEND - BUILD_ONLY - config - core - identity-management - s3 - sts - transfer) - set(BUILD_TOOL - OFF - CACHE BOOL "" FORCE) - set(ENABLE_TESTING - OFF - CACHE BOOL "" FORCE) - set(IN_SOURCE_BUILD - ON - CACHE BOOL "" FORCE) - set(MINIMIZE_SIZE - ON - CACHE BOOL "" FORCE) - set(USE_OPENSSL - ON - CACHE BOOL "" FORCE) + set(BUILD_DEPS OFF) + set(BUILD_TOOL OFF) + set(ENABLE_TESTING OFF) + set(IN_SOURCE_BUILD ON) + set(MINIMIZE_SIZE ON) + set(USE_OPENSSL ON) # For aws-c-common if(MINGW) @@ -5178,23 +5127,13 @@ function(build_awssdk) endif() # For aws-lc - set(DISABLE_GO - ON - CACHE BOOL "" FORCE) - set(DISABLE_PERL - ON - CACHE BOOL "" FORCE) + set(DISABLE_GO ON) + set(DISABLE_PERL ON) # For s2n-tls - set(crypto_INCLUDE_DIR - "$" - CACHE STRING "" FORCE) - set(crypto_STATIC_LIBRARY - "$" - CACHE STRING "" FORCE) - set(S2N_INTERN_LIBCRYPTO - ON - CACHE BOOL "" FORCE) + set(crypto_INCLUDE_DIR "$") + set(crypto_STATIC_LIBRARY "$") + set(S2N_INTERN_LIBCRYPTO ON) # For aws-lc and s2n-tls # @@ -5232,9 +5171,33 @@ function(build_awssdk) endif() # For aws-sdk-cpp + # + # We need to use CACHE variables because aws-sdk-cpp < 12.0.0 uses + # CMP0077 OLD policy. We can use normal variables when we use + # aws-sdk-cpp >= 12.0.0. set(AWS_SDK_WARNINGS_ARE_ERRORS OFF CACHE BOOL "" FORCE) + set(BUILD_DEPS + OFF + CACHE BOOL "" FORCE) + set(BUILD_ONLY + "" + CACHE STRING "" FORCE) + list(APPEND + BUILD_ONLY + config + core + identity-management + s3 + sts + transfer) + set(BUILD_SHARED_LIBS + OFF + CACHE BOOL "" FORCE) + set(ENABLE_TESTING + OFF + CACHE BOOL "" FORCE) set(ZLIB_INCLUDE_DIR "$" CACHE STRING "" FORCE) From 43d40b3f0926b12745d63a2dd13e919f3fa266ef Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Thu, 12 Jun 2025 05:44:10 +0900 Subject: [PATCH 46/52] Use the same curl --- cpp/cmake_modules/ThirdpartyToolchain.cmake | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index d649422bbe1..9874d09e3d4 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -5198,12 +5198,21 @@ function(build_awssdk) set(ENABLE_TESTING OFF CACHE BOOL "" FORCE) - set(ZLIB_INCLUDE_DIR - "$" - CACHE STRING "" FORCE) - set(ZLIB_LIBRARY - "$" - CACHE STRING "" FORCE) + if(NOT WIN32) + find_curl() + set(CURL_INCLUDE_DIR + "$" + CACHE STRING "" FORCE) + set(CURL_LIBRARY + "$" + CACHE STRING "" FORCE) + set(ZLIB_INCLUDE_DIR + "$" + CACHE STRING "" FORCE) + set(ZLIB_LIBRARY + "$" + CACHE STRING "" FORCE) + endif() if(MINGW AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS "9") # This is for RTools 40. We can remove this after we dropped # support for R < 4.2. schannel.h in RTools 40 is old. From 62209d46de221fa62c98de4cc9fa0ad1ae1b8e16 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Thu, 12 Jun 2025 13:54:04 +0900 Subject: [PATCH 47/52] Don't install aws-sdk-cpp --- cpp/cmake_modules/ThirdpartyToolchain.cmake | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index 9874d09e3d4..b2e98f0ee2e 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -5244,6 +5244,10 @@ function(build_awssdk) set(ADDITIONAL_FLAGS "-DCOMPILE_DEFINITIONS=-I${aws-lc_SOURCE_DIR}/include") endif() fetchcontent_makeavailable(${AWSSDK_PRODUCT}) + if(CMAKE_VERSION VERSION_LESS 3.28) + set_property(DIRECTORY ${${AWSSDK_PRODUCT}_SOURCE_DIR} PROPERTY EXCLUDE_FROM_ALL + TRUE) + endif() list(PREPEND CMAKE_MODULE_PATH "${${AWSSDK_PRODUCT}_SOURCE_DIR}/cmake") if(NOT "${AWSSDK_PRODUCT}" STREQUAL "aws-sdk-cpp") if("${AWSSDK_PRODUCT}" STREQUAL "aws-lc") From c1642d84e93db3f0538994bf26ae64eaa3744847 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Thu, 12 Jun 2025 14:25:24 +0900 Subject: [PATCH 48/52] Update versions --- cpp/thirdparty/versions.txt | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/cpp/thirdparty/versions.txt b/cpp/thirdparty/versions.txt index a1d5a3bd1bc..96ae7c6f035 100644 --- a/cpp/thirdparty/versions.txt +++ b/cpp/thirdparty/versions.txt @@ -27,22 +27,22 @@ ARROW_ABSL_BUILD_VERSION=20211102.0 ARROW_ABSL_BUILD_SHA256_CHECKSUM=dcf71b9cba8dc0ca9940c4b316a0c796be8fab42b070bb6b7cab62b48f0e66c4 ARROW_AWS_C_AUTH_BUILD_VERSION=v0.9.0 ARROW_AWS_C_AUTH_BUILD_SHA256_CHECKSUM=aa6e98864fefb95c249c100da4ae7aed36ba13a8a91415791ec6fad20bec0427 -ARROW_AWS_C_CAL_BUILD_VERSION=v0.9.1 -ARROW_AWS_C_CAL_BUILD_SHA256_CHECKSUM=1245f007e83a66805f7afe80ce4825f910dad0068028dd8efc3b6172e2679be5 +ARROW_AWS_C_CAL_BUILD_VERSION=v0.9.2 +ARROW_AWS_C_CAL_BUILD_SHA256_CHECKSUM=f9f3bc6a069e2efe25fcdf73e4d2b16b5608c327d2eb57c8f7a8524e9e1fcad0 ARROW_AWS_C_COMMON_BUILD_VERSION=v0.12.3 ARROW_AWS_C_COMMON_BUILD_SHA256_CHECKSUM=a4e7ac6c6f840cb6ab56b8ee0bcd94a61c59d68ca42570bca518432da4c94273 ARROW_AWS_C_COMPRESSION_BUILD_VERSION=v0.3.1 ARROW_AWS_C_COMPRESSION_BUILD_SHA256_CHECKSUM=d89fca17a37de762dc34f332d2da402343078da8dbd2224c46a11a88adddf754 ARROW_AWS_C_EVENT_STREAM_BUILD_VERSION=v0.5.4 ARROW_AWS_C_EVENT_STREAM_BUILD_SHA256_CHECKSUM=cef8b78e362836d15514110fb43a0a0c7a86b0a210d5fe25fd248a82027a7272 -ARROW_AWS_C_HTTP_BUILD_VERSION=v0.10.1 -ARROW_AWS_C_HTTP_BUILD_SHA256_CHECKSUM=1550f7bf9666bb8f86514db9e623f07249e3c53e868d2f36ff69b83bd3eadfec +ARROW_AWS_C_HTTP_BUILD_VERSION=v0.10.2 +ARROW_AWS_C_HTTP_BUILD_SHA256_CHECKSUM=048d9d683459ade363fd7cc448c2b6329c78f67a2a0c0cb61c16de4634a2fc6b ARROW_AWS_C_IO_BUILD_VERSION=v0.19.1 ARROW_AWS_C_IO_BUILD_SHA256_CHECKSUM=f2fea0c066924f7fe3c2b1c7b2fa9be640f5b16a6514854226330e63a1faacd0 ARROW_AWS_C_MQTT_BUILD_VERSION=v0.13.1 ARROW_AWS_C_MQTT_BUILD_SHA256_CHECKSUM=c54d02c1e46f55bae8d5e6f9c4b0d78d84c1c9d9ac16ba8d78c3361edcd8b5bb -ARROW_AWS_C_S3_BUILD_VERSION=v0.8.0 -ARROW_AWS_C_S3_BUILD_SHA256_CHECKSUM=0b2f2a6d3b17c6d0684b80cc6581dd1b99ced39bfbb633fc9a1b16bf3f8eaa66 +ARROW_AWS_C_S3_BUILD_VERSION=v0.8.1 +ARROW_AWS_C_S3_BUILD_SHA256_CHECKSUM=c8b09780691d2b94e50d101c68f01fa2d1c3debb0ff3aed313d93f0d3c9af663 ARROW_AWS_C_SDKUTILS_BUILD_VERSION=v0.2.4 ARROW_AWS_C_SDKUTILS_BUILD_SHA256_CHECKSUM=493cbed4fa57e0d4622fcff044e11305eb4fc12445f32c8861025597939175fc ARROW_AWS_CHECKSUMS_BUILD_VERSION=v0.2.7 @@ -51,8 +51,8 @@ ARROW_AWS_CRT_CPP_BUILD_VERSION=v0.32.8 ARROW_AWS_CRT_CPP_BUILD_SHA256_CHECKSUM=db44260452a0296341fb8e7b987e4c328f08f7829b9f1c740fed9c963e081e93 ARROW_AWS_LC_BUILD_VERSION=v1.52.1 ARROW_AWS_LC_BUILD_SHA256_CHECKSUM=fe552e3c3522f73afc3c30011745c431c633f7b4e25dcd7b38325f194a7b3b75 -ARROW_AWSSDK_BUILD_VERSION=1.11.581 -ARROW_AWSSDK_BUILD_SHA256_CHECKSUM=7c7bed16047f3092e7c1cf8de1e0e4d8b7ce008c68734d2cb7215f3c29609230 +ARROW_AWSSDK_BUILD_VERSION=1.11.587 +ARROW_AWSSDK_BUILD_SHA256_CHECKSUM=b9944ba9905a68d6e53abb4f36ab2b3bd18ac88d8571647bb9f2b8026b76f8cd # Despite the confusing version name this is still the whole Azure SDK for C++ including core, keyvault, storage-common, etc. ARROW_AZURE_SDK_BUILD_VERSION=azure-identity_1.9.0 ARROW_AZURE_SDK_BUILD_SHA256_CHECKSUM=97065bfc971ac8df450853ce805f820f52b59457bd7556510186a1569502e4a1 @@ -105,8 +105,8 @@ ARROW_SNAPPY_BUILD_VERSION=1.2.2 ARROW_SNAPPY_BUILD_SHA256_CHECKSUM=90f74bc1fbf78a6c56b3c4a082a05103b3a56bb17bca1a27e052ea11723292dc ARROW_SUBSTRAIT_BUILD_VERSION=v0.44.0 ARROW_SUBSTRAIT_BUILD_SHA256_CHECKSUM=f989a862f694e7dbb695925ddb7c4ce06aa6c51aca945105c075139aed7e55a2 -ARROW_S2N_TLS_BUILD_VERSION=v1.5.20 -ARROW_S2N_TLS_BUILD_SHA256_CHECKSUM=1d05303ba8383f62273c51b50147391c23375e918d525b7c827f7aeb69e6b102 +ARROW_S2N_TLS_BUILD_VERSION=v1.5.21 +ARROW_S2N_TLS_BUILD_SHA256_CHECKSUM=203d69d6f557f6ab303438ad186fca13fd2c60581b2cca6348a9fbee10d79995 ARROW_THRIFT_BUILD_VERSION=0.20.0 ARROW_THRIFT_BUILD_SHA256_CHECKSUM=b5d8311a779470e1502c027f428a1db542f5c051c8e1280ccd2163fa935ff2d6 ARROW_UTF8PROC_BUILD_VERSION=v2.10.0 From 7e77f8acc2c74c7b53aceec9b56dde846670d159 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Thu, 12 Jun 2025 14:25:28 +0900 Subject: [PATCH 49/52] Skip uwsgi test --- python/pyarrow/tests/test_fs.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/python/pyarrow/tests/test_fs.py b/python/pyarrow/tests/test_fs.py index 2838372bd17..e90613a6626 100644 --- a/python/pyarrow/tests/test_fs.py +++ b/python/pyarrow/tests/test_fs.py @@ -2043,6 +2043,8 @@ def test_concurrent_s3fs_init(): @pytest.mark.s3 +@pytest.mark.skip(reason="atexit(ensure_s3_finalized) will be called too late " + "with bundled aws-sdk-cpp 1.11.587") @pytest.mark.skipif(running_on_musllinux(), reason="Leaking S3ClientFinalizer causes " "segfault on musl based systems") def test_uwsgi_integration(): From 108ecb1937142a5d7655a2e351572922530130de Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Fri, 13 Jun 2025 13:30:48 +0900 Subject: [PATCH 50/52] Use function for LZ4 --- cpp/cmake_modules/ThirdpartyToolchain.cmake | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index b2e98f0ee2e..828ebb3fc1f 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -2664,11 +2664,13 @@ if(ARROW_WITH_ZLIB) resolve_dependency(ZLIB PC_PACKAGE_NAMES zlib) endif() -macro(build_lz4) +function(build_lz4) message(STATUS "Building LZ4 from source using FetchContent") # Set LZ4 as vendored - set(LZ4_VENDORED TRUE) + set(LZ4_VENDORED + TRUE + PARENT_SCOPE) # Declare the content fetchcontent_declare(lz4 @@ -2699,7 +2701,10 @@ macro(build_lz4) # Add to bundled static libs. # We must use lz4_static (not imported target) not LZ4::lz4 (imported target). list(APPEND ARROW_BUNDLED_STATIC_LIBS lz4_static) -endmacro() + set(ARROW_BUNDLED_STATIC_LIBS + ${ARROW_BUNDLED_STATIC_LIBS} + PARENT_SCOPE) +endfunction() if(ARROW_WITH_LZ4) resolve_dependency(lz4 @@ -4785,7 +4790,7 @@ function(build_orc) ${ARROW_PROTOBUF_PROTOC} ${ARROW_ZSTD_LIBZSTD} ${Snappy_TARGET} - LZ4::lz4 + ${ORC_LZ4_TARGET} ZLIB::ZLIB) add_library(orc::orc STATIC IMPORTED) set_target_properties(orc::orc PROPERTIES IMPORTED_LOCATION "${ORC_STATIC_LIB}") From bffd5799eb0362d1ff3a2608bbd432031a792a48 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Fri, 13 Jun 2025 13:30:55 +0900 Subject: [PATCH 51/52] Remove needless curl --- cpp/cmake_modules/ThirdpartyToolchain.cmake | 7 ------- 1 file changed, 7 deletions(-) diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index 828ebb3fc1f..587fec121f6 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -5204,13 +5204,6 @@ function(build_awssdk) OFF CACHE BOOL "" FORCE) if(NOT WIN32) - find_curl() - set(CURL_INCLUDE_DIR - "$" - CACHE STRING "" FORCE) - set(CURL_LIBRARY - "$" - CACHE STRING "" FORCE) set(ZLIB_INCLUDE_DIR "$" CACHE STRING "" FORCE) From 1007963d3d71291d7d8016b14826323a3f48590e Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Mon, 16 Jun 2025 16:59:09 +0900 Subject: [PATCH 52/52] Remove a needless version check --- cpp/cmake_modules/ThirdpartyToolchain.cmake | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index 587fec121f6..5264beed783 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -1012,13 +1012,10 @@ endif() set(MAKE_BUILD_ARGS "-j${NPROC}") include(FetchContent) -set(FC_DECLARE_COMMON_OPTIONS) +set(FC_DECLARE_COMMON_OPTIONS SYSTEM) if(CMAKE_VERSION VERSION_GREATER_EQUAL 3.28) list(APPEND FC_DECLARE_COMMON_OPTIONS EXCLUDE_FROM_ALL TRUE) endif() -if(CMAKE_VERSION VERSION_GREATER_EQUAL 3.25) - list(APPEND FC_DECLARE_COMMON_OPTIONS SYSTEM) -endif() macro(prepare_fetchcontent) set(BUILD_SHARED_LIBS OFF)