From 2503a03e94e7d6aeb9f5b3237c298e9d4edc7d5a Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Fri, 10 Jun 2022 16:45:46 +0900 Subject: [PATCH] ARROW-16804: [CI][Conan] Merge upstream changes We can automate this by ci/conan/merge_upstream.sh. --- ci/conan/all/conanfile.py | 24 +-- .../7.0.0-0004-remove-find-modules.patch | 149 ++++++++++++++++++ .../patches/7.0.0-0005-use-find-package.patch | 50 ++++-- ci/conan/merge_status.sh | 18 +++ ci/conan/merge_upstream.sh | 47 ++++++ 5 files changed, 263 insertions(+), 25 deletions(-) create mode 100644 ci/conan/merge_status.sh create mode 100755 ci/conan/merge_upstream.sh diff --git a/ci/conan/all/conanfile.py b/ci/conan/all/conanfile.py index 54fff62fae3..f34d6b69791 100644 --- a/ci/conan/all/conanfile.py +++ b/ci/conan/all/conanfile.py @@ -168,6 +168,8 @@ def validate(self): raise ConanInvalidConfiguration("CCI has no hiveserver2 recipe (yet)") if self.options.with_orc: raise ConanInvalidConfiguration("CCI has no orc recipe (yet)") + if self.options.with_s3 and not self.options["aws-sdk-cpp"].config: + raise ConanInvalidConfiguration("arrow:with_s3 requires aws-sdk-cpp:config is True.") if self.options.shared and self._with_jemalloc(): if self.options["jemalloc"].enable_cxx: @@ -259,7 +261,7 @@ def requirements(self): if self._with_thrift(): self.requires("thrift/0.15.0") if self._with_protobuf(): - self.requires("protobuf/3.19.2") + self.requires("protobuf/3.20.0") if self._with_jemalloc(): self.requires("jemalloc/5.2.1") if self._with_boost(): @@ -267,17 +269,21 @@ def requirements(self): if self._with_gflags(): self.requires("gflags/2.2.2") if self._with_glog(): - self.requires("glog/0.5.0") + self.requires("glog/0.6.0") if self._with_grpc(): - self.requires("grpc/1.44.0") + self.requires("grpc/1.45.2") if self.options.with_json: self.requires("rapidjson/1.1.0") if self._with_llvm(): self.requires("llvm-core/13.0.0") if self._with_openssl(): - self.requires("openssl/1.1.1m") + # aws-sdk-cpp requires openssl/1.1.1. it uses deprecated functions in openssl/3.0.0 + if self.options.with_s3: + self.requires("openssl/1.1.1o") + else: + self.requires("openssl/3.0.3") if self.options.with_s3: - self.requires("aws-sdk-cpp/1.9.100") + self.requires("aws-sdk-cpp/1.9.234") if self.options.with_brotli: self.requires("brotli/1.0.9") if self.options.with_bz2: @@ -295,11 +301,11 @@ def requirements(self): else: self.requires("xsimd/8.0.3") if self.options.with_zlib: - self.requires("zlib/1.2.11") + self.requires("zlib/1.2.12") if self.options.with_zstd: self.requires("zstd/1.5.2") if self._with_re2(): - self.requires("re2/20211101") + self.requires("re2/20220201") if self._with_utf8proc(): self.requires("utf8proc/2.7.0") if self.options.with_backtrace: @@ -410,8 +416,8 @@ def _configure_cmake(self): self._cmake.definitions["ORC_SOURCE"] = "SYSTEM" self._cmake.definitions["ARROW_WITH_THRIFT"] = self._with_thrift() self._cmake.definitions["Thrift_SOURCE"] = "SYSTEM" - self._cmake.definitions["THRIFT_VERSION"] = "1.0" # a recent thrift does not require boost if self._with_thrift(): + self._cmake.definitions["THRIFT_VERSION"] = self.deps_cpp_info["thrift"].version # a recent thrift does not require boost self._cmake.definitions["ARROW_THRIFT_USE_SHARED"] = self.options["thrift"].shared self._cmake.definitions["ARROW_USE_OPENSSL"] = self._with_openssl() if self._with_openssl(): @@ -563,7 +569,7 @@ def package_info(self): if self.options.with_json: self.cpp_info.components["libarrow"].requires.append("rapidjson::rapidjson") if self.options.with_s3: - self.cpp_info.components["libarrow"].requires.append("aws-sdk-cpp::filesystem") + self.cpp_info.components["libarrow"].requires.append("aws-sdk-cpp::s3") if self.options.with_orc: self.cpp_info.components["libarrow"].requires.append("orc::orc") if self.options.with_brotli: diff --git a/ci/conan/all/patches/7.0.0-0004-remove-find-modules.patch b/ci/conan/all/patches/7.0.0-0004-remove-find-modules.patch index 029c620d030..482bb7cd4d8 100644 --- a/ci/conan/all/patches/7.0.0-0004-remove-find-modules.patch +++ b/ci/conan/all/patches/7.0.0-0004-remove-find-modules.patch @@ -314,6 +314,155 @@ index 747df31..0000000 - PROPERTIES IMPORTED_LOCATION "${Snappy_LIB}" - INTERFACE_INCLUDE_DIRECTORIES "${Snappy_INCLUDE_DIR}") -endif() +diff --git a/cpp/cmake_modules/FindThrift.cmake b/cpp/cmake_modules/FindThrift.cmake +index 750d8ce..e69de29 100644 +--- a/cpp/cmake_modules/FindThrift.cmake ++++ b/cpp/cmake_modules/FindThrift.cmake +@@ -1,144 +0,0 @@ +-# Copyright 2012 Cloudera Inc. +-# +-# Licensed under the Apache License, Version 2.0 (the "License"); +-# you may not use this file except in compliance with the License. +-# You may obtain a copy of the License at +-# +-# http://www.apache.org/licenses/LICENSE-2.0 +-# +-# Unless required by applicable law or agreed to in writing, software +-# distributed under the License is distributed on an "AS IS" BASIS, +-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-# See the License for the specific language governing permissions and +-# limitations under the License. +- +-# - Find Thrift (a cross platform RPC lib/tool) +-# +-# Variables used by this module, they can change the default behaviour and need +-# to be set before calling find_package: +-# +-# Thrift_ROOT - When set, this path is inspected instead of standard library +-# locations as the root of the Thrift installation. +-# The environment variable THRIFT_HOME overrides this variable. +-# +-# This module defines +-# THRIFT_VERSION, version string of ant if found +-# THRIFT_INCLUDE_DIR, where to find THRIFT headers +-# THRIFT_LIB, THRIFT library +-# THRIFT_FOUND, If false, do not try to use ant +- +-function(EXTRACT_THRIFT_VERSION) +- if(THRIFT_INCLUDE_DIR) +- file(READ "${THRIFT_INCLUDE_DIR}/thrift/config.h" THRIFT_CONFIG_H_CONTENT) +- string(REGEX MATCH "#define PACKAGE_VERSION \"[0-9.]+\"" THRIFT_VERSION_DEFINITION +- "${THRIFT_CONFIG_H_CONTENT}") +- string(REGEX MATCH "[0-9.]+" THRIFT_VERSION "${THRIFT_VERSION_DEFINITION}") +- set(THRIFT_VERSION +- "${THRIFT_VERSION}" +- PARENT_SCOPE) +- else() +- set(THRIFT_VERSION +- "" +- PARENT_SCOPE) +- endif() +-endfunction(EXTRACT_THRIFT_VERSION) +- +-if(MSVC_TOOLCHAIN AND NOT DEFINED THRIFT_MSVC_LIB_SUFFIX) +- if(NOT ARROW_THRIFT_USE_SHARED) +- if(ARROW_USE_STATIC_CRT) +- set(THRIFT_MSVC_LIB_SUFFIX "mt") +- else() +- set(THRIFT_MSVC_LIB_SUFFIX "md") +- endif() +- endif() +-endif() +-set(THRIFT_LIB_NAME_BASE "thrift${THRIFT_MSVC_LIB_SUFFIX}") +- +-if(ARROW_THRIFT_USE_SHARED) +- set(THRIFT_LIB_NAMES thrift) +- if(CMAKE_IMPORT_LIBRARY_SUFFIX) +- list(APPEND +- THRIFT_LIB_NAMES +- "${CMAKE_IMPORT_LIBRARY_PREFIX}${THRIFT_LIB_NAME_BASE}${CMAKE_IMPORT_LIBRARY_SUFFIX}" +- ) +- endif() +- list(APPEND +- THRIFT_LIB_NAMES +- "${CMAKE_SHARED_LIBRARY_PREFIX}${THRIFT_LIB_NAME_BASE}${CMAKE_SHARED_LIBRARY_SUFFIX}" +- ) +-else() +- set(THRIFT_LIB_NAMES +- "${CMAKE_STATIC_LIBRARY_PREFIX}${THRIFT_LIB_NAME_BASE}${CMAKE_STATIC_LIBRARY_SUFFIX}" +- ) +-endif() +- +-if(Thrift_ROOT) +- find_library(THRIFT_LIB +- NAMES ${THRIFT_LIB_NAMES} +- PATHS ${Thrift_ROOT} +- PATH_SUFFIXES "lib/${CMAKE_LIBRARY_ARCHITECTURE}" "lib") +- find_path(THRIFT_INCLUDE_DIR thrift/Thrift.h +- PATHS ${Thrift_ROOT} +- PATH_SUFFIXES "include") +- find_program(THRIFT_COMPILER thrift +- PATHS ${Thrift_ROOT} +- PATH_SUFFIXES "bin") +- extract_thrift_version() +-else() +- # THRIFT-4760: The pkgconfig files are currently only installed when using autotools. +- # Starting with 0.13, they are also installed for the CMake-based installations of Thrift. +- find_package(PkgConfig QUIET) +- pkg_check_modules(THRIFT_PC thrift) +- if(THRIFT_PC_FOUND) +- set(THRIFT_INCLUDE_DIR "${THRIFT_PC_INCLUDEDIR}") +- +- list(APPEND THRIFT_PC_LIBRARY_DIRS "${THRIFT_PC_LIBDIR}") +- +- find_library(THRIFT_LIB +- NAMES ${THRIFT_LIB_NAMES} +- PATHS ${THRIFT_PC_LIBRARY_DIRS} +- NO_DEFAULT_PATH) +- find_program(THRIFT_COMPILER thrift +- HINTS ${THRIFT_PC_PREFIX} +- NO_DEFAULT_PATH +- PATH_SUFFIXES "bin") +- set(THRIFT_VERSION ${THRIFT_PC_VERSION}) +- else() +- find_library(THRIFT_LIB +- NAMES ${THRIFT_LIB_NAMES} +- PATH_SUFFIXES "lib/${CMAKE_LIBRARY_ARCHITECTURE}" "lib") +- find_path(THRIFT_INCLUDE_DIR thrift/Thrift.h PATH_SUFFIXES "include") +- find_program(THRIFT_COMPILER thrift PATH_SUFFIXES "bin") +- extract_thrift_version() +- endif() +-endif() +- +-if(THRIFT_COMPILER) +- set(Thrift_COMPILER_FOUND TRUE) +-else() +- set(Thrift_COMPILER_FOUND FALSE) +-endif() +- +-find_package_handle_standard_args( +- Thrift +- REQUIRED_VARS THRIFT_LIB THRIFT_INCLUDE_DIR +- VERSION_VAR THRIFT_VERSION +- HANDLE_COMPONENTS) +- +-if(Thrift_FOUND OR THRIFT_FOUND) +- set(Thrift_FOUND TRUE) +- if(ARROW_THRIFT_USE_SHARED) +- add_library(thrift::thrift SHARED IMPORTED) +- else() +- add_library(thrift::thrift STATIC IMPORTED) +- endif() +- set_target_properties(thrift::thrift +- PROPERTIES IMPORTED_LOCATION "${THRIFT_LIB}" +- INTERFACE_INCLUDE_DIRECTORIES "${THRIFT_INCLUDE_DIR}") +- if(WIN32 AND NOT MSVC_TOOLCHAIN) +- # We don't need this for Visual C++ because Thrift uses +- # "#pragma comment(lib, "Ws2_32.lib")" in +- # thrift/windows/config.h for Visual C++. +- set_target_properties(thrift::thrift PROPERTIES INTERFACE_LINK_LIBRARIES "ws2_32") +- endif() +-endif() diff --git a/cpp/cmake_modules/Findjemalloc.cmake a/cpp/cmake_modules/Findjemalloc.cmake deleted file mode 100644 index 84bb81f..0000000 diff --git a/ci/conan/all/patches/7.0.0-0005-use-find-package.patch b/ci/conan/all/patches/7.0.0-0005-use-find-package.patch index dcb8cae3c48..085871aaea3 100644 --- a/ci/conan/all/patches/7.0.0-0005-use-find-package.patch +++ b/ci/conan/all/patches/7.0.0-0005-use-find-package.patch @@ -45,7 +45,7 @@ index 2d7baf1..c2e86e0 100644 if(ARROW_MIMALLOC) diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake -index bc38952..8196e09 100644 +index bc38952..ff81d00 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -953,14 +953,7 @@ else() @@ -120,7 +120,25 @@ index bc38952..8196e09 100644 # TODO: Don't use global includes but rather target_include_directories include_directories(SYSTEM ${GFLAGS_INCLUDE_DIR}) -@@ -1606,7 +1598,7 @@ if(ARROW_JEMALLOC) +@@ -1400,6 +1392,7 @@ macro(build_thrift) + endmacro() + + if(ARROW_WITH_THRIFT) ++if (0) + # We already may have looked for Thrift earlier, when considering whether + # to build Boost, so don't look again if already found. + if(NOT Thrift_FOUND) +@@ -1412,6 +1405,9 @@ if(ARROW_WITH_THRIFT) + endif() + # TODO: Don't use global includes but rather target_include_directories + include_directories(SYSTEM ${THRIFT_INCLUDE_DIR}) ++else() ++ find_package(Thrift REQUIRED CONFIG) ++endif() + + string(REPLACE "." ";" VERSION_LIST ${THRIFT_VERSION}) + list(GET VERSION_LIST 0 THRIFT_VERSION_MAJOR) +@@ -1606,7 +1602,7 @@ if(ARROW_JEMALLOC) # conflict with the default allocator as well as other jemalloc # installations. # find_package(jemalloc) @@ -129,7 +147,7 @@ index bc38952..8196e09 100644 set(ARROW_JEMALLOC_USE_SHARED OFF) set(JEMALLOC_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/jemalloc_ep-prefix/src/jemalloc_ep/dist/") -@@ -1664,6 +1656,9 @@ if(ARROW_JEMALLOC) +@@ -1664,6 +1660,9 @@ if(ARROW_JEMALLOC) "${CMAKE_CURRENT_BINARY_DIR}/jemalloc_ep-prefix/src") add_dependencies(jemalloc::jemalloc jemalloc_ep) @@ -139,7 +157,7 @@ index bc38952..8196e09 100644 list(APPEND ARROW_BUNDLED_STATIC_LIBS jemalloc::jemalloc) endif() -@@ -1671,6 +1666,8 @@ endif() +@@ -1671,6 +1670,8 @@ endif() # mimalloc - Cross-platform high-performance allocator, from Microsoft if(ARROW_MIMALLOC) @@ -148,7 +166,7 @@ index bc38952..8196e09 100644 message(STATUS "Building (vendored) mimalloc from source") # We only use a vendored mimalloc as we want to control its build options. -@@ -1715,6 +1712,13 @@ if(ARROW_MIMALLOC) +@@ -1715,6 +1716,13 @@ if(ARROW_MIMALLOC) add_dependencies(mimalloc::mimalloc mimalloc_ep) add_dependencies(toolchain mimalloc_ep) @@ -162,7 +180,7 @@ index bc38952..8196e09 100644 list(APPEND ARROW_BUNDLED_STATIC_LIBS mimalloc::mimalloc) endif() -@@ -2036,10 +2040,21 @@ macro(build_xsimd) +@@ -2036,10 +2044,21 @@ macro(build_xsimd) set(XSIMD_VENDORED TRUE) endmacro() @@ -186,7 +204,7 @@ index bc38952..8196e09 100644 # TODO: Don't use global includes but rather target_include_directories include_directories(SYSTEM ${XSIMD_INCLUDE_DIR}) endif() -@@ -2082,11 +2097,14 @@ macro(build_zlib) +@@ -2082,11 +2101,14 @@ macro(build_zlib) endmacro() if(ARROW_WITH_ZLIB) @@ -201,7 +219,7 @@ index bc38952..8196e09 100644 endif() macro(build_lz4) -@@ -2140,11 +2158,14 @@ macro(build_lz4) +@@ -2140,11 +2162,14 @@ macro(build_lz4) endmacro() if(ARROW_WITH_LZ4) @@ -216,7 +234,7 @@ index bc38952..8196e09 100644 endif() macro(build_zstd) -@@ -2205,6 +2226,7 @@ macro(build_zstd) +@@ -2205,6 +2230,7 @@ macro(build_zstd) endmacro() if(ARROW_WITH_ZSTD) @@ -224,7 +242,7 @@ index bc38952..8196e09 100644 # ARROW-13384: ZSTD_minCLevel was added in v1.4.0, required by ARROW-13091 resolve_dependency(zstd PC_PACKAGE_NAMES -@@ -2232,6 +2254,8 @@ if(ARROW_WITH_ZSTD) +@@ -2232,6 +2258,8 @@ if(ARROW_WITH_ZSTD) get_target_property(ZSTD_INCLUDE_DIR ${ARROW_ZSTD_LIBZSTD} INTERFACE_INCLUDE_DIRECTORIES) include_directories(SYSTEM ${ZSTD_INCLUDE_DIR}) @@ -233,7 +251,7 @@ index bc38952..8196e09 100644 endif() # ---------------------------------------------------------------------- -@@ -2271,6 +2295,7 @@ macro(build_re2) +@@ -2271,6 +2299,7 @@ macro(build_re2) endmacro() if(ARROW_WITH_RE2) @@ -241,7 +259,7 @@ index bc38952..8196e09 100644 # Don't specify "PC_PACKAGE_NAMES re2" here because re2.pc may # include -std=c++11. It's not compatible with C source and C++ # source not uses C++ 11. -@@ -2284,6 +2309,8 @@ if(ARROW_WITH_RE2) +@@ -2284,6 +2313,8 @@ if(ARROW_WITH_RE2) # TODO: Don't use global includes but rather target_include_directories get_target_property(RE2_INCLUDE_DIR re2::re2 INTERFACE_INCLUDE_DIRECTORIES) include_directories(SYSTEM ${RE2_INCLUDE_DIR}) @@ -250,7 +268,7 @@ index bc38952..8196e09 100644 endif() macro(build_bzip2) -@@ -2335,10 +2362,7 @@ macro(build_bzip2) +@@ -2335,10 +2366,7 @@ macro(build_bzip2) endmacro() if(ARROW_WITH_BZ2) @@ -262,7 +280,7 @@ index bc38952..8196e09 100644 if(NOT TARGET BZip2::BZip2) add_library(BZip2::BZip2 UNKNOWN IMPORTED) -@@ -2390,11 +2414,7 @@ macro(build_utf8proc) +@@ -2390,11 +2418,7 @@ macro(build_utf8proc) endmacro() if(ARROW_WITH_UTF8PROC) @@ -271,11 +289,11 @@ index bc38952..8196e09 100644 - "2.2.0" - PC_PACKAGE_NAMES - libutf8proc) -+ find_package(BZip2 REQUIRED CONFIG) ++ find_package(utf8proc REQUIRED CONFIG) add_definitions(-DARROW_WITH_UTF8PROC) -@@ -3554,33 +3574,12 @@ if(ARROW_WITH_GRPC) +@@ -3554,33 +3578,12 @@ if(ARROW_WITH_GRPC) message(STATUS "Forcing gRPC_SOURCE to Protobuf_SOURCE (${Protobuf_SOURCE})") set(gRPC_SOURCE "${Protobuf_SOURCE}") endif() diff --git a/ci/conan/merge_status.sh b/ci/conan/merge_status.sh new file mode 100644 index 00000000000..daed3b81eb6 --- /dev/null +++ b/ci/conan/merge_status.sh @@ -0,0 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +UPSTREAM_REVISION=232a32d832f9754b81dde348e8fd8ded37ad404b diff --git a/ci/conan/merge_upstream.sh b/ci/conan/merge_upstream.sh new file mode 100755 index 00000000000..3d2c3334250 --- /dev/null +++ b/ci/conan/merge_upstream.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -eu + +source_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +if [ "$#" -ne 1 ]; then + echo "Usage: $0 CONAN_CENTER_INDEX_WORK_DIRECTORY" + echo " e.g.: $0 ~/work/conan/conan-center-index" + exit 1 +fi + +conan_center_index="$1" + +. "${source_dir}/merge_status.sh" + +UPSTREAM_HEAD=$(git -C "${conan_center_index}" log -n1 --format=%H) +git \ + -C "${conan_center_index}" \ + diff \ + ${UPSTREAM_REVISION}..${UPSTREAM_HEAD} \ + recipes/arrow | \ + (cd "${source_dir}" && patch -p3) + +sed \ + -i.bak \ + -E \ + -e "s/^(UPSTREAM_REVISION)=.*$/\\1=${UPSTREAM_HEAD}/g" \ + "${source_dir}/merge_status.sh" +rm "${source_dir}/merge_status.sh.bak"