From 3373defb885b8218b5d0b2630bed18c78b5e87df Mon Sep 17 00:00:00 2001 From: David Li Date: Wed, 25 Aug 2021 16:27:11 -0400 Subject: [PATCH 1/8] ARROW-13760: [C++] Bump required Protobuf when using Flight --- cpp/cmake_modules/ThirdpartyToolchain.cmake | 2 +- cpp/thirdparty/versions.txt | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index 9c4da8bcfce..a3ab0f54b8f 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -1421,7 +1421,7 @@ endmacro() if(ARROW_WITH_PROTOBUF) if(ARROW_WITH_GRPC) # gRPC 1.21.0 or later require Protobuf 3.7.0 or later. - set(ARROW_PROTOBUF_REQUIRED_VERSION "3.7.0") + set(ARROW_PROTOBUF_REQUIRED_VERSION "3.15.0") elseif(ARROW_GANDIVA_JAVA) # google::protobuf::MessageLite::ByteSize() is deprecated since # Protobuf 3.4.0. diff --git a/cpp/thirdparty/versions.txt b/cpp/thirdparty/versions.txt index 80219147c94..12c95fd4349 100644 --- a/cpp/thirdparty/versions.txt +++ b/cpp/thirdparty/versions.txt @@ -62,8 +62,8 @@ ARROW_MIMALLOC_BUILD_VERSION=v1.7.2 ARROW_MIMALLOC_BUILD_SHA256_CHECKSUM=b1912e354565a4b698410f7583c0f83934a6dbb3ade54ab7ddcb1569320936bd ARROW_ORC_BUILD_VERSION=1.6.9 ARROW_ORC_BUILD_SHA256_CHECKSUM=2b79f1152454a38abd7b361b407f65689e275f1fb28d4293892e42fbebcc0a9d -ARROW_PROTOBUF_BUILD_VERSION=v3.14.0 -ARROW_PROTOBUF_BUILD_SHA256_CHECKSUM=6dd0f6b20094910fbb7f1f7908688df01af2d4f6c5c21331b9f636048674aebf +ARROW_PROTOBUF_BUILD_VERSION=v3.17.3 +ARROW_PROTOBUF_BUILD_SHA256_CHECKSUM=77ad26d3f65222fd96ccc18b055632b0bfedf295cb748b712a98ba1ac0b704b2 # Because of https://github.com/Tencent/rapidjson/pull/1323, we require # a pre-release version of RapidJSON to build with GCC 8 without # warnings. From ee7ca51ba2d85df3883751b1456f24b3ea1ba264 Mon Sep 17 00:00:00 2001 From: David Li Date: Wed, 25 Aug 2021 16:38:17 -0400 Subject: [PATCH 2/8] ARROW-13760: [C++] Update comment --- cpp/cmake_modules/ThirdpartyToolchain.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index a3ab0f54b8f..71fb6b18834 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -1420,7 +1420,7 @@ endmacro() if(ARROW_WITH_PROTOBUF) if(ARROW_WITH_GRPC) - # gRPC 1.21.0 or later require Protobuf 3.7.0 or later. + # FlightSQL uses proto3 optionals, which require 3.15 or later. set(ARROW_PROTOBUF_REQUIRED_VERSION "3.15.0") elseif(ARROW_GANDIVA_JAVA) # google::protobuf::MessageLite::ByteSize() is deprecated since From 8ed8f84a9c880760faebc1b2d29d834b646779f3 Mon Sep 17 00:00:00 2001 From: David Li Date: Thu, 26 Aug 2021 14:50:38 -0400 Subject: [PATCH 3/8] ARROW-13760: [C++] Bundle Protobuf where appropriate --- docker-compose.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docker-compose.yml b/docker-compose.yml index cdbba66aebc..2c3736d8a00 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -449,6 +449,7 @@ services: <<: *ccache ARROW_ENABLE_TIMING_TESTS: # inherit ARROW_MIMALLOC: "ON" + Protobuf_SOURCE: "BUNDLED" # Need Protobuf >= 3.15 volumes: &fedora-volumes - .:/arrow:delegated - ${DOCKER_VOLUME_PREFIX}fedora-ccache:/ccache:delegated @@ -687,6 +688,7 @@ services: shm_size: *shm-size environment: <<: *ccache + Protobuf_SOURCE: "BUNDLED" # Need Protobuf >= 3.15 volumes: *fedora-volumes command: *python-command @@ -1353,6 +1355,7 @@ services: <<: *ccache ARROW_CUDA: "ON" ARROW_GLIB_GTK_DOC: "true" + Protobuf_SOURCE: "BUNDLED" # Need Protobuf >= 3.15 volumes: *ubuntu-volumes command: &docs-command > /bin/bash -c " From fc8aec207ae7056403254aa213968b0b182f6d4c Mon Sep 17 00:00:00 2001 From: David Li Date: Thu, 26 Aug 2021 14:51:43 -0400 Subject: [PATCH 4/8] ARROW-13760: [C++] Attempt to fix ubuntu-hirsute-amd64 --- dev/tasks/linux-packages/apache-arrow/debian/rules | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dev/tasks/linux-packages/apache-arrow/debian/rules b/dev/tasks/linux-packages/apache-arrow/debian/rules index 2de53361559..65dc098446b 100755 --- a/dev/tasks/linux-packages/apache-arrow/debian/rules +++ b/dev/tasks/linux-packages/apache-arrow/debian/rules @@ -47,6 +47,8 @@ override_dh_auto_configure: -DCMAKE_BUILD_TYPE=$(BUILD_TYPE) \ -DCMAKE_UNITY_BUILD=ON \ -DPARQUET_REQUIRE_ENCRYPTION=ON \ + -DProtobuf_SOURCE=BUNDLED \ + -DgRPC_SOURCE=BUNDLED \ -DPythonInterp_FIND_VERSION=ON \ -DPythonInterp_FIND_VERSION_MAJOR=3 From 7fd4e1938517f8eef34383b39ba20c44729fad5f Mon Sep 17 00:00:00 2001 From: David Li Date: Thu, 26 Aug 2021 21:05:01 -0400 Subject: [PATCH 5/8] ARROW-13760: [C++] Revert previous attempt --- dev/tasks/linux-packages/apache-arrow/debian/rules | 2 -- 1 file changed, 2 deletions(-) diff --git a/dev/tasks/linux-packages/apache-arrow/debian/rules b/dev/tasks/linux-packages/apache-arrow/debian/rules index 65dc098446b..2de53361559 100755 --- a/dev/tasks/linux-packages/apache-arrow/debian/rules +++ b/dev/tasks/linux-packages/apache-arrow/debian/rules @@ -47,8 +47,6 @@ override_dh_auto_configure: -DCMAKE_BUILD_TYPE=$(BUILD_TYPE) \ -DCMAKE_UNITY_BUILD=ON \ -DPARQUET_REQUIRE_ENCRYPTION=ON \ - -DProtobuf_SOURCE=BUNDLED \ - -DgRPC_SOURCE=BUNDLED \ -DPythonInterp_FIND_VERSION=ON \ -DPythonInterp_FIND_VERSION_MAJOR=3 From c97ae11fb1d424fad65082b61356ec5efbdf7363 Mon Sep 17 00:00:00 2001 From: David Li Date: Fri, 27 Aug 2021 09:16:38 -0400 Subject: [PATCH 6/8] ARROW-13760: [C++] Strip LTO flags from Protobuf build --- cpp/cmake_modules/ThirdpartyToolchain.cmake | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index 71fb6b18834..ef9c33dc498 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -1371,19 +1371,33 @@ macro(build_protobuf) BUILD_COMMAND ${PROTOBUF_BUILD_COMMAND}) else() + # Strip lto flags (which may be added by dh_auto_configure) + # See https://github.com/protocolbuffers/protobuf/issues/7092 + set(PROTOBUF_C_FLAGS ${EP_C_FLAGS}) + set(PROTOBUF_CXX_FLAGS ${EP_CXX_FLAGS}) + string(REPLACE "-flto=auto" "" PROTOBUF_C_FLAGS "${PROTOBUF_C_FLAGS}") + string(REPLACE "-ffat-lto-objects" "" PROTOBUF_C_FLAGS "${PROTOBUF_C_FLAGS}") + string(REPLACE "-flto=auto" "" PROTOBUF_CXX_FLAGS "${PROTOBUF_CXX_FLAGS}") + string(REPLACE "-ffat-lto-objects" "" PROTOBUF_CXX_FLAGS "${PROTOBUF_CXX_FLAGS}") set(PROTOBUF_CMAKE_ARGS ${EP_COMMON_CMAKE_ARGS} -DBUILD_SHARED_LIBS=OFF -DCMAKE_INSTALL_LIBDIR=lib "-DCMAKE_INSTALL_PREFIX=${PROTOBUF_PREFIX}" -Dprotobuf_BUILD_TESTS=OFF - -Dprotobuf_DEBUG_POSTFIX=) + -Dprotobuf_DEBUG_POSTFIX= + "-DCMAKE_C_FLAGS=${PROTOBUF_C_FLAGS}" + "-DCMAKE_CXX_FLAGS=${PROTOBUF_CXX_FLAGS}" + "-DCMAKE_C_FLAGS_${UPPERCASE_BUILD_TYPE}=${PROTOBUF_C_FLAGS}" + "-DCMAKE_CXX_FLAGS_${UPPERCASE_BUILD_TYPE}=${PROTOBUF_CXX_FLAGS}" + ) if(MSVC AND NOT ARROW_USE_STATIC_CRT) list(APPEND PROTOBUF_CMAKE_ARGS "-Dprotobuf_MSVC_STATIC_RUNTIME=OFF") endif() if(ZLIB_ROOT) list(APPEND PROTOBUF_CMAKE_ARGS "-DZLIB_ROOT=${ZLIB_ROOT}") endif() + list(APPEND PROTOBUF_CMAKE_ARGS "-DCMAKE_VERBOSE_MAKEFILE:BOOL=ON") set(PROTOBUF_EXTERNAL_PROJECT_ADD_ARGS CMAKE_ARGS ${PROTOBUF_CMAKE_ARGS} SOURCE_SUBDIR "cmake") endif() From 193359829b7459c57e10d8515acf7ab746018dbc Mon Sep 17 00:00:00 2001 From: David Li Date: Fri, 27 Aug 2021 11:39:40 -0400 Subject: [PATCH 7/8] ARROW-13760: [C++] Lint --- cpp/cmake_modules/ThirdpartyToolchain.cmake | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index ef9c33dc498..b4145e804b8 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -1389,8 +1389,7 @@ macro(build_protobuf) "-DCMAKE_C_FLAGS=${PROTOBUF_C_FLAGS}" "-DCMAKE_CXX_FLAGS=${PROTOBUF_CXX_FLAGS}" "-DCMAKE_C_FLAGS_${UPPERCASE_BUILD_TYPE}=${PROTOBUF_C_FLAGS}" - "-DCMAKE_CXX_FLAGS_${UPPERCASE_BUILD_TYPE}=${PROTOBUF_CXX_FLAGS}" - ) + "-DCMAKE_CXX_FLAGS_${UPPERCASE_BUILD_TYPE}=${PROTOBUF_CXX_FLAGS}") if(MSVC AND NOT ARROW_USE_STATIC_CRT) list(APPEND PROTOBUF_CMAKE_ARGS "-Dprotobuf_MSVC_STATIC_RUNTIME=OFF") endif() From 9cb44e258fc19ff2fbac75a417f7561af52a2eae Mon Sep 17 00:00:00 2001 From: David Li Date: Mon, 30 Aug 2021 17:23:43 -0400 Subject: [PATCH 8/8] ARROW-13760: [C++] Remove debug flag --- cpp/cmake_modules/ThirdpartyToolchain.cmake | 1 - 1 file changed, 1 deletion(-) diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index b4145e804b8..4120662e5f1 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -1396,7 +1396,6 @@ macro(build_protobuf) if(ZLIB_ROOT) list(APPEND PROTOBUF_CMAKE_ARGS "-DZLIB_ROOT=${ZLIB_ROOT}") endif() - list(APPEND PROTOBUF_CMAKE_ARGS "-DCMAKE_VERBOSE_MAKEFILE:BOOL=ON") set(PROTOBUF_EXTERNAL_PROJECT_ADD_ARGS CMAKE_ARGS ${PROTOBUF_CMAKE_ARGS} SOURCE_SUBDIR "cmake") endif()