diff --git a/ci/appveyor-cpp-build.bat b/ci/appveyor-cpp-build.bat index 287197458e1..e9441c63e20 100644 --- a/ci/appveyor-cpp-build.bat +++ b/ci/appveyor-cpp-build.bat @@ -28,6 +28,9 @@ set PARQUET_TEST_DATA=%CD%\cpp\submodules\parquet-testing\data set ARROW_DEBUG_MEMORY_POOL=trap +set CMAKE_BUILD_PARALLEL_LEVEL=%NUMBER_OF_PROCESSORS% +set CTEST_PARALLEL_LEVEL=%NUMBER_OF_PROCESSORS% + @rem @rem In the configurations below we disable building the Arrow static library @rem to save some time. Unfortunately this will still build the Parquet static @@ -51,7 +54,7 @@ if "%JOB%" == "Build_Debug" ( .. || exit /B cmake --build . --config Debug || exit /B - ctest --output-on-failure -j2 || exit /B + ctest --output-on-failure || exit /B popd @rem Finish Debug build successfully @@ -121,13 +124,13 @@ cmake -G "%GENERATOR%" %CMAKE_ARGS% ^ -DPARQUET_BUILD_EXECUTABLES=ON ^ -DPARQUET_REQUIRE_ENCRYPTION=ON ^ .. || exit /B -cmake --build . --target install --config %CONFIGURATION% || exit /B +cmake --build . --target install --config Release || exit /B @rem Needed so arrow-python-test.exe works set OLD_PYTHONHOME=%PYTHONHOME% set PYTHONHOME=%CONDA_PREFIX% -ctest --output-on-failure -j2 || exit /B +ctest --output-on-failure || exit /B set PYTHONHOME=%OLD_PYTHONHOME% popd diff --git a/ci/appveyor-cpp-setup.bat b/ci/appveyor-cpp-setup.bat index 936306296f6..af3456c452d 100644 --- a/ci/appveyor-cpp-setup.bat +++ b/ci/appveyor-cpp-setup.bat @@ -71,19 +71,13 @@ if "%JOB%" NEQ "Build_Debug" ( mamba create -n arrow -q -y -c conda-forge ^ --file=ci\conda_env_python.txt ^ %CONDA_PACKAGES% ^ - "cmake=3.17" ^ + "cmake" ^ "ninja" ^ "nomkl" ^ "pandas" ^ "fsspec" ^ "python=%PYTHON%" ^ || exit /B - - @rem On Windows, GTest is always bundled from source instead of using - @rem conda binaries, avoid any interference between the two versions. - if "%JOB%" == "Toolchain" ( - mamba uninstall -n arrow -q -y -c conda-forge gtest || exit /B - ) ) @rem @@ -96,6 +90,8 @@ if defined need_vcvarsall ( exit /B ) call "C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\VC\Auxiliary\Build\vcvarsall.bat" amd64 + set CC=cl.exe + set CXX=cl.exe ) @rem @@ -128,4 +124,4 @@ move tzdata %USERPROFILE%\Downloads\tzdata @rem Also need Windows timezone mapping curl https://raw.githubusercontent.com/unicode-org/cldr/master/common/supplemental/windowsZones.xml ^ --output %USERPROFILE%\Downloads\tzdata\windowsZones.xml -@rem (Doc section: Download timezone database) \ No newline at end of file +@rem (Doc section: Download timezone database) diff --git a/cpp/cmake_modules/BuildUtils.cmake b/cpp/cmake_modules/BuildUtils.cmake index f3e3269d4fe..c153f3ab889 100644 --- a/cpp/cmake_modules/BuildUtils.cmake +++ b/cpp/cmake_modules/BuildUtils.cmake @@ -382,7 +382,7 @@ function(ADD_ARROW_LIB LIB_NAME) set(LIB_NAME_STATIC ${LIB_NAME}) endif() - if(ARROW_BUILD_STATIC AND WIN32) + if(WIN32) target_compile_definitions(${LIB_NAME}_static PUBLIC ARROW_STATIC) target_compile_definitions(${LIB_NAME}_static PUBLIC ARROW_FLIGHT_STATIC) endif() diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index 158be06d7d9..637bf9d04cb 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -1972,8 +1972,6 @@ macro(build_gtest) -DCMAKE_MACOSX_RPATH=OFF) set(GMOCK_INCLUDE_DIR "${GTEST_PREFIX}/include") - add_definitions(-DGTEST_LINKED_AS_SHARED_LIBRARY=1) - if(MSVC AND NOT ARROW_USE_STATIC_CRT) set(GTEST_CMAKE_ARGS ${GTEST_CMAKE_ARGS} -Dgtest_force_shared_crt=ON) endif() @@ -2031,6 +2029,8 @@ macro(build_gtest) add_library(GTest::gtest SHARED IMPORTED) set_target_properties(GTest::gtest PROPERTIES ${_GTEST_IMPORTED_TYPE} "${GTEST_SHARED_LIB}" + INTERFACE_COMPILE_DEFINITIONS + "GTEST_LINKED_AS_SHARED_LIBRARY=1" INTERFACE_INCLUDE_DIRECTORIES "${GTEST_INCLUDE_DIR}") add_library(GTest::gtest_main SHARED IMPORTED) @@ -2041,6 +2041,8 @@ macro(build_gtest) add_library(GTest::gmock SHARED IMPORTED) set_target_properties(GTest::gmock PROPERTIES ${_GTEST_IMPORTED_TYPE} "${GMOCK_SHARED_LIB}" + INTERFACE_COMPILE_DEFINITIONS + "GMOCK_LINKED_AS_SHARED_LIBRARY=1" INTERFACE_INCLUDE_DIRECTORIES "${GTEST_INCLUDE_DIR}") add_dependencies(toolchain-tests googletest_ep) add_dependencies(GTest::gtest googletest_ep) @@ -2049,40 +2051,23 @@ macro(build_gtest) endmacro() if(ARROW_TESTING) + if(CMAKE_VERSION VERSION_LESS 3.23) + set(GTEST_USE_CONFIG TRUE) + else() + set(GTEST_USE_CONFIG FALSE) + endif() + # We can't find shred library version of GoogleTest on Windows with + # Conda's gtest package because it doesn't provide GTestConfig.cmake + # provided by GoogleTest and CMake's built-in FindGTtest.cmake + # doesn't support gtest_dll.dll. If we find static library version + # of GoogleTest and ARROW_BUILD_SHARED=ON/ARROW_BUILD_STATIC=OFF are + # specified, we need to build arrow_flight_testing as a static + # library. See cpp/src/arrow/flight/CMakeLists.txt for details. resolve_dependency(GTest REQUIRED_VERSION 1.10.0 USE_CONFIG - TRUE) - - if(NOT GTEST_VENDORED) - # TODO(wesm): This logic does not work correctly with the MSVC static libraries - # built for the shared crt - - # set(CMAKE_REQUIRED_LIBRARIES GTest::GTest GTest::Main GTest::GMock) - # CHECK_CXX_SOURCE_COMPILES(" - # #include - # #include - - # class A { - # public: - # int run() const { return 1; } - # }; - - # class B : public A { - # public: - # MOCK_CONST_METHOD0(run, int()); - # }; - - # TEST(Base, Test) { - # B b; - # }" GTEST_COMPILES_WITHOUT_MACRO) - # if (NOT GTEST_COMPILES_WITHOUT_MACRO) - # message(STATUS "Setting GTEST_LINKED_AS_SHARED_LIBRARY=1 on GTest::GTest") - # add_compile_definitions("GTEST_LINKED_AS_SHARED_LIBRARY=1") - # endif() - # set(CMAKE_REQUIRED_LIBRARIES) - endif() + ${GTEST_USE_CONFIG}) endif() macro(build_benchmark) diff --git a/cpp/src/arrow/CMakeLists.txt b/cpp/src/arrow/CMakeLists.txt index 3b49f4ca00a..77f9959fdbe 100644 --- a/cpp/src/arrow/CMakeLists.txt +++ b/cpp/src/arrow/CMakeLists.txt @@ -754,20 +754,6 @@ endif() if(ARROW_FLIGHT) add_subdirectory(flight) - - if(ARROW_WITH_UCX) - add_subdirectory(flight/transport/ucx) - endif() -endif() - -if(ARROW_FLIGHT_SQL) - add_subdirectory(flight/sql) -endif() - -if(ARROW_FLIGHT - AND ARROW_FLIGHT_SQL - AND ARROW_BUILD_INTEGRATION) - add_subdirectory(flight/integration_tests) endif() if(ARROW_HIVESERVER2) diff --git a/cpp/src/arrow/flight/CMakeLists.txt b/cpp/src/arrow/flight/CMakeLists.txt index c09159eb151..ab4a47d39aa 100644 --- a/cpp/src/arrow/flight/CMakeLists.txt +++ b/cpp/src/arrow/flight/CMakeLists.txt @@ -25,27 +25,52 @@ if(WIN32) list(APPEND ARROW_FLIGHT_LINK_LIBS ws2_32.lib) endif() -set(ARROW_FLIGHT_TEST_LINKAGE - "${ARROW_TEST_LINKAGE}" - PARENT_SCOPE) +set(ARROW_FLIGHT_TEST_LINKAGE "${ARROW_TEST_LINKAGE}") if(Protobuf_USE_STATIC_LIBS) message(STATUS "Linking Arrow Flight tests statically due to static Protobuf") - set(ARROW_FLIGHT_TEST_LINKAGE - "static" - PARENT_SCOPE) + set(ARROW_FLIGHT_TEST_LINKAGE "static") endif() if(NOT ARROW_GRPC_USE_SHARED) message(STATUS "Linking Arrow Flight tests statically due to static gRPC") - set(ARROW_FLIGHT_TEST_LINKAGE - "static" - PARENT_SCOPE) + set(ARROW_FLIGHT_TEST_LINKAGE "static") +endif() +if(ARROW_TESTING) + if(WIN32) + # We need to use the same library type for arrow_flight_testing + # and Google Test because arrow_flight_testing has + # arrow::flight:FlightTest that inherits testing::Test. If + # arrow_flight_testing's library type and gtest's library type are + # different such as arrow_flight_testing is a shared library and + # gtest is a static library, link error is occurred. + get_target_property(GTEST_TYPE GTest::gtest TYPE) + if(GTEST_TYPE STREQUAL "SHARED_LIBRARY") + set(ARROW_FLIGHT_TESTING_BUILD_SHARED TRUE) + set(ARROW_FLIGHT_TESTING_BUILD_STATIC FALSE) + else() + set(ARROW_FLIGHT_TESTING_BUILD_SHARED FALSE) + set(ARROW_FLIGHT_TESTING_BUILD_STATIC TRUE) + message(STATUS "Linking Arrow Flight tests statically due to static GTest") + set(ARROW_FLIGHT_TEST_LINKAGE "static") + endif() + else() + set(ARROW_FLIGHT_TESTING_BUILD_SHARED ${ARROW_BUILD_SHARED}) + set(ARROW_FLIGHT_TESTING_BUILD_STATIC ${ARROW_BUILD_STATIC}) + endif() endif() set(ARROW_FLIGHT_TEST_INTERFACE_LIBS) if(ARROW_FLIGHT_TEST_LINKAGE STREQUAL "static") - set(ARROW_FLIGHT_TEST_LINK_LIBS - arrow_flight_static arrow_flight_testing_static ${ARROW_FLIGHT_STATIC_LINK_LIBS} - ${ARROW_TEST_LINK_LIBS}) + if(ARROW_BUILD_STATIC) + set(ARROW_FLIGHT_TEST_LINK_LIBS arrow_flight_static) + else() + set(ARROW_FLIGHT_TEST_LINK_LIBS arrow_flight_shared) + endif() + if(ARROW_FLIGHT_TESTING_BUILD_STATIC) + list(APPEND ARROW_FLIGHT_TEST_LINK_LIBS arrow_flight_testing_static) + else() + list(APPEND ARROW_FLIGHT_TEST_LINK_LIBS arrow_flight_testing_shared) + endif() + list(APPEND ARROW_FLIGHT_TEST_LINK_LIBS ${ARROW_TEST_LINK_LIBS}) if(ARROW_CUDA) list(APPEND ARROW_FLIGHT_TEST_INTERFACE_LIBS arrow_cuda_static) list(APPEND ARROW_FLIGHT_TEST_LINK_LIBS arrow_cuda_static) @@ -65,11 +90,7 @@ list(APPEND Boost::system GTest::gtest GTest::gmock) - -# Needed for Flight SQL and integration -set(ARROW_FLIGHT_TEST_LINK_LIBS - ${ARROW_FLIGHT_TEST_LINK_LIBS} - PARENT_SCOPE) +list(APPEND ARROW_FLIGHT_TEST_LINK_LIBS gRPC::grpc++) # TODO(wesm): Protobuf shared vs static linking @@ -239,7 +260,30 @@ endforeach() # Define arrow_flight_testing library if(ARROW_TESTING) + if(ARROW_BUILD_SHARED AND ARROW_BUILD_STATIC) + set(ARROW_FLIGHT_TESTING_SHARED_LINK_LIBS arrow_shared arrow_flight_shared) + set(ARROW_FLIGHT_TESTING_STATIC_LINK_LIBS arrow_static arrow_flight_static) + elseif(ARROW_BUILD_SHARED) + set(ARROW_FLIGHT_TESTING_SHARED_LINK_LIBS arrow_shared arrow_flight_shared) + set(ARROW_FLIGHT_TESTING_STATIC_LINK_LIBS arrow_shared arrow_flight_shared) + else() + set(ARROW_FLIGHT_TESTING_SHARED_LINK_LIBS arrow_static arrow_flight_static) + set(ARROW_FLIGHT_TESTING_STATIC_LINK_LIBS arrow_static arrow_flight_static) + endif() + if(ARROW_TEST_LINKAGE STREQUAL "shared") + list(APPEND ARROW_FLIGHT_TESTING_SHARED_LINK_LIBS arrow_testing_shared) + list(APPEND ARROW_FLIGHT_TESTING_STATIC_LINK_LIBS arrow_testing_shared) + else() + list(APPEND ARROW_FLIGHT_TESTING_SHARED_LINK_LIBS arrow_testing_static) + list(APPEND ARROW_FLIGHT_TESTING_STATIC_LINK_LIBS arrow_testing_static) + endif() + list(APPEND ARROW_FLIGHT_TESTING_SHARED_LINK_LIBS ${ARROW_FLIGHT_TEST_INTERFACE_LIBS}) + list(APPEND ARROW_FLIGHT_TESTING_STATIC_LINK_LIBS ${ARROW_FLIGHT_TEST_INTERFACE_LIBS}) add_arrow_lib(arrow_flight_testing + BUILD_SHARED + ${ARROW_FLIGHT_TESTING_BUILD_SHARED} + BUILD_STATIC + ${ARROW_FLIGHT_TESTING_BUILD_STATIC} CMAKE_PACKAGE_NAME ArrowFlightTesting PKG_CONFIG_NAME @@ -254,15 +298,9 @@ if(ARROW_TESTING) flight_grpc_gen arrow_dependencies SHARED_LINK_LIBS - arrow_shared - arrow_flight_shared - arrow_testing_shared - ${ARROW_FLIGHT_TEST_INTERFACE_LIBS} + ${ARROW_FLIGHT_TESTING_SHARED_LINK_LIBS} STATIC_LINK_LIBS - arrow_static - arrow_flight_static - arrow_testing_static - ${ARROW_FLIGHT_TEST_INTERFACE_LIBS} + ${ARROW_FLIGHT_TESTING_STATIC_LINK_LIBS} PRIVATE_INCLUDES "${Protobuf_INCLUDE_DIRS}") @@ -328,3 +366,15 @@ if(ARROW_BUILD_BENCHMARKS) endif() endif() endif(ARROW_BUILD_BENCHMARKS) + +if(ARROW_WITH_UCX) + add_subdirectory(transport/ucx) +endif() + +if(ARROW_FLIGHT_SQL) + add_subdirectory(sql) + + if(ARROW_BUILD_INTEGRATION) + add_subdirectory(integration_tests) + endif() +endif() diff --git a/cpp/src/arrow/flight/integration_tests/CMakeLists.txt b/cpp/src/arrow/flight/integration_tests/CMakeLists.txt index c7a8c4fe459..66a021b4b59 100644 --- a/cpp/src/arrow/flight/integration_tests/CMakeLists.txt +++ b/cpp/src/arrow/flight/integration_tests/CMakeLists.txt @@ -17,27 +17,26 @@ add_custom_target(arrow_flight_integration_tests) -if(ARROW_FLIGHT_TEST_LINKAGE STREQUAL "static") - set(ARROW_FLIGHT_TEST_LINK_LIBS - arrow_flight_static - arrow_flight_testing_static - arrow_flight_sql_static - ${ARROW_FLIGHT_STATIC_LINK_LIBS} - ${ARROW_FLIGHT_TEST_LINK_LIBS}) +if(ARROW_FLIGHT_TEST_LINKAGE STREQUAL "static" AND ARROW_BUILD_STATIC) + set(ARROW_FLIGHT_INTEGRATION_TEST_LINK_LIBS arrow_flight_sql_static) else() - set(ARROW_FLIGHT_TEST_LINK_LIBS arrow_flight_shared arrow_flight_testing_shared - arrow_flight_sql_shared ${ARROW_FLIGHT_TEST_LINK_LIBS}) + set(ARROW_FLIGHT_INTEGRATION_TEST_LINK_LIBS arrow_flight_sql_shared) endif() +list(APPEND + ARROW_FLIGHT_INTEGRATION_TEST_LINK_LIBS + ${ARROW_FLIGHT_TEST_LINK_LIBS} + ${GFLAGS_LIBRARIES} + GTest::gtest) add_executable(flight-test-integration-server test_integration_server.cc test_integration.cc) -target_link_libraries(flight-test-integration-server ${ARROW_FLIGHT_TEST_LINK_LIBS} - ${GFLAGS_LIBRARIES} GTest::gtest) +target_link_libraries(flight-test-integration-server + ${ARROW_FLIGHT_INTEGRATION_TEST_LINK_LIBS}) add_executable(flight-test-integration-client test_integration_client.cc test_integration.cc) -target_link_libraries(flight-test-integration-client ${ARROW_FLIGHT_TEST_LINK_LIBS} - ${GFLAGS_LIBRARIES} GTest::gtest) +target_link_libraries(flight-test-integration-client + ${ARROW_FLIGHT_INTEGRATION_TEST_LINK_LIBS}) add_dependencies(arrow-integration flight-test-integration-client flight-test-integration-server) diff --git a/cpp/src/arrow/flight/sql/CMakeLists.txt b/cpp/src/arrow/flight/sql/CMakeLists.txt index 6a3c9222105..1f5a72b50da 100644 --- a/cpp/src/arrow/flight/sql/CMakeLists.txt +++ b/cpp/src/arrow/flight/sql/CMakeLists.txt @@ -63,13 +63,12 @@ add_arrow_lib(arrow_flight_sql PRIVATE_INCLUDES "${Protobuf_INCLUDE_DIRS}") -if(ARROW_FLIGHT_TEST_LINKAGE STREQUAL "static") - set(ARROW_FLIGHT_SQL_TEST_LINK_LIBS arrow_flight_sql_static - ${ARROW_FLIGHT_TEST_LINK_LIBS}) +if(ARROW_FLIGHT_TEST_LINKAGE STREQUAL "static" AND ARROW_BUILD_STATIC) + set(ARROW_FLIGHT_SQL_TEST_LINK_LIBS arrow_flight_sql_static) else() - set(ARROW_FLIGHT_SQL_TEST_LINK_LIBS arrow_flight_sql_shared - ${ARROW_FLIGHT_TEST_LINK_LIBS}) + set(ARROW_FLIGHT_SQL_TEST_LINK_LIBS arrow_flight_sql_shared) endif() +list(APPEND ARROW_FLIGHT_SQL_TEST_LINK_LIBS ${ARROW_FLIGHT_TEST_LINK_LIBS}) # Build test server for unit tests if(ARROW_BUILD_TESTS OR ARROW_BUILD_EXAMPLES) diff --git a/dev/release/verify-release-candidate.bat b/dev/release/verify-release-candidate.bat index 30bb69a2253..396f0475e29 100644 --- a/dev/release/verify-release-candidate.bat +++ b/dev/release/verify-release-candidate.bat @@ -69,10 +69,6 @@ call conda create --no-shortcuts -c conda-forge -f -q -y -p %_VERIFICATION_CONDA call activate %_VERIFICATION_CONDA_ENV% || exit /B 1 -@rem With MSVC we always bundle a more recent GTest which causes a link conflict -@rem for more context, see https://issues.apache.org/jira/browse/ARROW-15378 -call conda remove -y gtest gmock || exit /B 1 - set GENERATOR=Visual Studio 16 2019 set ARCHITECTURE=x64 set CONFIGURATION=release diff --git a/dev/release/verify-release-candidate.sh b/dev/release/verify-release-candidate.sh index e9bc6b02b2b..a512449aea5 100755 --- a/dev/release/verify-release-candidate.sh +++ b/dev/release/verify-release-candidate.sh @@ -632,9 +632,10 @@ test_and_install_cpp() { LD_LIBRARY_PATH=$PWD/release:$LD_LIBRARY_PATH PYTHONPATH=$pythonpath ctest \ --exclude-regex "plasma-serialization_tests" \ - -j$NPROC \ + --label-regex unittest \ --output-on-failure \ - -L unittest + --parallel $NPROC \ + --timeout 300 popd }