From 67e002b1dba4be2f7a94b52661414e1cd2902ab9 Mon Sep 17 00:00:00 2001 From: Will Jones Date: Sat, 22 Jan 2022 11:37:36 -0800 Subject: [PATCH 1/2] Provide various fixes --- cpp/cmake_modules/BuildUtils.cmake | 2 +- cpp/cmake_modules/FindThrift.cmake | 12 +++++-- cpp/cmake_modules/ThirdpartyToolchain.cmake | 5 ++- cpp/cmake_modules/Usevcpkg.cmake | 3 ++ docs/source/developers/cpp/windows.rst | 39 ++++++--------------- 5 files changed, 29 insertions(+), 32 deletions(-) diff --git a/cpp/cmake_modules/BuildUtils.cmake b/cpp/cmake_modules/BuildUtils.cmake index 391c43e0ac5..74091ba61d5 100644 --- a/cpp/cmake_modules/BuildUtils.cmake +++ b/cpp/cmake_modules/BuildUtils.cmake @@ -370,7 +370,7 @@ function(ADD_ARROW_LIB LIB_NAME) target_link_libraries(${LIB_NAME}_shared LINK_PUBLIC - "$" + ${ARG_SHARED_LINK_LIBS} "$" LINK_PRIVATE ${ARG_SHARED_PRIVATE_LINK_LIBS}) diff --git a/cpp/cmake_modules/FindThrift.cmake b/cpp/cmake_modules/FindThrift.cmake index 750d8ce8341..dd54fd10f9a 100644 --- a/cpp/cmake_modules/FindThrift.cmake +++ b/cpp/cmake_modules/FindThrift.cmake @@ -46,9 +46,17 @@ endfunction(EXTRACT_THRIFT_VERSION) if(MSVC_TOOLCHAIN AND NOT DEFINED THRIFT_MSVC_LIB_SUFFIX) if(NOT ARROW_THRIFT_USE_SHARED) if(ARROW_USE_STATIC_CRT) - set(THRIFT_MSVC_LIB_SUFFIX "mt") + if("${CMAKE_BUILD_TYPE}" STREQUAL "DEBUG") + set(THRIFT_MSVC_LIB_SUFFIX "mtd") + else() + set(THRIFT_MSVC_LIB_SUFFIX "mt") + endif() else() - set(THRIFT_MSVC_LIB_SUFFIX "md") + if("${CMAKE_BUILD_TYPE}" STREQUAL "DEBUG") + set(THRIFT_MSVC_LIB_SUFFIX "mdd") + else() + set(THRIFT_MSVC_LIB_SUFFIX "md") + endif() endif() endif() endif() diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index c35b9e3102a..0872ad9cb27 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -79,7 +79,7 @@ set(ARROW_THIRDPARTY_DEPENDENCIES # supported when building with MSVC because of the way that # conda-forge packages have 4 variants of the libraries packaged # together -if(MSVC AND "${GTest_SOURCE}" STREQUAL "") +if(MSVC AND "${GTest_SOURCE}" STREQUAL "" AND "${ARROW_DEPENDENCY_SOURCE}" STREQUAL "CONDA") set(GTest_SOURCE "BUNDLED") endif() @@ -228,6 +228,7 @@ macro(resolve_dependency DEPENDENCY_NAME) list(APPEND FIND_PACKAGE_ARGUMENTS CONFIG) endif() if(${DEPENDENCY_NAME}_SOURCE STREQUAL "AUTO") + message("Finding auto package with args: ${FIND_PACKAGE_ARGUMENTS}") find_package(${FIND_PACKAGE_ARGUMENTS}) if(${${PACKAGE_NAME}_FOUND}) set(${DEPENDENCY_NAME}_SOURCE "SYSTEM") @@ -236,8 +237,10 @@ macro(resolve_dependency DEPENDENCY_NAME) set(${DEPENDENCY_NAME}_SOURCE "BUNDLED") endif() elseif(${DEPENDENCY_NAME}_SOURCE STREQUAL "BUNDLED") + message("Finding bundled package with args: ${FIND_PACKAGE_ARGUMENTS}") build_dependency(${DEPENDENCY_NAME}) elseif(${DEPENDENCY_NAME}_SOURCE STREQUAL "SYSTEM") + message("Finding package with args: ${FIND_PACKAGE_ARGUMENTS}") find_package(${FIND_PACKAGE_ARGUMENTS} REQUIRED) endif() if(${DEPENDENCY_NAME}_SOURCE STREQUAL "SYSTEM" AND ARG_IS_RUNTIME_DEPENDENCY) diff --git a/cpp/cmake_modules/Usevcpkg.cmake b/cpp/cmake_modules/Usevcpkg.cmake index 06ac4dd075d..cef31f16b44 100644 --- a/cpp/cmake_modules/Usevcpkg.cmake +++ b/cpp/cmake_modules/Usevcpkg.cmake @@ -235,6 +235,9 @@ set(BROTLI_ROOT set(LZ4_ROOT "${ARROW_VCPKG_PREFIX}" CACHE STRING "") +set(GTEST_ROOT + "${ARROW_VCPKG_PREFIX}" + CACHE STRING "") if(CMAKE_HOST_WIN32) set(LZ4_MSVC_LIB_PREFIX diff --git a/docs/source/developers/cpp/windows.rst b/docs/source/developers/cpp/windows.rst index b2c0c238ff2..401b9b4fec0 100644 --- a/docs/source/developers/cpp/windows.rst +++ b/docs/source/developers/cpp/windows.rst @@ -45,6 +45,12 @@ For Visual Studio 2019, the script is: "C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\Common7\Tools\VsDevCmd.bat" -arch=amd64 +For Visual Studio 2022, the script is: + +.. code-block:: shell + + "C:\Program Files\Microsoft Visual Studio\2022\Community\Common7\Tools\VsDevCmd.bat" -arch=amd64 + One can configure a console emulator like `cmder `_ to automatically launch this when starting a new development console. @@ -178,14 +184,11 @@ For newer versions of Visual Studio, specify the generator ``Visual Studio 16 2019`` or see ``cmake --help`` for available generators. -Building with Ninja and clcache -=============================== +Building with Ninja +=================== The `Ninja `_ build system offers better build -parallelization, and the optional `clcache -`_ compiler cache keeps track of -past compilations to avoid running them over and over again (in a way similar -to the Unix-specific ``ccache``). +parallelization. Newer versions of Visual Studio include Ninja. To see if your Visual Studio includes Ninja, run the initialization command shown @@ -193,19 +196,15 @@ includes Ninja, run the initialization command shown run ``ninja --version``. If Ninja is not included in your version of Visual Studio, and you are using -conda, activate your conda environment and install Ninja and clcache: +conda, activate your conda environment and install Ninja: .. code-block:: shell activate arrow-dev conda install -c conda-forge ninja - pip install git+https://github.com/frerich/clcache.git If you are not using conda, -`install Ninja from another source `_ -and optionally -`install clcache from another source `_ -. +`install Ninja from another source `_. After installation is complete, change working directory in ``cmd.exe`` to the root directory of Arrow and do an out of source build by generating Ninja files: @@ -216,25 +215,10 @@ do an out of source build by generating Ninja files: mkdir build cd build cmake -G "Ninja" ^ - -DCMAKE_C_COMPILER=clcache ^ - -DCMAKE_CXX_COMPILER=clcache ^ -DARROW_BUILD_TESTS=ON ^ -DGTest_SOURCE=BUNDLED .. cmake --build . --config Release -Setting ``CMAKE_C_COMPILER`` and ``CMAKE_CXX_COMPILER`` in the command line -of ``cmake`` is the preferred method of using ``clcache``. Alternatively, you -can set ``CC`` and ``CXX`` environment variables before calling ``cmake``: - -.. code-block:: shell - - ... - set CC=clcache - set CXX=clcache - cmake -G "Ninja" ^ - ... - - Building with NMake =================== @@ -414,7 +398,6 @@ tests can be made with there individual make targets). SET JOB=Static_Crt_Build SET GENERATOR=Ninja SET APPVEYOR_BUILD_WORKER_IMAGE=Visual Studio 2017 - SET USE_CLCACHE=false SET ARROW_BUILD_GANDIVA=OFF SET ARROW_LLVM_VERSION=8.0.* SET PYTHON=3.9 From 762623a356a8adbfbb40750924f9414e1acf238b Mon Sep 17 00:00:00 2001 From: Will Jones Date: Tue, 25 Jan 2022 13:14:32 -0800 Subject: [PATCH 2/2] Better python library handling; add clcache back --- cpp/cmake_modules/BuildUtils.cmake | 2 +- cpp/cmake_modules/FindPython3Alt.cmake | 17 ++++++++++++- docs/source/developers/cpp/windows.rst | 33 ++++++++++++++++++++------ 3 files changed, 43 insertions(+), 9 deletions(-) diff --git a/cpp/cmake_modules/BuildUtils.cmake b/cpp/cmake_modules/BuildUtils.cmake index 74091ba61d5..391c43e0ac5 100644 --- a/cpp/cmake_modules/BuildUtils.cmake +++ b/cpp/cmake_modules/BuildUtils.cmake @@ -370,7 +370,7 @@ function(ADD_ARROW_LIB LIB_NAME) target_link_libraries(${LIB_NAME}_shared LINK_PUBLIC - ${ARG_SHARED_LINK_LIBS} + "$" "$" LINK_PRIVATE ${ARG_SHARED_PRIVATE_LINK_LIBS}) diff --git a/cpp/cmake_modules/FindPython3Alt.cmake b/cpp/cmake_modules/FindPython3Alt.cmake index ab91c7be052..80b69ccfab7 100644 --- a/cpp/cmake_modules/FindPython3Alt.cmake +++ b/cpp/cmake_modules/FindPython3Alt.cmake @@ -65,9 +65,24 @@ endif() set(PYTHON_EXECUTABLE ${Python3_EXECUTABLE}) set(PYTHON_INCLUDE_DIRS ${Python3_INCLUDE_DIRS}) -set(PYTHON_LIBRARIES ${Python3_LIBRARIES}) set(PYTHON_OTHER_LIBS) +# On Windows, debug builds of Python will have format: +# "optimized python3x.lib debug python3xd.lib" +list(LENGTH Python3_LIBRARIES NUM_LIBS) +if(NUM_LIBS EQUAL 4) + if("${CMAKE_BUILD_TYPE}" STREQUAL "DEBUG") + list(SUBLIST Python3_LIBRARIES 3 1 PYTHON_LIBRARIES) + else() + list(SUBLIST Python3_LIBRARIES 1 1 PYTHON_LIBRARIES) + endif() +else() + set(PYTHON_LIBRARIES ${Python3_LIBRARIES}) +endif() +unset(NUM_LIBS) + +message("Python libraries: ${PYTHON_LIBRARIES}") + get_target_property(NUMPY_INCLUDE_DIRS Python3::NumPy INTERFACE_INCLUDE_DIRECTORIES) # CMake's python3_add_library() doesn't apply the required extension suffix, diff --git a/docs/source/developers/cpp/windows.rst b/docs/source/developers/cpp/windows.rst index 401b9b4fec0..6494354e96c 100644 --- a/docs/source/developers/cpp/windows.rst +++ b/docs/source/developers/cpp/windows.rst @@ -184,11 +184,14 @@ For newer versions of Visual Studio, specify the generator ``Visual Studio 16 2019`` or see ``cmake --help`` for available generators. -Building with Ninja -=================== +Building with Ninja and clcache +=============================== The `Ninja `_ build system offers better build -parallelization. +parallelization, and the optional `clcache +`_ compiler cache keeps track of +past compilations to avoid running them over and over again (in a way similar +to the Unix-specific ``ccache``). Newer versions of Visual Studio include Ninja. To see if your Visual Studio includes Ninja, run the initialization command shown @@ -196,15 +199,17 @@ includes Ninja, run the initialization command shown run ``ninja --version``. If Ninja is not included in your version of Visual Studio, and you are using -conda, activate your conda environment and install Ninja: +conda, activate your conda environment and install Ninja and clcache: .. code-block:: shell - activate arrow-dev conda install -c conda-forge ninja - + pip install git+https://github.com/Nuitka/clcache.git If you are not using conda, -`install Ninja from another source `_. +`install Ninja from another source `_ +and optionally +`install clcache from another source `_ +. After installation is complete, change working directory in ``cmd.exe`` to the root directory of Arrow and do an out of source build by generating Ninja files: @@ -215,10 +220,24 @@ do an out of source build by generating Ninja files: mkdir build cd build cmake -G "Ninja" ^ + -DCMAKE_C_COMPILER=clcache ^ + -DCMAKE_CXX_COMPILER=clcache ^ -DARROW_BUILD_TESTS=ON ^ -DGTest_SOURCE=BUNDLED .. cmake --build . --config Release +Setting ``CMAKE_C_COMPILER`` and ``CMAKE_CXX_COMPILER`` in the command line +of ``cmake`` is the preferred method of using ``clcache``. Alternatively, you +can set ``CC`` and ``CXX`` environment variables before calling ``cmake``: + +.. code-block:: shell + ... + set CC=clcache + set CXX=clcache + cmake -G "Ninja" ^ + ... + + Building with NMake ===================