From 88e11fcfb9b20d58d184ea5648a877ac2b491be4 Mon Sep 17 00:00:00 2001 From: "Uwe L. Korn" Date: Mon, 18 Feb 2019 19:01:43 +0100 Subject: [PATCH] ARROW-4611: [C++] Rework CMake logic --- .travis.yml | 107 +- LICENSE.txt | 10 + NOTICE.txt | 4 + appveyor.yml | 8 +- ci/appveyor-build.bat | 2 - ci/appveyor-cpp-build-mingw.bat | 7 +- ci/appveyor-cpp-build.bat | 4 +- ci/appveyor-cpp-setup-mingw.bat | 2 + ci/appveyor-cpp-test-cmake-script.bat | 193 -- ci/conda_env_cpp.yml | 4 + ci/conda_env_unix.yml | 3 + ci/cpp-msvc-build-main.bat | 10 +- ci/docker_build_cpp.sh | 11 + ci/travis_before_script_cpp.sh | 16 +- ci/travis_env_common.sh | 7 - ci/travis_install_linux.sh | 33 + ci/travis_install_osx_sdk.sh | 40 + ci/travis_install_toolchain.sh | 15 +- ci/travis_script_cpp.sh | 7 +- ci/travis_script_integration.sh | 14 +- ci/travis_script_plasma_java_client.sh | 4 + ci/travis_script_python.sh | 19 +- cpp/CMakeLists.txt | 91 +- cpp/Dockerfile | 2 +- cpp/Dockerfile.alpine | 12 +- cpp/Dockerfile.debian-testing | 76 + cpp/Dockerfile.fedora | 76 + cpp/Dockerfile.ubuntu-bionic | 88 + cpp/Dockerfile.ubuntu-xenial | 78 + cpp/cmake_modules/BuildUtils.cmake | 24 + cpp/cmake_modules/FindBrotli.cmake | 196 +- cpp/cmake_modules/FindBz2.cmake | 57 - cpp/cmake_modules/FindDoubleConversion.cmake | 42 + cpp/cmake_modules/FindFlatbuffers.cmake | 95 - cpp/cmake_modules/FindGBenchmark.cmake | 88 - cpp/cmake_modules/FindGFlags.cmake | 142 - cpp/cmake_modules/FindGLOG.cmake | 102 +- cpp/cmake_modules/FindGTest.cmake | 441 +-- cpp/cmake_modules/FindLLVM.cmake | 1 + cpp/cmake_modules/FindLz4.cmake | 65 +- cpp/cmake_modules/FindProtobuf.cmake | 102 - cpp/cmake_modules/FindRE2.cmake | 119 +- cpp/cmake_modules/FindRapidJSONAlt.cmake | 28 + cpp/cmake_modules/FindSnappy.cmake | 94 - cpp/cmake_modules/FindSnappyAlt.cmake | 42 + cpp/cmake_modules/FindThrift.cmake | 108 +- cpp/cmake_modules/FindZLIB.cmake | 103 - cpp/cmake_modules/FindZSTD.cmake | 65 +- cpp/cmake_modules/Findc-ares.cmake | 109 - cpp/cmake_modules/Findc-aresAlt.cmake | 45 + cpp/cmake_modules/FindgRPC.cmake | 101 - cpp/cmake_modules/FindgRPCAlt.cmake | 114 + cpp/cmake_modules/FindgflagsAlt.cmake | 43 + cpp/cmake_modules/SetupCxxFlags.cmake | 5 +- cpp/cmake_modules/ThirdpartyToolchain.cmake | 2760 +++++++++-------- cpp/src/arrow/CMakeLists.txt | 4 +- cpp/src/arrow/array-struct-test.cc | 6 +- cpp/src/arrow/buffer-test.cc | 4 +- .../compute/kernels/aggregate-benchmark.cc | 18 +- cpp/src/arrow/dbi/hiveserver2/CMakeLists.txt | 2 +- cpp/src/arrow/flight/CMakeLists.txt | 42 +- cpp/src/arrow/flight/client.cc | 5 + cpp/src/arrow/flight/customize_protobuf.h | 9 + cpp/src/arrow/flight/internal.cc | 5 + .../arrow/flight/serialization-internal.cc | 7 + cpp/src/arrow/flight/server.cc | 5 + cpp/src/arrow/flight/test-util.cc | 4 +- cpp/src/arrow/gpu/CMakeLists.txt | 2 +- cpp/src/arrow/io/hdfs-test.cc | 8 +- cpp/src/arrow/io/memory-benchmark.cc | 10 +- cpp/src/arrow/io/memory-test.cc | 4 +- cpp/src/arrow/ipc/CMakeLists.txt | 15 +- cpp/src/arrow/ipc/feather-test.cc | 8 +- cpp/src/arrow/python/CMakeLists.txt | 2 +- cpp/src/arrow/python/util/CMakeLists.txt | 4 +- cpp/src/arrow/symbols.map | 3 + cpp/src/arrow/testing/gtest_common.h | 12 +- cpp/src/arrow/testing/gtest_util.cc | 12 +- cpp/src/arrow/testing/gtest_util.h | 6 +- cpp/src/arrow/util/CMakeLists.txt | 13 - cpp/src/arrow/util/compression_lz4.cc | 8 +- cpp/src/arrow/util/config.h.cmake | 19 + cpp/src/arrow/util/parsing.h | 6 + cpp/src/gandiva/CMakeLists.txt | 11 +- cpp/src/gandiva/decimal_type_util_test.cc | 2 +- cpp/src/gandiva/jni/CMakeLists.txt | 6 +- cpp/src/gandiva/precompiled/CMakeLists.txt | 2 +- .../gandiva/precompiled/decimal_ops_test.cc | 2 +- cpp/src/gandiva/tests/decimal_single_test.cc | 6 +- cpp/src/gandiva/tests/generate_data.h | 5 +- cpp/src/parquet/CMakeLists.txt | 13 +- .../parquet/arrow/arrow-reader-writer-test.cc | 6 +- cpp/src/parquet/arrow/test-util.h | 5 +- cpp/src/parquet/arrow/writer.cc | 4 +- cpp/src/plasma/CMakeLists.txt | 32 +- cpp/thirdparty/versions.txt | 8 +- docker-compose.yml | 78 +- python/manylinux1/Dockerfile-x86_64_base | 12 + python/manylinux1/build_arrow.sh | 6 +- .../scripts/build_double_conversion.sh | 29 + python/manylinux1/scripts/build_gflags.sh | 38 + python/manylinux1/scripts/build_rapidjson.sh | 37 + python/manylinux1/scripts/build_re2.sh | 32 + python/manylinux1/scripts/build_snappy.sh | 9 +- python/manylinux1/scripts/build_zstd.sh | 14 +- python/requirements-wheel.txt | 1 + 106 files changed, 3459 insertions(+), 3176 deletions(-) delete mode 100644 ci/appveyor-cpp-test-cmake-script.bat create mode 100755 ci/travis_install_osx_sdk.sh create mode 100644 cpp/Dockerfile.debian-testing create mode 100644 cpp/Dockerfile.fedora create mode 100644 cpp/Dockerfile.ubuntu-bionic create mode 100644 cpp/Dockerfile.ubuntu-xenial delete mode 100644 cpp/cmake_modules/FindBz2.cmake create mode 100644 cpp/cmake_modules/FindDoubleConversion.cmake delete mode 100644 cpp/cmake_modules/FindFlatbuffers.cmake delete mode 100644 cpp/cmake_modules/FindGBenchmark.cmake delete mode 100644 cpp/cmake_modules/FindGFlags.cmake delete mode 100644 cpp/cmake_modules/FindProtobuf.cmake create mode 100644 cpp/cmake_modules/FindRapidJSONAlt.cmake delete mode 100644 cpp/cmake_modules/FindSnappy.cmake create mode 100644 cpp/cmake_modules/FindSnappyAlt.cmake delete mode 100644 cpp/cmake_modules/FindZLIB.cmake delete mode 100644 cpp/cmake_modules/Findc-ares.cmake create mode 100644 cpp/cmake_modules/Findc-aresAlt.cmake delete mode 100644 cpp/cmake_modules/FindgRPC.cmake create mode 100644 cpp/cmake_modules/FindgRPCAlt.cmake create mode 100644 cpp/cmake_modules/FindgflagsAlt.cmake create mode 100644 cpp/src/arrow/util/config.h.cmake create mode 100755 python/manylinux1/scripts/build_double_conversion.sh create mode 100755 python/manylinux1/scripts/build_gflags.sh create mode 100755 python/manylinux1/scripts/build_rapidjson.sh create mode 100755 python/manylinux1/scripts/build_re2.sh diff --git a/.travis.yml b/.travis.yml index de9832a8f66..ea4d609988a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -56,16 +56,15 @@ matrix: - $TRAVIS_BUILD_DIR/ci/travis_install_clang_tools.sh script: - $TRAVIS_BUILD_DIR/ci/travis_lint.sh - - name: "C++ unit tests w/ Valgrind, clang 7.0" + - name: "C++ unit tests w/ clang 7.0, system packages" os: linux env: - - ARROW_TRAVIS_VALGRIND=1 - - ARROW_TRAVIS_USE_TOOLCHAIN=1 - - ARROW_TRAVIS_FLIGHT=1 + - ARROW_TRAVIS_USE_SYSTEM=1 - ARROW_TRAVIS_PLASMA=1 - ARROW_TRAVIS_ORC=1 - ARROW_TRAVIS_PARQUET=1 - ARROW_TRAVIS_GANDIVA=1 + - ARROW_TRAVIS_VERBOSE=1 - ARROW_TRAVIS_USE_SYSTEM_JAVA=1 - ARROW_BUILD_WARNING_LEVEL=CHECKIN - CC="clang-7" @@ -80,56 +79,25 @@ matrix: script: - $TRAVIS_BUILD_DIR/ci/travis_script_cpp.sh || travis_terminate 1 # Separating Valgrind and C++ coverage makes individual jobs shorter - - name: "C++ unit tests w/ gcc 5.4, coverage" + - name: "C++ unit tests w/ conda-forge toolchain, coverage" compiler: gcc language: cpp os: linux jdk: openjdk8 env: + - ARROW_BUILD_WARNING_LEVEL=CHECKIN - ARROW_TRAVIS_COVERAGE=1 - - ARROW_TRAVIS_USE_TOOLCHAIN=1 - ARROW_TRAVIS_FLIGHT=1 - - ARROW_TRAVIS_PLASMA=1 - - ARROW_TRAVIS_ORC=1 - - ARROW_TRAVIS_PARQUET=1 - ARROW_TRAVIS_GANDIVA=1 - ARROW_TRAVIS_GANDIVA_JAVA=1 - - ARROW_TRAVIS_USE_SYSTEM_JAVA=1 - - ARROW_BUILD_WARNING_LEVEL=CHECKIN - before_script: - - if [ $ARROW_CI_CPP_AFFECTED != "1" ] && [ $ARROW_CI_JAVA_AFFECTED != "1" ]; then exit; fi - - $TRAVIS_BUILD_DIR/ci/travis_install_clang_tools.sh - - $TRAVIS_BUILD_DIR/ci/travis_install_linux.sh - # If either C++ or Python changed, we must install the C++ libraries - - git submodule update --init - - $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh - script: - - $TRAVIS_BUILD_DIR/ci/travis_script_cpp.sh || travis_terminate 1 - - $TRAVIS_BUILD_DIR/ci/travis_script_gandiva_java.sh || travis_terminate 1 - - $TRAVIS_BUILD_DIR/ci/travis_upload_cpp_coverage.sh || travis_terminate 1 - - name: "C++ unit tests w/ gcc 4.8, trusty" - dist: trusty - compiler: gcc - language: cpp - os: linux - jdk: openjdk8 - env: - - ARROW_TRAVIS_USE_TOOLCHAIN=1 - - ARROW_TRAVIS_PLASMA=1 - ARROW_TRAVIS_ORC=1 - ARROW_TRAVIS_PARQUET=1 - - ARROW_TRAVIS_GANDIVA=1 - - ARROW_TRAVIS_GANDIVA_JAVA=1 - - ARROW_BUILD_WARNING_LEVEL=CHECKIN - before_install: - - ulimit -c unlimited -S - - | - if [ $TRAVIS_OS_NAME == "linux" ]; then - sudo bash -c "echo -e 'Acquire::Retries 10; Acquire::http::Timeout \"20\";' > /etc/apt/apt.conf.d/99-travis-retry" - sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test - sudo apt-get update -qq - fi - - eval `python $TRAVIS_BUILD_DIR/ci/detect-changes.py` + - ARROW_TRAVIS_PLASMA=1 + - ARROW_TRAVIS_USE_SYSTEM_JAVA=1 + - ARROW_TRAVIS_USE_TOOLCHAIN=1 + # TODO: This fails in re2 code + # - ARROW_TRAVIS_VALGRIND=1 + - ARROW_TRAVIS_VERBOSE=1 before_script: - if [ $ARROW_CI_CPP_AFFECTED != "1" ] && [ $ARROW_CI_JAVA_AFFECTED != "1" ]; then exit; fi - $TRAVIS_BUILD_DIR/ci/travis_install_clang_tools.sh @@ -140,21 +108,23 @@ matrix: script: - $TRAVIS_BUILD_DIR/ci/travis_script_cpp.sh || travis_terminate 1 - $TRAVIS_BUILD_DIR/ci/travis_script_gandiva_java.sh || travis_terminate 1 - - name: "Python 2.7 and 3.6 unit tests w/ Valgrind, gcc 5.4, coverage" + - $TRAVIS_BUILD_DIR/ci/travis_upload_cpp_coverage.sh || travis_terminate 1 + - name: "Python 2.7 and 3.6 unit tests w/ Valgrind, conda-forge toolchain, coverage" compiler: gcc language: cpp os: linux jdk: openjdk8 env: # Valgrind is needed for the Plasma store tests - - ARROW_TRAVIS_VALGRIND=1 - - ARROW_TRAVIS_USE_TOOLCHAIN=1 + - ARROW_BUILD_WARNING_LEVEL=CHECKIN - ARROW_TRAVIS_COVERAGE=1 + - ARROW_TRAVIS_FLIGHT=1 + - ARROW_TRAVIS_OPTIONAL_INSTALL=1 - ARROW_TRAVIS_PYTHON_DOCS=1 - ARROW_TRAVIS_PYTHON_JVM=1 - - ARROW_TRAVIS_OPTIONAL_INSTALL=1 - - ARROW_BUILD_WARNING_LEVEL=CHECKIN - ARROW_TRAVIS_USE_SYSTEM_JAVA=1 + - ARROW_TRAVIS_USE_TOOLCHAIN=1 + - ARROW_TRAVIS_VALGRIND=1 # TODO(wesm): Run the benchmarks outside of Travis # - ARROW_TRAVIS_PYTHON_BENCHMARKS=1 before_script: @@ -172,10 +142,9 @@ matrix: - export PLASMA_VALGRIND=1 - $TRAVIS_BUILD_DIR/ci/travis_script_python.sh 3.6 || travis_terminate 1 - $TRAVIS_BUILD_DIR/ci/travis_upload_cpp_coverage.sh - - name: "[OS X] C++ w/ XCode 8.3" + - name: "[OS X] C++ w/ XCode 9.3" compiler: clang - language: cpp - osx_image: xcode8.3 + osx_image: xcode9.3 os: osx cache: addons: @@ -185,17 +154,13 @@ matrix: - ARROW_TRAVIS_FLIGHT=1 - ARROW_TRAVIS_ORC=1 - ARROW_TRAVIS_PARQUET=1 - - ARROW_TRAVIS_GANDIVA=1 - - ARROW_TRAVIS_GANDIVA_JAVA=1 + # TODO(ARROW-4763): llvm and llvmdev packages are in conflict: + # https://github.com/conda-forge/llvmdev-feedstock/issues/60 + # - ARROW_TRAVIS_GANDIVA=1 + # - ARROW_TRAVIS_GANDIVA_JAVA=1 - ARROW_TRAVIS_OPTIONAL_INSTALL=1 - - ARROW_TRAVIS_VERBOSE=0 + - ARROW_TRAVIS_VERBOSE=1 - ARROW_BUILD_WARNING_LEVEL=CHECKIN - # ARROW-3803: The Xcode 8.3 image has Boost libraries in /usr/local/lib - # which can get loaded before the toolchain Boost libraries. These seem to - # get loaded even though we are modifying LD_LIBRARY_PATH. We build our own - # Boost and statically link to get around the issue until this can be - # investigated further - - ARROW_TRAVIS_VENDORED_BOOST=1 before_script: - if [ $ARROW_CI_CPP_AFFECTED != "1" ] && [ $ARROW_CI_JAVA_AFFECTED != "1" ]; then exit; fi # If either C++ or Python changed, we must install the C++ libraries @@ -203,23 +168,24 @@ matrix: - $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh script: - $TRAVIS_BUILD_DIR/ci/travis_script_cpp.sh || travis_terminate 1 - - $TRAVIS_BUILD_DIR/ci/travis_script_gandiva_java.sh - - name: "[OS X] Python w/ XCode 7.3" + # Disabled because of ARROW-4763 + # - $TRAVIS_BUILD_DIR/ci/travis_script_gandiva_java.sh + - name: "[OS X] Python w/ XCode 9.3" compiler: clang - language: cpp - osx_image: xcode7.3 + osx_image: xcode9.3 os: osx cache: addons: env: + - ARROW_TRAVIS_PLASMA=1 - ARROW_TRAVIS_USE_TOOLCHAIN=1 - ARROW_BUILD_WARNING_LEVEL=CHECKIN - ARROW_TRAVIS_OPTIONAL_INSTALL=1 + - ARROW_TRAVIS_VERBOSE=1 - MACOSX_DEPLOYMENT_TARGET="10.9" before_script: script: - if [ $ARROW_CI_PYTHON_AFFECTED != "1" ]; then exit; fi - - $TRAVIS_BUILD_DIR/ci/travis_install_toolchain.sh || travis_terminate 1 - $TRAVIS_BUILD_DIR/ci/travis_script_python.sh 2.7 || travis_terminate 1 - $TRAVIS_BUILD_DIR/ci/travis_script_python.sh 3.6 - name: "[manylinux1] Python" @@ -256,7 +222,7 @@ matrix: - if [ $ARROW_CI_JAVA_AFFECTED != "1" ]; then exit; fi script: - $TRAVIS_BUILD_DIR/ci/travis_script_java.sh - - name: "Integration w/ OpenJDK 8, clang 7" + - name: "Integration w/ OpenJDK 8, conda-forge toolchain" language: java os: linux env: ARROW_TEST_GROUP=integration @@ -264,8 +230,7 @@ matrix: env: - ARROW_TRAVIS_PLASMA=1 - ARROW_TRAVIS_PLASMA_JAVA_CLIENT=1 - - CC="clang-7" - - CXX="clang++-7" + - ARROW_TRAVIS_USE_TOOLCHAIN=1 before_script: - if [ $ARROW_CI_INTEGRATION_AFFECTED != "1" ]; then exit; fi - $TRAVIS_BUILD_DIR/ci/travis_install_clang_tools.sh @@ -275,7 +240,8 @@ matrix: - $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh script: - $TRAVIS_BUILD_DIR/ci/travis_script_integration.sh - - $TRAVIS_BUILD_DIR/ci/travis_script_plasma_java_client.sh + # TODO(ARROW-4764): Using system java and plasma built with conda doesn't work + # - $TRAVIS_BUILD_DIR/ci/travis_script_plasma_java_client.sh - name: "NodeJS" language: node_js os: linux @@ -289,7 +255,6 @@ matrix: - $TRAVIS_BUILD_DIR/ci/travis_script_js.sh - name: "C++ & GLib & Ruby w/ gcc 5.4" compiler: gcc - language: cpp os: linux env: - ARROW_TRAVIS_ORC=1 @@ -369,6 +334,7 @@ matrix: dist: trusty env: - ARROW_TRAVIS_PARQUET=1 + - ARROW_TRAVIS_USE_SYSTEM=1 before_install: # Have to copy-paste this here because of how R's build steps work - eval `python $TRAVIS_BUILD_DIR/ci/detect-changes.py` @@ -379,6 +345,7 @@ matrix: sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test sudo apt-get update -qq fi + - $TRAVIS_BUILD_DIR/ci/travis_install_clang_tools.sh - $TRAVIS_BUILD_DIR/ci/travis_install_linux.sh - $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh --only-library - export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$TRAVIS_BUILD_DIR/cpp-install/lib diff --git a/LICENSE.txt b/LICENSE.txt index 582daa48a99..1605d4cb74f 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -837,3 +837,13 @@ This project includes code from the manylinux project. Copyright: 2016 manylinux Homepage: https://github.com/pypa/manylinux License: The MIT License (MIT) + +-------------------------------------------------------------------------------- + +This project include code from CMake. + +* cpp/cmake_modules/FindGTest.cmake is based on code from CMake. + +Copyright: Copyright 2000-2019 Kitware, Inc. and Contributors +Homepage: https://gitlab.kitware.com/cmake/cmake +License: 3-clause BSD diff --git a/NOTICE.txt b/NOTICE.txt index b4664a5003b..02765f2bf6e 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -49,6 +49,10 @@ This product includes software from Google Guava (Apache 2.0) * Copyright (C) 2007 The Guava Authors * https://github.com/google/guava +This product include software from CMake (BSD 3-Clause) + * CMake - Cross Platform Makefile Generator + * Copyright 2000-2019 Kitware, Inc. and Contributors + The web site includes files generated by Jekyll. -------------------------------------------------------------------------------- diff --git a/appveyor.yml b/appveyor.yml index 3c4c2c3dc76..5d7e20bdee2 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -34,8 +34,8 @@ cache: - C:\Users\Appveyor\clcache - C:\Users\Appveyor\.cargo\registry -matrix: - fast_finish: true +#matrix: +# fast_finish: true environment: global: @@ -61,10 +61,6 @@ environment: - JOB: "Build_Debug" GENERATOR: Ninja CONFIGURATION: "Debug" - - JOB: "Cmake_Script_Tests" - GENERATOR: Ninja - CONFIGURATION: "Release" - BUILD_SCRIPT: "CMake_Build_Script" - JOB: "MinGW32" MINGW_PACKAGE_PREFIX: mingw-w64-i686 MINGW_PREFIX: c:\msys64\mingw32 diff --git a/ci/appveyor-build.bat b/ci/appveyor-build.bat index 2d5b37aaf20..26445e21492 100644 --- a/ci/appveyor-build.bat +++ b/ci/appveyor-build.bat @@ -28,8 +28,6 @@ git config core.symlinks true git reset --hard if "%JOB:~,5%" == "MinGW" ( call ci\appveyor-cpp-build-mingw.bat -) else if "%JOB%" == "Cmake_Script_Tests" ( - call ci\appveyor-cpp-test-cmake-script.bat ) else ( call ci\appveyor-cpp-build.bat ) diff --git a/ci/appveyor-cpp-build-mingw.bat b/ci/appveyor-cpp-build-mingw.bat index a7ea37477e7..87698a05f47 100644 --- a/ci/appveyor-cpp-build-mingw.bat +++ b/ci/appveyor-cpp-build-mingw.bat @@ -32,17 +32,12 @@ set CPP_BUILD_DIR=cpp\build mkdir %CPP_BUILD_DIR% pushd %CPP_BUILD_DIR% -set BOOST_ROOT=%MINGW_PREFIX% -set LZ4_HOME=%MINGW_PREFIX% -set ZSTD_HOME=%MINGW_PREFIX% -set SNAPPY_HOME=%MINGW_PREFIX% -set BROTLI_HOME=%MINGW_PREFIX% -set FLATBUFFERS_HOME=%MINGW_PREFIX% cmake ^ -G "MSYS Makefiles" ^ -DCMAKE_INSTALL_PREFIX=%INSTALL_DIR% ^ -DCMAKE_BUILD_TYPE=%CMAKE_BUILD_TYPE% ^ -DARROW_VERBOSE_THIRDPARTY_BUILD=OFF ^ + -DARROW_PACKAGE_PREFIX=%MINGW_PREFIX% ^ -DARROW_JEMALLOC=OFF ^ -DARROW_USE_GLOG=OFF ^ -DARROW_PYTHON=ON ^ diff --git a/ci/appveyor-cpp-build.bat b/ci/appveyor-cpp-build.bat index fa310a3b82c..b735d0f53d2 100644 --- a/ci/appveyor-cpp-build.bat +++ b/ci/appveyor-cpp-build.bat @@ -30,7 +30,7 @@ if "%JOB%" == "Static_Crt_Build" ( pushd cpp\build-debug cmake -G "%GENERATOR%" ^ - -DARROW_VERBOSE_THIRDPARTY_BUILD=OFF ^ + -DARROW_VERBOSE_THIRDPARTY_BUILD=ON ^ -DARROW_USE_STATIC_CRT=ON ^ -DARROW_BOOST_USE_SHARED=OFF ^ -DARROW_BUILD_SHARED=OFF ^ @@ -108,7 +108,7 @@ if "%JOB%" == "Toolchain" ( set CONDA_PACKAGES=%CONDA_PACKAGES% --file=ci\conda_env_cpp.yml ) -conda create -n arrow -q -y %CONDA_PACKAGES% -c conda-forge +conda create -n arrow -q -y %CONDA_PACKAGES% -c conda-forge || exit /B call activate arrow diff --git a/ci/appveyor-cpp-setup-mingw.bat b/ci/appveyor-cpp-setup-mingw.bat index 0c3d633cf8a..b6eb986a151 100644 --- a/ci/appveyor-cpp-setup-mingw.bat +++ b/ci/appveyor-cpp-setup-mingw.bat @@ -23,6 +23,7 @@ pacman -S --noconfirm ^ "%MINGW_PACKAGE_PREFIX%-boost" ^ "%MINGW_PACKAGE_PREFIX%-brotli" ^ "%MINGW_PACKAGE_PREFIX%-cmake" ^ + "%MINGW_PACKAGE_PREFIX%-double-conversion" ^ "%MINGW_PACKAGE_PREFIX%-flatbuffers" ^ "%MINGW_PACKAGE_PREFIX%-gcc" ^ "%MINGW_PACKAGE_PREFIX%-gflags" ^ @@ -32,6 +33,7 @@ pacman -S --noconfirm ^ "%MINGW_PACKAGE_PREFIX%-meson" ^ "%MINGW_PACKAGE_PREFIX%-protobuf" ^ "%MINGW_PACKAGE_PREFIX%-python3-numpy" ^ + "%MINGW_PACKAGE_PREFIX%-rapidjson" ^ "%MINGW_PACKAGE_PREFIX%-snappy" ^ "%MINGW_PACKAGE_PREFIX%-zlib" ^ "%MINGW_PACKAGE_PREFIX%-zstd" || exit /B diff --git a/ci/appveyor-cpp-test-cmake-script.bat b/ci/appveyor-cpp-test-cmake-script.bat deleted file mode 100644 index 415406c4ac3..00000000000 --- a/ci/appveyor-cpp-test-cmake-script.bat +++ /dev/null @@ -1,193 +0,0 @@ -@rem Licensed to the Apache Software Foundation (ASF) under one -@rem or more contributor license agreements. See the NOTICE file -@rem distributed with this work for additional information -@rem regarding copyright ownership. The ASF licenses this file -@rem to you under the Apache License, Version 2.0 (the -@rem "License"); you may not use this file except in compliance -@rem with the License. You may obtain a copy of the License at -@rem -@rem http://www.apache.org/licenses/LICENSE-2.0 -@rem -@rem Unless required by applicable law or agreed to in writing, -@rem software distributed under the License is distributed on an -@rem "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -@rem KIND, either express or implied. See the License for the -@rem specific language governing permissions and limitations -@rem under the License. - -@echo on - -@rem Validate cmake script behaviour on missed lib in toolchain -set CONDA_ENV=arrow-cmake-tests-libs -conda create -n %CONDA_ENV% -q -y -conda install -n %CONDA_ENV% -q -y -c conda-forge boost-cpp -call activate %CONDA_ENV% - -set BUILD_DIR=cpp\build-cmake-test -mkdir %BUILD_DIR% -pushd %BUILD_DIR% - -echo Test cmake script errors out on flatbuffers missed -set FLATBUFFERS_HOME=WrongPath - -cmake -G "%GENERATOR%" ^ - -DARROW_BOOST_USE_SHARED=OFF ^ - -DARROW_BUILD_TESTS=ON ^ - -DARROW_BUILD_EXAMPLES=ON ^ - -DCMAKE_BUILD_TYPE=%CONFIGURATION% ^ - -DARROW_CXXFLAGS="/MP" ^ - .. >nul 2>error.txt - -FINDSTR /M /C:"Could not find the Flatbuffers library" error.txt || exit /B -set FLATBUFFERS_HOME= - -popd -rmdir /S /Q %BUILD_DIR% -mkdir %BUILD_DIR% -pushd %BUILD_DIR% - -echo Test cmake script errors out on gflags missed -set GFLAGS_HOME=WrongPath - -cmake -G "%GENERATOR%" ^ - -DARROW_BOOST_USE_SHARED=OFF ^ - -DARROW_BUILD_TESTS=ON ^ - -DARROW_BUILD_EXAMPLES=ON ^ - -DCMAKE_BUILD_TYPE=%CONFIGURATION% ^ - -DARROW_CXXFLAGS="/MP" ^ - .. >nul 2>error.txt - -FINDSTR /M /C:"No static or shared library provided for gflags" error.txt || exit /B -set GFLAGS_HOME= - -popd -rmdir /S /Q %BUILD_DIR% -mkdir %BUILD_DIR% -pushd %BUILD_DIR% - -echo Test cmake script errors out on snappy missed -set SNAPPY_HOME=WrongPath - -cmake -G "%GENERATOR%" ^ - -DARROW_BOOST_USE_SHARED=OFF ^ - -DARROW_BUILD_TESTS=ON ^ - -DARROW_BUILD_EXAMPLES=ON ^ - -DCMAKE_BUILD_TYPE=%CONFIGURATION% ^ - -DARROW_CXXFLAGS="/MP" ^ - .. >nul 2>error.txt - -FINDSTR /M /C:"Could not find the Snappy library" error.txt || exit /B -set SNAPPY_HOME= - -popd -rmdir /S /Q %BUILD_DIR% -mkdir %BUILD_DIR% -pushd %BUILD_DIR% - -echo Test cmake script errors out on zlib missed -set ZLIB_HOME=WrongPath - -cmake -G "%GENERATOR%" ^ - -DARROW_BOOST_USE_SHARED=OFF ^ - -DARROW_BUILD_TESTS=ON ^ - -DARROW_BUILD_EXAMPLES=ON ^ - -DCMAKE_BUILD_TYPE=%CONFIGURATION% ^ - -DARROW_CXXFLAGS="/MP" ^ - .. >nul 2>error.txt - -FINDSTR /M /C:"Could not find the ZLIB library" error.txt || exit /B -set ZLIB_HOME= - -popd -rmdir /S /Q %BUILD_DIR% -mkdir %BUILD_DIR% -pushd %BUILD_DIR% - -echo Test cmake script errors out on brotli missed -set BROTLI_HOME=WrongPath - -cmake -G "%GENERATOR%" ^ - -DARROW_BOOST_USE_SHARED=OFF ^ - -DARROW_BUILD_TESTS=ON ^ - -DARROW_BUILD_EXAMPLES=ON ^ - -DCMAKE_BUILD_TYPE=%CONFIGURATION% ^ - -DARROW_CXXFLAGS="/MP" ^ - .. >nul 2>error.txt - -FINDSTR /M /C:"Could not find the Brotli library" error.txt || exit /B -set BROTLI_HOME= - -popd -rmdir /S /Q %BUILD_DIR% -mkdir %BUILD_DIR% -pushd %BUILD_DIR% - -echo Test cmake script errors out on lz4 missed -set LZ4_HOME=WrongPath - -cmake -G "%GENERATOR%" ^ - -DARROW_BOOST_USE_SHARED=OFF ^ - -DARROW_BUILD_TESTS=ON ^ - -DARROW_BUILD_EXAMPLES=ON ^ - -DCMAKE_BUILD_TYPE=%CONFIGURATION% ^ - -DARROW_CXXFLAGS="/MP" ^ - .. >nul 2>error.txt - -FINDSTR /M /C:"No static or shared library provided for lz4" error.txt || exit /B -set LZ4_HOME= - -popd -rmdir /S /Q %BUILD_DIR% -mkdir %BUILD_DIR% -pushd %BUILD_DIR% - -echo Test cmake script errors out on zstd missed -set ZSTD_HOME=WrongPath - -cmake -G "%GENERATOR%" ^ - -DARROW_BOOST_USE_SHARED=OFF ^ - -DARROW_BUILD_TESTS=ON ^ - -DARROW_BUILD_EXAMPLES=ON ^ - -DCMAKE_BUILD_TYPE=%CONFIGURATION% ^ - -DARROW_CXXFLAGS="/MP" ^ - .. >nul 2>error.txt - -FINDSTR /M /C:"Could NOT find ZSTD" error.txt || exit /B -set ZSTD_HOME= - -popd -rmdir /S /Q %BUILD_DIR% -call deactivate - -@rem Validate libs availability in conda toolchain -set CONDA_ENV=arrow-cmake-tests-toolchain -conda create -n %CONDA_ENV% -q -y -conda install -n %CONDA_ENV% -q -y -c conda-forge ^ - --file=ci\conda_env_cpp.yml -call activate %CONDA_ENV% - -mkdir %BUILD_DIR% -pushd %BUILD_DIR% - -set ARROW_BUILD_TOOLCHAIN=%CONDA_PREFIX%\Library -cmake -G "%GENERATOR%" ^ - -DARROW_BOOST_USE_SHARED=OFF ^ - -DARROW_BUILD_TESTS=ON ^ - -DARROW_BUILD_EXAMPLES=ON ^ - -DCMAKE_BUILD_TYPE=%CONFIGURATION% ^ - -DARROW_CXXFLAGS="/MP" ^ - .. 2>output.txt - -set LIBRARY_FOUND_MSG=Added static library dependency -for %%x in (snappy gflags brotli_enc brotli_dec brotli_common lz4 zstd) do ( - echo Checking %%x library path - FINDSTR /C:"%LIBRARY_FOUND_MSG% %%x_static: %CONDA_PREFIX:\=/%" output.txt || exit /B -) -set LIBRARY_FOUND_MSG=Added shared library dependency -for %%x in (zlib) do ( - echo Checking %%x library path - FINDSTR /C:"%LIBRARY_FOUND_MSG% %%x_shared: %CONDA_PREFIX:\=/%" output.txt || exit /B -) - -popd -rmdir /S /Q %BUILD_DIR% diff --git a/ci/conda_env_cpp.yml b/ci/conda_env_cpp.yml index fbe45769a72..cf6f060803f 100644 --- a/ci/conda_env_cpp.yml +++ b/ci/conda_env_cpp.yml @@ -15,6 +15,7 @@ # specific language governing permissions and limitations # under the License. +benchmark # ARROW-4056: The conda-forge boost 1.69.0 seems to break the Parquet unit # tests with Xcode 8.3. Root cause not yet determined boost-cpp=1.68.0 @@ -29,8 +30,11 @@ glog gmock grpc-cpp gtest +clangdev=7 +llvmdev=7 libprotobuf lz4-c +ninja python rapidjson re2 diff --git a/ci/conda_env_unix.yml b/ci/conda_env_unix.yml index 9ecf549b504..2d3f0535e3a 100644 --- a/ci/conda_env_unix.yml +++ b/ci/conda_env_unix.yml @@ -18,4 +18,7 @@ # conda package dependencies specific to Unix-like environments (Linux and macOS) autoconf +ccache +pkg-config rsync +valgrind diff --git a/ci/cpp-msvc-build-main.bat b/ci/cpp-msvc-build-main.bat index db15de61585..97ed401aa6b 100644 --- a/ci/cpp-msvc-build-main.bat +++ b/ci/cpp-msvc-build-main.bat @@ -22,8 +22,16 @@ set ARROW_HOME=%CONDA_PREFIX%\Library set CMAKE_ARGS=-DARROW_VERBOSE_THIRDPARTY_BUILD=OFF if "%JOB%" == "Toolchain" ( - set CMAKE_ARGS=%CMAKE_ARGS% -DARROW_WITH_BZ2=ON + @rem Toolchain gtest does not currently work with Visual Studio 2015 + set CMAKE_ARGS=^ + %CMAKE_ARGS% ^ + -DARROW_WITH_BZ2=ON ^ + -DARROW_DEPENDENCY_SOURCE=CONDA ^ + -DGTest_SOURCE=BUNDLED set ARROW_BUILD_TOOLCHAIN=%CONDA_PREFIX%\Library +) else ( + @rem We're in a conda enviroment but don't want to use it for the dependencies + set CMAKE_ARGS=%CMAKE_ARGE% -DARROW_DEPENDENCY_SOURCE=AUTO ) @rem Retrieve git submodules, configure env var for Parquet unit tests diff --git a/ci/docker_build_cpp.sh b/ci/docker_build_cpp.sh index 5bf6094ae2a..782267a1526 100755 --- a/ci/docker_build_cpp.sh +++ b/ci/docker_build_cpp.sh @@ -22,13 +22,23 @@ source_dir=${1:-/arrow/cpp} build_dir=${2:-/build/cpp} install_dir=${3:-${ARROW_HOME:-/usr/local}} +export CCACHE_DIR=/build/ccache + +rm -rf ${build_dir} mkdir -p ${build_dir} pushd ${build_dir} cmake -GNinja \ + -DARROW_DEPENDENCY_SOURCE=${ARROW_DEPENDENCY_SOURCE:-AUTO} \ + -DARROW_VERBOSE_THIRDPARTY_BUILD=ON \ -DCMAKE_BUILD_TYPE=${ARROW_BUILD_TYPE:-debug} \ -DCMAKE_INSTALL_PREFIX=${install_dir} \ -DCMAKE_INSTALL_LIBDIR=lib \ + -DARROW_WITH_BZ2=${ARROW_WITH_BZ2:-ON} \ + -DARROW_WITH_ZSTD=${ARROW_WITH_ZSTD:-ON} \ + -DARROW_GANDIVA=${ARROW_GANDIVA:-ON} \ + -DARROW_BUILD_BENCHMARKS=${ARROW_BUILD_BENCHMARKS:-ON} \ + -DARROW_FLIGHT=${ARROW_FLIGHT:-ON} \ -DARROW_ORC=${ARROW_ORC:-ON} \ -DARROW_PLASMA=${ARROW_PLASMA:-ON} \ -DARROW_PARQUET=${ARROW_PARQUET:-ON} \ @@ -41,6 +51,7 @@ cmake -GNinja \ -DARROW_INSTALL_NAME_RPATH=${ARROW_INSTALL_NAME_RPATH:-ON} \ -DARROW_EXTRA_ERROR_CONTEXT=ON \ -DCMAKE_CXX_FLAGS=$CXXFLAGS \ + ${CMAKE_ARGS} \ ${source_dir} ninja ninja install diff --git a/ci/travis_before_script_cpp.sh b/ci/travis_before_script_cpp.sh index 2ed2f62411e..26a45ecd6ed 100755 --- a/ci/travis_before_script_cpp.sh +++ b/ci/travis_before_script_cpp.sh @@ -141,6 +141,18 @@ if [ "$ARROW_TRAVIS_OPTIONAL_INSTALL" == "1" ]; then CMAKE_COMMON_FLAGS="$CMAKE_COMMON_FLAGS -DARROW_OPTIONAL_INSTALL=ON" fi +if [ "$ARROW_TRAVIS_USE_TOOLCHAIN" == "1" ]; then + conda activate $CPP_TOOLCHAIN +fi + +# conda-forge sets the build flags by default to -02, skip this to speed up the build +export CFLAGS=${CFLAGS//-O2} +export CXXFLAGS=${CXXFLAGS//-O2} + +if [ $TRAVIS_OS_NAME == "osx" ]; then + source $TRAVIS_BUILD_DIR/ci/travis_install_osx_sdk.sh +fi + if [ $TRAVIS_OS_NAME == "linux" ]; then cmake $CMAKE_COMMON_FLAGS \ $CMAKE_LINUX_FLAGS \ @@ -164,7 +176,7 @@ fi # Build and install libraries. Configure ARROW_CPP_BUILD_TARGETS environment # variable to only build certain targets. If you use this, you must also set # the environment variable ARROW_TRAVIS_OPTIONAL_INSTALL=1 -$TRAVIS_MAKE -j4 $ARROW_CPP_BUILD_TARGETS -$TRAVIS_MAKE install +time $TRAVIS_MAKE -j4 $ARROW_CPP_BUILD_TARGETS +time $TRAVIS_MAKE install popd diff --git a/ci/travis_env_common.sh b/ci/travis_env_common.sh index 8da83bb76f9..a1cc125c760 100755 --- a/ci/travis_env_common.sh +++ b/ci/travis_env_common.sh @@ -63,14 +63,7 @@ export ARROW_BUILD_WARNING_LEVEL=${ARROW_BUILD_WARNING_LEVEL:=Production} if [ "$ARROW_TRAVIS_USE_TOOLCHAIN" == "1" ]; then # C++ toolchain export CPP_TOOLCHAIN=$TRAVIS_BUILD_DIR/cpp-toolchain - export ARROW_BUILD_TOOLCHAIN=$CPP_TOOLCHAIN - export BOOST_ROOT=$CPP_TOOLCHAIN - # Protocol buffers used by Apache ORC thirdparty build - export PROTOBUF_HOME=$CPP_TOOLCHAIN - - export PATH=$CPP_TOOLCHAIN/bin:$PATH - export LD_LIBRARY_PATH=$CPP_TOOLCHAIN/lib:$LD_LIBRARY_PATH export TRAVIS_MAKE=ninja else export TRAVIS_MAKE=make diff --git a/ci/travis_install_linux.sh b/ci/travis_install_linux.sh index 5be5e7a77fe..441d50d9b16 100755 --- a/ci/travis_install_linux.sh +++ b/ci/travis_install_linux.sh @@ -42,6 +42,39 @@ if [ "$ARROW_TRAVIS_GANDIVA" == "1" ]; then sudo apt-get install -y -qq llvm-$ARROW_LLVM_MAJOR_VERSION-dev fi +if [ "$ARROW_TRAVIS_USE_SYSTEM" == "1" ]; then + if [ "$DISTRO_CODENAME" == "xenial" ]; then + # TODO(ARROW-4761): Install libzstd-dev once we support zstd<1 + # TODO(ARROW-4762): Install rapidjson-dev once we support rapidjson<1.1.0 + export ADDITIONAL_APT_PKGS="libre2-dev libbrotli-dev liblz4-dev" + fi + sudo apt-get install -y -q --no-install-recommends \ + autoconf \ + ca-certificates \ + ccache \ + cmake \ + g++ \ + gcc \ + git \ + libboost-all-dev \ + libdouble-conversion-dev \ + libsnappy-dev \ + libssl-dev \ + libgflags-dev \ + libprotobuf-dev \ + libprotoc-dev \ + llvm-7-dev \ + clang-7 \ + protobuf-compiler \ + libbz2-dev \ + ninja-build \ + pkg-config \ + tzdata \ + flex \ + bison \ + ${ADDITIONAL_APT_PKGS} +fi + set -x if [ "$DISTRO_CODENAME" != "trusty" ]; then sudo apt-get install -y -qq maven diff --git a/ci/travis_install_osx_sdk.sh b/ci/travis_install_osx_sdk.sh new file mode 100755 index 00000000000..3c3624cdb00 --- /dev/null +++ b/ci/travis_install_osx_sdk.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -ex + +export MACOSX_DEPLOYMENT_TARGET="10.9" +export CONDA_BUILD_SYSROOT="$(xcode-select -p)/Platforms/MacOSX.platform/Developer/SDKs/MacOSX${MACOSX_DEPLOYMENT_TARGET}.sdk" + +if [[ ! -d ${CONDA_BUILD_SYSROOT} || "$OSX_FORCE_SDK_DOWNLOAD" == "1" ]]; then + echo "downloading ${macosx_deployment_target} sdk" + curl -L -O https://github.com/phracker/MacOSX-SDKs/releases/download/10.13/MacOSX${MACOSX_DEPLOYMENT_TARGET}.sdk.tar.xz + tar -xf MacOSX${MACOSX_DEPLOYMENT_TARGET}.sdk.tar.xz -C "$(dirname "$CONDA_BUILD_SYSROOT")" + # set minimum sdk version to our target + plutil -replace MinimumSDKVersion -string ${MACOSX_DEPLOYMENT_TARGET} $(xcode-select -p)/Platforms/MacOSX.platform/Info.plist + plutil -replace DTSDKName -string macosx${MACOSX_DEPLOYMENT_TARGET}internal $(xcode-select -p)/Platforms/MacOSX.platform/Info.plist +fi + +if [ -d "${CONDA_BUILD_SYSROOT}" ]; then + echo "Found CONDA_BUILD_SYSROOT: ${CONDA_BUILD_SYSROOT}" +else + echo "Missing CONDA_BUILD_SYSROOT: ${CONDA_BUILD_SYSROOT}" + exit 1 +fi + diff --git a/ci/travis_install_toolchain.sh b/ci/travis_install_toolchain.sh index 4d674a1f791..d5fdb11e01b 100755 --- a/ci/travis_install_toolchain.sh +++ b/ci/travis_install_toolchain.sh @@ -29,26 +29,17 @@ if [ ! -e $CPP_TOOLCHAIN ]; then CONDA_PACKAGES="$CONDA_PACKAGES llvmdev=$CONDA_LLVM_VERSION" fi - if [ $TRAVIS_OS_NAME == "linux" ]; then - if [ "$DISTRO_CODENAME" == "trusty" ]; then - CONDA_LABEL=" -c conda-forge/label/cf201901" - else - # Use newer binutils when linking against conda-provided libraries - CONDA_PACKAGES="$CONDA_PACKAGES binutils=$CONDA_BINUTILS_VERSION" - fi - fi - if [ "$ARROW_TRAVIS_VALGRIND" == "1" ]; then # Use newer Valgrind CONDA_PACKAGES="$CONDA_PACKAGES valgrind" fi # Set up C++ toolchain from conda-forge packages for faster builds - conda create -y -q -p $CPP_TOOLCHAIN $CONDA_LABEL \ + time conda create -y -q -p $CPP_TOOLCHAIN \ --file=$TRAVIS_BUILD_DIR/ci/conda_env_cpp.yml \ + --file=$TRAVIS_BUILD_DIR/ci/conda_env_unix.yml \ + compilers \ $CONDA_PACKAGES \ - ccache \ - ninja \ nomkl \ python=3.6 fi diff --git a/ci/travis_script_cpp.sh b/ci/travis_script_cpp.sh index 14529b03160..718cda88288 100755 --- a/ci/travis_script_cpp.sh +++ b/ci/travis_script_cpp.sh @@ -23,7 +23,12 @@ source $TRAVIS_BUILD_DIR/ci/travis_env_common.sh pushd $CPP_BUILD_DIR -PATH=$ARROW_BUILD_TYPE:$PATH ctest -j2 --output-on-failure -L unittest +if [ $TRAVIS_OS_NAME == "osx" ]; then + # TODO: This does not seem to terminate + CTEST_ARGS="-E arrow-flight-test" +fi + +PATH=$ARROW_BUILD_TYPE:$PATH ctest -j2 --output-on-failure -L unittest ${CTEST_ARGS} popd diff --git a/ci/travis_script_integration.sh b/ci/travis_script_integration.sh index 02e2eae8150..e8d67acc16c 100755 --- a/ci/travis_script_integration.sh +++ b/ci/travis_script_integration.sh @@ -17,7 +17,7 @@ # specific language governing permissions and limitations # under the License. -set -e +set -ex source $TRAVIS_BUILD_DIR/ci/travis_env_common.sh @@ -42,15 +42,9 @@ popd pushd $ARROW_INTEGRATION_DIR -CONDA_ENV_NAME=arrow-integration-test -conda create -y -q -n $CONDA_ENV_NAME python=3.6 -conda activate $CONDA_ENV_NAME - -# faster builds, please -conda install -y nomkl - -# Expensive dependencies install from Continuum package repo -conda install -y pip numpy six +conda activate $CPP_TOOLCHAIN +# Install integration test requirements +conda install -y -q python=3.6 six numpy # ARROW-4008: Create a directory to write temporary files since /tmp can be # unstable in Travis CI diff --git a/ci/travis_script_plasma_java_client.sh b/ci/travis_script_plasma_java_client.sh index 0b291ed32a5..61027b58d3d 100755 --- a/ci/travis_script_plasma_java_client.sh +++ b/ci/travis_script_plasma_java_client.sh @@ -21,6 +21,10 @@ set -e source $TRAVIS_BUILD_DIR/ci/travis_env_common.sh +source $TRAVIS_BUILD_DIR/ci/travis_install_conda.sh + +conda activate $CPP_TOOLCHAIN + PLASMA_JAVA_DIR=${TRAVIS_BUILD_DIR}/java/plasma pushd $PLASMA_JAVA_DIR diff --git a/ci/travis_script_python.sh b/ci/travis_script_python.sh index 99126e59278..5e4aaac0527 100755 --- a/ci/travis_script_python.sh +++ b/ci/travis_script_python.sh @@ -46,12 +46,14 @@ if [ "$ARROW_TRAVIS_PYTHON_JVM" == "1" ]; then fi conda create -y -q -p $CONDA_ENV_DIR \ + --file $TRAVIS_BUILD_DIR/ci/conda_env_cpp.yml \ --file $TRAVIS_BUILD_DIR/ci/conda_env_python.yml \ nomkl \ pip \ numpy=1.14 \ 'libgfortran<4' \ python=${PYTHON_VERSION} \ + compilers \ ${CONDA_JVM_DEPS} conda activate $CONDA_ENV_DIR @@ -79,11 +81,12 @@ if [ $TRAVIS_OS_NAME != "osx" ]; then fi # Re-build C++ libraries with the right Python setup + +# Clear out prior build files +rm -rf $ARROW_CPP_BUILD_DIR mkdir -p $ARROW_CPP_BUILD_DIR pushd $ARROW_CPP_BUILD_DIR -# Clear out prior build files -rm -rf * # XXX Can we simply reuse CMAKE_COMMON_FLAGS from travis_before_script_cpp.sh? CMAKE_COMMON_FLAGS="-DARROW_EXTRA_ERROR_CONTEXT=ON" @@ -99,6 +102,18 @@ if [ "$ARROW_TRAVIS_PYTHON_GANDIVA" == "1" ]; then PYTHON_CPP_BUILD_TARGETS="$PYTHON_CPP_BUILD_TARGETS gandiva" fi +if [ "$ARROW_TRAVIS_VERBOSE" == "1" ]; then + CMAKE_COMMON_FLAGS="$CMAKE_COMMON_FLAGS -DARROW_VERBOSE_THIRDPARTY_BUILD=ON" +fi + +if [ $TRAVIS_OS_NAME == "osx" ]; then + source $TRAVIS_BUILD_DIR/ci/travis_install_osx_sdk.sh +fi + +# conda-forge sets the build flags by default to -02, skip this to speed up the build +export CFLAGS=${CFLAGS//-O2} +export CXXFLAGS=${CXXFLAGS//-O2} + cmake -GNinja \ $CMAKE_COMMON_FLAGS \ -DARROW_BUILD_TESTS=ON \ diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 428330b71d7..7a757d83cdc 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -186,6 +186,33 @@ static|shared (default shared)") #---------------------------------------------------------------------- # Thirdparty toolchain options + # Determine how we will look for dependencies: + # * AUTO: Guess which packaging systems we're running in and pull the + # dependencies from there. Build any missing ones through the + # ExternalProject setup. + # * BUNDLED: Build all dependencies through CMake's ExternalProject facility. + # * SYSTEM: Use CMake's find_package and find_library without any custom + # paths. If packages are on non-default locations, let the user indicate it + # from the outside using the *_ROOT variables. + # If your system packages are in general in a non-default location, you can + # set a general default for the *_ROOT variables using ARROW_PACKAGE_PREFIX. + # * CONDA: Same as system but set all *_ROOT variables to ENV{CONDA_PREFIX}. + # * BREW: Use SYSTEM but search for select packages with brew. + if(NOT "$ENV{CONDA_PREFIX}" STREQUAL "") + set(ARROW_DEPENDENCY_SOURCE_DEFAULT "CONDA") + else() + set(ARROW_DEPENDENCY_SOURCE_DEFAULT "AUTO") + endif() + set(ARROW_DEPENDENCY_SOURCE "${ARROW_DEPENDENCY_SOURCE_DEFAULT}" + CACHE STRING "Compiler flags to append when compiling Arrow") + set_property(CACHE ARROW_DEPENDENCY_SOURCE + PROPERTY STRINGS + "AUTO" + "BUNDLED" + "SYSTEM" + "CONDA" + "BREW") + option(ARROW_VERBOSE_THIRDPARTY_BUILD "If off, output from ExternalProjects will be logged to files rather than shown" OFF) @@ -196,7 +223,7 @@ static|shared (default shared)") Note that this requires linking Boost statically" OFF) option(ARROW_PROTOBUF_USE_SHARED - "Rely on Protocol Buffers shared libraries where relevant" OFF) + "Rely on Protocol Buffers shared libraries where relevant" ON) option(ARROW_GFLAGS_USE_SHARED "Rely on GFlags shared libraries where relevant" ON) @@ -712,53 +739,60 @@ endif(UNIX) # Linker and Dependencies # +# TODO: Also rework how these libs work set(ARROW_LINK_LIBS) set(ARROW_SHARED_INSTALL_INTERFACE_LIBS) set(ARROW_STATIC_INSTALL_INTERFACE_LIBS) # Libraries to link statically with libarrow.so -set(ARROW_STATIC_LINK_LIBS double-conversion_static) -set(ARROW_STATIC_INSTALL_INTERFACE_LIBS double-conversion) +set(ARROW_LINK_LIBS ${double-conversion_LIBRARIES}) +set(ARROW_STATIC_LINK_LIBS ${double-conversion_LIBRARIES}) +set(ARROW_STATIC_INSTALL_INTERFACE_LIBS ${double-conversion_LIBRARIES}) if(ARROW_WITH_BROTLI) - list(APPEND ARROW_STATIC_LINK_LIBS brotli_dec_static brotli_enc_static - brotli_common_static) - list(APPEND ARROW_STATIC_INSTALL_INTERFACE_LIBS brotlidec brotlienc brotlicommon) + # Order is important for static linking + list(APPEND ARROW_LINK_LIBS Brotli::brotlienc Brotli::brotlidec Brotli::brotlicommon) + list(APPEND ARROW_STATIC_LINK_LIBS Brotli::brotlienc Brotli::brotlidec + Brotli::brotlicommon) + list(APPEND ARROW_STATIC_INSTALL_INTERFACE_LIBS Brotli::brotlienc Brotli::brotlidec + Brotli::brotlicommon) endif() if(ARROW_WITH_BZ2) - list(APPEND ARROW_STATIC_LINK_LIBS bz2_static) - list(APPEND ARROW_STATIC_INSTALL_INTERFACE_LIBS bz2) + list(APPEND ARROW_STATIC_LINK_LIBS BZip2::BZip2) + list(APPEND ARROW_STATIC_INSTALL_INTERFACE_LIBS BZip2::BZip2) endif() if(ARROW_WITH_LZ4) - list(APPEND ARROW_STATIC_LINK_LIBS lz4_static) - list(APPEND ARROW_STATIC_INSTALL_INTERFACE_LIBS lz4) + list(APPEND ARROW_STATIC_LINK_LIBS LZ4::lz4) + list(APPEND ARROW_STATIC_INSTALL_INTERFACE_LIBS LZ4::lz4) endif() if(ARROW_WITH_SNAPPY) - list(APPEND ARROW_STATIC_LINK_LIBS snappy_static) - list(APPEND ARROW_STATIC_INSTALL_INTERFACE_LIBS snappy) + list(APPEND ARROW_STATIC_LINK_LIBS Snappy::snappy) + list(APPEND ARROW_STATIC_INSTALL_INTERFACE_LIBS Snappy::snappy) endif() if(ARROW_WITH_ZLIB) - list(APPEND ARROW_STATIC_LINK_LIBS ${ZLIB_LIBRARY}) - list(APPEND ARROW_STATIC_INSTALL_INTERFACE_LIBS z) + list(APPEND ARROW_STATIC_LINK_LIBS ZLIB::ZLIB) + list(APPEND ARROW_STATIC_INSTALL_INTERFACE_LIBS ZLIB::ZLIB) endif() if(ARROW_WITH_ZSTD) - list(APPEND ARROW_STATIC_LINK_LIBS zstd_static) - list(APPEND ARROW_STATIC_INSTALL_INTERFACE_LIBS zstd) + list(APPEND ARROW_STATIC_LINK_LIBS ZSTD::zstd) + list(APPEND ARROW_STATIC_INSTALL_INTERFACE_LIBS ZSTD::zstd) endif() if(ARROW_ORC) - list(APPEND ARROW_STATIC_LINK_LIBS orc_static) - list(APPEND ARROW_STATIC_INSTALL_INTERFACE_LIBS orc) + list(APPEND ARROW_LINK_LIBS protobuf::libprotobuf orc_static) + list(APPEND ARROW_STATIC_LINK_LIBS protobuf::libprotobuf orc_static) + list(APPEND ARROW_STATIC_INSTALL_INTERFACE_LIBS protobuf::libprotobuf orc) endif() if(ARROW_USE_GLOG) - list(APPEND ARROW_STATIC_LINK_LIBS glog_static) - list(APPEND ARROW_STATIC_INSTALL_INTERFACE_LIBS glog) + list(APPEND ARROW_LINK_LIBS GLOG::glog) + list(APPEND ARROW_STATIC_LINK_LIBS GLOG::glog) + list(APPEND ARROW_STATIC_INSTALL_INTERFACE_LIBS GLOG::glog) add_definitions("-DARROW_USE_GLOG") endif() @@ -792,7 +826,7 @@ if(NOT MSVC) list(APPEND ARROW_SHARED_INSTALL_INTERFACE_LIBS ${CMAKE_DL_LIBS}) endif() -set(ARROW_TEST_LINK_TOOLCHAIN ${GTEST_LIBRARY} ${GMOCK_MAIN_LIBRARY} ${GMOCK_LIBRARY}) +set(ARROW_TEST_LINK_TOOLCHAIN GTest::Main GTest::GTest GTest::GMock) if(ARROW_BUILD_TESTS) add_dependencies(arrow_test_dependencies ${ARROW_TEST_LINK_TOOLCHAIN}) @@ -814,7 +848,7 @@ set(ARROW_TEST_SHARED_LINK_LIBS arrow_testing_shared arrow_shared ${ARROW_LINK_LIBS} - double-conversion_static + ${double-conversion_LIBRARIES} ${BOOST_SYSTEM_LIBRARY} ${BOOST_FILESYSTEM_LIBRARY} ${BOOST_REGEX_LIBRARY} @@ -842,7 +876,16 @@ pass ARROW_BUILD_STATIC=on") endif() if(ARROW_BUILD_BENCHMARKS) - set(ARROW_BENCHMARK_LINK_LIBS arrow_benchmark_main ${ARROW_TEST_LINK_LIBS}) + # In the case that benchmark::benchmark_main is not available, + # we need to provide our own version. This only happens for older versions + # of benchmark. + if(NOT TARGET benchmark::benchmark_main) + add_library(arrow_benchmark_main STATIC src/arrow/util/benchmark_main.cc) + add_library(benchmark::benchmark_main ALIAS arrow_benchmark_main) + endif() + + set(ARROW_BENCHMARK_LINK_LIBS benchmark::benchmark_main benchmark::benchmark + ${ARROW_TEST_LINK_LIBS}) endif() set(ARROW_SYSTEM_LINK_LIBS) @@ -850,7 +893,7 @@ set(ARROW_SYSTEM_LINK_LIBS) if(ARROW_JEMALLOC) add_definitions(-DARROW_JEMALLOC) add_definitions(-DARROW_JEMALLOC_INCLUDE_DIR=${JEMALLOC_INCLUDE_DIR}) - list(APPEND ARROW_SYSTEM_LINK_LIBS jemalloc_static) + list(APPEND ARROW_SYSTEM_LINK_LIBS jemalloc::jemalloc) endif(ARROW_JEMALLOC) if(THREADS_FOUND) diff --git a/cpp/Dockerfile b/cpp/Dockerfile index f1eafd4b0e0..0b177af95f1 100644 --- a/cpp/Dockerfile +++ b/cpp/Dockerfile @@ -50,7 +50,7 @@ ENV CC=gcc \ CXX=g++ \ ARROW_GANDIVA=OFF \ ARROW_BUILD_TESTS=ON \ - ARROW_BUILD_TOOLCHAIN=$CONDA_PREFIX \ + ARROW_DEPENDENCY_SOURCE=CONDA \ ARROW_HOME=$CONDA_PREFIX \ PARQUET_HOME=$CONDA_PREFIX diff --git a/cpp/Dockerfile.alpine b/cpp/Dockerfile.alpine index 4f04d876d23..9cef4389856 100644 --- a/cpp/Dockerfile.alpine +++ b/cpp/Dockerfile.alpine @@ -17,12 +17,14 @@ FROM alpine -# install dependencies +# Install basic build dependencies +# grpc requires libnsl-dev to be present, this cannot be installed via a bundle. RUN apk add --no-cache -q \ autoconf \ bash \ bison \ boost-dev \ + ccache \ cmake \ flex \ g++ \ @@ -30,16 +32,20 @@ RUN apk add --no-cache -q \ git \ gzip \ make \ + libnsl-dev \ musl-dev \ ninja \ openssl-dev \ wget \ zlib-dev +# Ganidva is deactivated as we don't support building LLVM via ExternalProject +# and Alpine only has LLVM 6 in its repositories yet. ENV CC=gcc \ CXX=g++ \ - ARROW_ORC=OFF \ - ARROW_PARQUET=OFF \ + ARROW_GANDIVA=OFF \ + ARROW_ORC=ON \ + ARROW_PARQUET=ON \ ARROW_BUILD_TESTS=ON \ ARROW_HOME=/usr/local diff --git a/cpp/Dockerfile.debian-testing b/cpp/Dockerfile.debian-testing new file mode 100644 index 00000000000..1b4729ea622 --- /dev/null +++ b/cpp/Dockerfile.debian-testing @@ -0,0 +1,76 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +FROM debian:testing + +# install build essentials +RUN export DEBIAN_FRONTEND=noninteractive && \ + apt-get update -y -q && \ + apt-get install -y -q --no-install-recommends \ + autoconf \ + ca-certificates \ + ccache \ + clang-7 \ + cmake \ + g++ \ + gcc \ + git \ + libbenchmark-dev \ + libboost-all-dev \ + libbrotli-dev \ + libbz2-dev \ + libc-ares-dev \ + libdouble-conversion-dev \ + libgflags-dev \ + libgmock-dev \ + libgoogle-glog-dev \ + libgtest-dev \ + liblz4-dev \ + libprotobuf-dev \ + libprotoc-dev \ + libre2-dev \ + libsnappy-dev \ + libssl-dev \ + libthrift-dev \ + libzstd-dev \ + llvm-7-dev \ + make \ + ninja-build \ + openjdk-11-jdk \ + openjdk-11-jdk-headless \ + pkg-config \ + protobuf-compiler \ + rapidjson-dev \ + thrift-compiler \ + tzdata \ + wget + +ENV CC=gcc \ + CXX=g++ \ + ARROW_BUILD_TESTS=ON \ + ARROW_DEPENDENCY_SOURCE=SYSTEM \ + ARROW_FLIGHT=ON \ + ARROW_GANDIVA=ON \ + ARROW_GANDIVA_JAVA=ON \ + ARROW_HOME=/usr \ + JAVA_HOME=/usr/lib/jvm/java-11-openjdk-amd64 \ + CMAKE_ARGS="-DFlatbuffers_SOURCE=BUNDLED -Dc-ares_SOURCE=BUNDLED -DgRPC_SOURCE=BUNDLED" + +# build and test +CMD arrow/ci/docker_build_cpp.sh && \ + cd /build/cpp && ctest -j2 --output-on-failure -L unittest + diff --git a/cpp/Dockerfile.fedora b/cpp/Dockerfile.fedora new file mode 100644 index 00000000000..d9da9f716fe --- /dev/null +++ b/cpp/Dockerfile.fedora @@ -0,0 +1,76 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +FROM fedora:29 + +# install dependencies +RUN dnf update -y && \ + dnf install -y \ + autoconf \ + bison \ + boost-devel \ + brotli-devel \ + bzip2-devel \ + c-ares-devel \ + ccache \ + clang-devel \ + cmake \ + double-conversion-devel \ + flatbuffers-devel \ + flex \ + java-openjdk-devel \ + java-openjdk-headless \ + gcc \ + gcc-c++ \ + glog-devel \ + gflags-devel \ + gtest-devel \ + gmock-devel \ + google-benchmark-devel \ + git \ + libzstd-devel \ + llvm-devel \ + llvm-static \ + lz4-devel \ + make \ + ninja-build \ + openssl-devel \ + protobuf-compiler \ + protobuf-devel \ + python \ + rapidjson-devel \ + re2-devel \ + snappy-devel \ + thrift-devel \ + zlib-devel + +# c-ares cmake config is not installed on Fedora but gRPC needs it +# when built via ExternalProject: https://bugzilla.redhat.com/show_bug.cgi?id=1687844 +ENV CC=gcc \ + CXX=g++ \ + ARROW_ORC=ON \ + ARROW_DEPENDENCY_SOURCE=SYSTEM \ + ARROW_PARQUET=ON \ + ARROW_FLIGHT=OFF \ + ARROW_GANDIVA=ON \ + ARROW_GANDIVA_JAVA=ON \ + ARROW_BUILD_TESTS=ON \ + ARROW_HOME=/usr/local \ + CMAKE_ARGS="-DgRPC_SOURCE=BUNDLED" + +# build; tests don't work on Fedora at the moment due to compiler problems in gcc 8.2 (fixed in 8.3) +CMD arrow/ci/docker_build_cpp.sh diff --git a/cpp/Dockerfile.ubuntu-bionic b/cpp/Dockerfile.ubuntu-bionic new file mode 100644 index 00000000000..a7c5c445b35 --- /dev/null +++ b/cpp/Dockerfile.ubuntu-bionic @@ -0,0 +1,88 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +FROM ubuntu:bionic + +# install build essentials +RUN export DEBIAN_FRONTEND=noninteractive && \ + apt-get update -y -q && \ + apt-get install -y wget software-properties-common +RUN wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - +RUN apt-add-repository -y "deb http://apt.llvm.org/bionic/ llvm-toolchain-bionic-7 main" +RUN export DEBIAN_FRONTEND=noninteractive && \ + apt-get update -y -q && \ + apt-get install -y -q --no-install-recommends \ + autoconf \ + bison \ + ca-certificates \ + ccache \ + clang-7 \ + cmake \ + flex \ + g++ \ + gcc \ + git \ + libbenchmark-dev \ + libboost-all-dev \ + libbrotli-dev \ + libbz2-dev \ + libc-ares-dev \ + libdouble-conversion-dev \ + libgflags-dev \ + libgoogle-glog-dev \ + libgrpc-dev \ + libgrpc++-dev \ + liblz4-dev \ + libprotoc-dev \ + libprotobuf-dev \ + libre2-dev \ + libsnappy-dev \ + libssl-dev \ + libzstd-dev \ + llvm-7-dev \ + make \ + ninja-build \ + pkg-config \ + protobuf-compiler \ + protobuf-compiler-grpc \ + rapidjson-dev \ + tzdata + +# Ubuntu's gtest just provides sources, the compiled version is only available +# from Ubuntu Cosmic on. +# ARROW_GANDIVA_JAVA requires CMake 3.11 +# TODO: gRPC is too old on Bionic and c-ares CMake config is not installed thus +# we need to build both from source. +# protobuf does not come with PHP but grpc needs it to built, thus also +# built Protobuf from source: https://github.com/grpc/grpc/issues/15949 +ENV CC=gcc \ + CXX=g++ \ + ARROW_BUILD_BENCHMARKS=ON \ + ARROW_BUILD_TESTS=ON \ + ARROW_DEPENDENCY_SOURCE=SYSTEM \ + ARROW_FLIGHT=ON \ + ARROW_GANDIVA=ON \ + ARROW_GANDIVA_JAVA=OFF \ + ARROW_PARQUET=ON \ + ARROW_HOME=/usr \ + ARROW_WITH_ZSTD=ON \ + CMAKE_ARGS="-DThrift_SOURCE=BUNDLED -DFlatbuffers_SOURCE=BUNDLED -DGTest_SOURCE=BUNDLED -DgRPC_SOURCE=BUNDLED -Dc-ares_SOURCE=BUNDLED -DProtobuf_SOURCE=BUNDLED" + +# build and test +CMD arrow/ci/docker_build_cpp.sh && \ + cd /build/cpp && ctest -j2 --output-on-failure -L unittest + diff --git a/cpp/Dockerfile.ubuntu-xenial b/cpp/Dockerfile.ubuntu-xenial new file mode 100644 index 00000000000..8384bdc38fa --- /dev/null +++ b/cpp/Dockerfile.ubuntu-xenial @@ -0,0 +1,78 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +FROM ubuntu:xenial + +# install build essentials +RUN export DEBIAN_FRONTEND=noninteractive && \ + apt-get update -y -q && \ + apt-get install -y wget software-properties-common +RUN wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - +RUN apt-add-repository -y "deb http://apt.llvm.org/xenial/ llvm-toolchain-xenial-7 main" +RUN export DEBIAN_FRONTEND=noninteractive && \ + apt-get update -y -q && \ + apt-get install -y -q --no-install-recommends \ + autoconf \ + bison \ + ca-certificates \ + ccache \ + clang-7 \ + cmake \ + flex \ + g++ \ + gcc \ + git \ + libboost-all-dev \ + libbrotli-dev \ + libbz2-dev \ + libc-ares-dev \ + libdouble-conversion-dev \ + libgflags-dev \ + libgoogle-glog-dev \ + liblz4-dev \ + libprotobuf-dev \ + libre2-dev \ + libsnappy-dev \ + libssl-dev \ + libzstd-dev \ + llvm-7-dev \ + make \ + ninja-build \ + pkg-config \ + protobuf-compiler \ + tzdata + +# Benchmark is deactivated as the external project requires CMake 3.6+ +# Flight is deactivated as Ubuntu provides gflags but not the CMake config for it. +# Gandiva JNI is deactivated as it requires CMake 3.11+ +# TODO(ARROW-4761): libzstd is too old and external project requires CMake 3.7+ +ENV CC=gcc \ + CXX=g++ \ + ARROW_BUILD_BENCHMARKS=OFF \ + ARROW_BUILD_TESTS=ON \ + ARROW_DEPENDENCY_SOURCE=SYSTEM \ + ARROW_FLIGHT=OFF \ + ARROW_GANDIVA=ON \ + ARROW_GANDIVA_JAVA=OFF \ + ARROW_PARQUET=ON \ + ARROW_HOME=/usr \ + ARROW_WITH_ZSTD=OFF \ + CMAKE_ARGS="-DThrift_SOURCE=BUNDLED -DGTest_SOURCE=BUNDLED -DFlatbuffers_SOURCE=BUNDLED -DRapidJSON_SOURCE=BUNDLED" + +# build and test +CMD arrow/ci/docker_build_cpp.sh && \ + cd /build/cpp && ctest -j2 --output-on-failure -L unittest diff --git a/cpp/cmake_modules/BuildUtils.cmake b/cpp/cmake_modules/BuildUtils.cmake index 092b6556cc1..d64e35e432a 100644 --- a/cpp/cmake_modules/BuildUtils.cmake +++ b/cpp/cmake_modules/BuildUtils.cmake @@ -387,6 +387,18 @@ function(ADD_BENCHMARK REL_BENCHMARK_NAME) set(NO_COLOR "") endif() + # With OSX and conda, we need to set the correct RPATH so that dependencies + # are found. The installed libraries with conda have an RPATH that matches + # for executables and libraries lying in $ENV{CONDA_PREFIX}/bin or + # $ENV{CONDA_PREFIX}/lib but our test libraries and executables are not + # installed there. + if (NOT "$ENV{CONDA_PREFIX}" STREQUAL "" AND APPLE) + set_target_properties(${BENCHMARK_NAME} PROPERTIES + BUILD_WITH_INSTALL_RPATH TRUE + INSTALL_RPATH_USE_LINK_PATH TRUE + INSTALL_RPATH "$ENV{CONDA_PREFIX}/lib;${EXECUTABLE_OUTPUT_PATH}") + endif() + # Add test as dependency of relevant label targets add_dependencies(all-benchmarks ${BENCHMARK_NAME}) foreach (TARGET ${ARG_LABELS}) @@ -465,6 +477,18 @@ function(ADD_TEST_CASE REL_TEST_NAME) set(TEST_PATH "${EXECUTABLE_OUTPUT_PATH}/${TEST_NAME}") add_executable(${TEST_NAME} ${SOURCES}) + # With OSX and conda, we need to set the correct RPATH so that dependencies + # are found. The installed libraries with conda have an RPATH that matches + # for executables and libraries lying in $ENV{CONDA_PREFIX}/bin or + # $ENV{CONDA_PREFIX}/lib but our test libraries and executables are not + # installed there. + if (NOT "$ENV{CONDA_PREFIX}" STREQUAL "" AND APPLE) + set_target_properties(${TEST_NAME} PROPERTIES + BUILD_WITH_INSTALL_RPATH TRUE + INSTALL_RPATH_USE_LINK_PATH TRUE + INSTALL_RPATH "$ENV{CONDA_PREFIX}/lib;${EXECUTABLE_OUTPUT_PATH}") + endif() + if (ARG_STATIC_LINK_LIBS) # Customize link libraries target_link_libraries(${TEST_NAME} PRIVATE ${ARG_STATIC_LINK_LIBS}) diff --git a/cpp/cmake_modules/FindBrotli.cmake b/cpp/cmake_modules/FindBrotli.cmake index ea971f09ed8..56f877a83b3 100644 --- a/cpp/cmake_modules/FindBrotli.cmake +++ b/cpp/cmake_modules/FindBrotli.cmake @@ -16,112 +16,100 @@ # Usage of this module as follows: # # find_package(Brotli) -# -# Variables used by this module, they can change the default behaviour and need -# to be set before calling find_package: -# -# Brotli_HOME - When set, this path is inspected instead of standard library -# locations as the root of the Brotli installation. -# The environment variable BROTLI_HOME overrides this veriable. -# -# This module defines -# BROTLI_INCLUDE_DIR, directory containing headers -# BROTLI_LIBS, directory containing brotli libraries -# BROTLI_STATIC_LIB, path to libbrotli.a -# BROTLI_SHARED_LIB, path to libbrotli's shared library -# BROTLI_FOUND, whether brotli has been found - -if( NOT "${BROTLI_HOME}" STREQUAL "") - file( TO_CMAKE_PATH "${BROTLI_HOME}" _native_path ) - list( APPEND _brotli_roots ${_native_path} ) -elseif ( Brotli_HOME ) - list( APPEND _brotli_roots ${Brotli_HOME} ) -endif() -find_path( BROTLI_INCLUDE_DIR NAMES brotli/decode.h - PATHS ${_brotli_roots} - NO_DEFAULT_PATH - PATH_SUFFIXES "include" ) +pkg_check_modules(BROTLI_PC libbrotlicommon libbrotlienc libbrotlidec) +if (BROTLI_PC_FOUND) + set(BROTLI_INCLUDE_DIR "${BROTLI_PC_libbrotlicommon_INCLUDEDIR}") -find_library( BROTLI_LIBRARY_ENC NAMES libbrotlienc.a libbrotlienc-static.a brotlienc - PATHS ${_brotli_roots} - NO_DEFAULT_PATH - PATH_SUFFIXES "lib/${CMAKE_LIBRARY_ARCHITECTURE}" "lib" ) + # Some systems (e.g. Fedora) don't fill Brotli_LIBRARY_DIRS, so add the other dirs here. + list(APPEND BROTLI_PC_LIBRARY_DIRS "${BROTLI_PC_libbrotlicommon_LIBDIR}") + list(APPEND BROTLI_PC_LIBRARY_DIRS "${BROTLI_PC_libbrotlienc_LIBDIR}") + list(APPEND BROTLI_PC_LIBRARY_DIRS "${BROTLI_PC_libbrotlidec_LIBDIR}") -find_library( BROTLI_LIBRARY_DEC NAMES libbrotlidec.a libbrotlidec-static.a brotlidec - PATHS ${_brotli_roots} - NO_DEFAULT_PATH - PATH_SUFFIXES "lib/${CMAKE_LIBRARY_ARCHITECTURE}" "lib" ) - -find_library( BROTLI_LIBRARY_COMMON NAMES libbrotlicommon.a libbrotlicommon-static.a brotlicommon - PATHS ${_brotli_roots} - NO_DEFAULT_PATH - PATH_SUFFIXES "lib/${CMAKE_LIBRARY_ARCHITECTURE}" "lib" ) - -set(BROTLI_LIBRARIES ${BROTLI_LIBRARY_ENC} ${BROTLI_LIBRARY_DEC} - ${BROTLI_LIBRARY_COMMON}) - -if (BROTLI_INCLUDE_DIR AND (PARQUET_MINIMAL_DEPENDENCY OR BROTLI_LIBRARIES)) - set(BROTLI_FOUND TRUE) - get_filename_component( BROTLI_LIBS ${BROTLI_LIBRARY_ENC} PATH ) - set(BROTLI_LIB_NAME brotli) - if (BROTLI_MSVC_STATIC_LIB_SUFFIX) - set(BROTLI_STATIC_LIB_SUFFIX "${BROTLI_MSVC_STATIC_LIB_SUFFIX}") - endif() - if (NOT BROTLI_STATIC_LIB_SUFFIX) - if (EXISTS "${BROTLI_LIBS}/${CMAKE_STATIC_LIBRARY_PREFIX}${BROTLI_LIB_NAME}enc-static${CMAKE_STATIC_LIBRARY_SUFFIX}") - set(BROTLI_STATIC_LIB_SUFFIX -static) - else() - if (MSVC) - set(BROTLI_STATIC_LIB_SUFFIX _static) - else() - set(BROTLI_STATIC_LIB_SUFFIX "") - endif() - endif() - endif() - set(BROTLI_STATIC_LIB - ${BROTLI_LIBS}/${CMAKE_STATIC_LIBRARY_PREFIX}${BROTLI_LIB_NAME}enc${BROTLI_STATIC_LIB_SUFFIX}${CMAKE_STATIC_LIBRARY_SUFFIX} - ${BROTLI_LIBS}/${CMAKE_STATIC_LIBRARY_PREFIX}${BROTLI_LIB_NAME}dec${BROTLI_STATIC_LIB_SUFFIX}${CMAKE_STATIC_LIBRARY_SUFFIX} - ${BROTLI_LIBS}/${CMAKE_STATIC_LIBRARY_PREFIX}${BROTLI_LIB_NAME}common${BROTLI_STATIC_LIB_SUFFIX}${CMAKE_STATIC_LIBRARY_SUFFIX}) - set(BROTLI_STATIC_LIBRARY_ENC ${BROTLI_LIBS}/${CMAKE_STATIC_LIBRARY_PREFIX}${BROTLI_LIB_NAME}enc${BROTLI_STATIC_LIB_SUFFIX}${CMAKE_STATIC_LIBRARY_SUFFIX}) - set(BROTLI_STATIC_LIBRARY_DEC ${BROTLI_LIBS}/${CMAKE_STATIC_LIBRARY_PREFIX}${BROTLI_LIB_NAME}dec${BROTLI_STATIC_LIB_SUFFIX}${CMAKE_STATIC_LIBRARY_SUFFIX}) - set(BROTLI_STATIC_LIBRARY_COMMON ${BROTLI_LIBS}/${CMAKE_STATIC_LIBRARY_PREFIX}${BROTLI_LIB_NAME}common${BROTLI_STATIC_LIB_SUFFIX}${CMAKE_STATIC_LIBRARY_SUFFIX}) - set(BROTLI_SHARED_LIB - ${BROTLI_LIBS}/${CMAKE_SHARED_LIBRARY_PREFIX}${BROTLI_LIB_NAME}enc${CMAKE_SHARED_LIBRARY_SUFFIX} - ${BROTLI_LIBS}/${CMAKE_SHARED_LIBRARY_PREFIX}${BROTLI_LIB_NAME}dec${CMAKE_SHARED_LIBRARY_SUFFIX} - ${BROTLI_LIBS}/${CMAKE_SHARED_LIBRARY_PREFIX}${BROTLI_LIB_NAME}common${CMAKE_SHARED_LIBRARY_SUFFIX}) -else () - set(BROTLI_FOUND FALSE) -endif () + find_library(BROTLI_COMMON_LIBRARY brotlicommon PATHS ${BROTLI_PC_LIBRARY_DIRS} NO_DEFAULT_PATH) + find_library(BROTLI_ENC_LIBRARY brotlienc PATHS ${BROTLI_PC_LIBRARY_DIRS} NO_DEFAULT_PATH) + find_library(BROTLI_DEC_LIBRARY brotlidec PATHS ${BROTLI_PC_LIBRARY_DIRS} NO_DEFAULT_PATH) +elseif(BROTLI_ROOT) + find_library(BROTLI_COMMON_LIBRARY + NAMES + brotlicommon + ${CMAKE_SHARED_LIBRARY_PREFIX}brotlicommon${CMAKE_SHARED_LIBRARY_SUFFIX} + ${CMAKE_STATIC_LIBRARY_PREFIX}brotlicommon${CMAKE_STATIC_LIBRARY_SUFFIX} + ${CMAKE_STATIC_LIBRARY_PREFIX}brotlicommon-static${CMAKE_STATIC_LIBRARY_SUFFIX} + ${CMAKE_STATIC_LIBRARY_PREFIX}brotlicommon_static${CMAKE_STATIC_LIBRARY_SUFFIX} + PATHS ${BROTLI_ROOT} "${BROTLI_ROOT}/Library" + PATH_SUFFIXES "lib64" "lib" "bin" + NO_DEFAULT_PATH) + find_library(BROTLI_ENC_LIBRARY + NAMES + brotlienc + ${CMAKE_SHARED_LIBRARY_PREFIX}brotlienc${CMAKE_SHARED_LIBRARY_SUFFIX} + ${CMAKE_STATIC_LIBRARY_PREFIX}brotlienc${CMAKE_STATIC_LIBRARY_SUFFIX} + ${CMAKE_STATIC_LIBRARY_PREFIX}brotlienc-static${CMAKE_STATIC_LIBRARY_SUFFIX} + ${CMAKE_STATIC_LIBRARY_PREFIX}brotlienc_static${CMAKE_STATIC_LIBRARY_SUFFIX} + PATHS ${BROTLI_ROOT} "${BROTLI_ROOT}/Library" + PATH_SUFFIXES "lib64" "lib" "bin" + NO_DEFAULT_PATH) + find_library(BROTLI_DEC_LIBRARY + NAMES + brotlidec + ${CMAKE_SHARED_LIBRARY_PREFIX}brotlidec${CMAKE_SHARED_LIBRARY_SUFFIX} + ${CMAKE_STATIC_LIBRARY_PREFIX}brotlidec${CMAKE_STATIC_LIBRARY_SUFFIX} + ${CMAKE_STATIC_LIBRARY_PREFIX}brotlidec-static${CMAKE_STATIC_LIBRARY_SUFFIX} + ${CMAKE_STATIC_LIBRARY_PREFIX}brotlidec_static${CMAKE_STATIC_LIBRARY_SUFFIX} + PATHS ${BROTLI_ROOT} "${BROTLI_ROOT}/Library" + PATH_SUFFIXES "lib64" "lib" "bin" + NO_DEFAULT_PATH) + find_path(BROTLI_INCLUDE_DIR NAMES brotli/decode.h + PATHS ${BROTLI_ROOT} "${BROTLI_ROOT}/Library" + NO_DEFAULT_PATH + PATH_SUFFIXES "include") +else() + find_library(BROTLI_COMMON_LIBRARY + NAMES + brotlicommon + ${CMAKE_SHARED_LIBRARY_PREFIX}brotlicommon${CMAKE_SHARED_LIBRARY_SUFFIX} + ${CMAKE_STATIC_LIBRARY_PREFIX}brotlicommon${CMAKE_STATIC_LIBRARY_SUFFIX} + ${CMAKE_STATIC_LIBRARY_PREFIX}brotlicommon-static${CMAKE_STATIC_LIBRARY_SUFFIX} + ${CMAKE_STATIC_LIBRARY_PREFIX}brotlicommon_static${CMAKE_STATIC_LIBRARY_SUFFIX} + PATH_SUFFIXES "lib64" "lib" "bin") + find_library(BROTLI_ENC_LIBRARY + NAMES + brotlienc + ${CMAKE_SHARED_LIBRARY_PREFIX}brotlienc${CMAKE_SHARED_LIBRARY_SUFFIX} + ${CMAKE_STATIC_LIBRARY_PREFIX}brotlienc${CMAKE_STATIC_LIBRARY_SUFFIX} + ${CMAKE_STATIC_LIBRARY_PREFIX}brotlienc-static${CMAKE_STATIC_LIBRARY_SUFFIX} + ${CMAKE_STATIC_LIBRARY_PREFIX}brotlienc_static${CMAKE_STATIC_LIBRARY_SUFFIX} + PATH_SUFFIXES "lib64" "lib" "bin") + find_library(BROTLI_DEC_LIBRARY + NAMES + brotlidec + ${CMAKE_SHARED_LIBRARY_PREFIX}brotlidec${CMAKE_SHARED_LIBRARY_SUFFIX} + ${CMAKE_STATIC_LIBRARY_PREFIX}brotlidec${CMAKE_STATIC_LIBRARY_SUFFIX} + ${CMAKE_STATIC_LIBRARY_PREFIX}brotlidec-static${CMAKE_STATIC_LIBRARY_SUFFIX} + ${CMAKE_STATIC_LIBRARY_PREFIX}brotlidec_static${CMAKE_STATIC_LIBRARY_SUFFIX} + PATH_SUFFIXES "lib64" "lib" "bin") + find_path(BROTLI_INCLUDE_DIR NAMES brotli/decode.h + PATH_SUFFIXES "include") +endif() -if (BROTLI_FOUND) - if (NOT Brotli_FIND_QUIETLY) - if (PARQUET_MINIMAL_DEPENDENCY) - message(STATUS "Found the Brotli headers: ${BROTLI_INCLUDE_DIR}") - else () - message(STATUS "Found the Brotli library: ${BROTLI_LIBRARIES}") - endif () - endif () -else () - if (NOT Brotli_FIND_QUIETLY) - set(BROTLI_ERR_MSG "Could not find the Brotli library. Looked in ") - if ( _brotli_roots ) - set(BROTLI_ERR_MSG "${BROTLI_ERR_MSG} in ${_brotli_roots}.") - else () - set(BROTLI_ERR_MSG "${BROTLI_ERR_MSG} system search paths.") - endif () - if (Brotli_FIND_REQUIRED) - message(FATAL_ERROR "${BROTLI_ERR_MSG}") - else (Brotli_FIND_REQUIRED) - message(STATUS "${BROTLI_ERR_MSG}") - endif (Brotli_FIND_REQUIRED) - endif () -endif () +find_package_handle_standard_args(Brotli + REQUIRED_VARS BROTLI_COMMON_LIBRARY BROTLI_ENC_LIBRARY BROTLI_DEC_LIBRARY BROTLI_INCLUDE_DIR) -mark_as_advanced( - BROTLI_INCLUDE_DIR - BROTLI_LIBS - BROTLI_LIBRARIES - BROTLI_STATIC_LIB - BROTLI_SHARED_LIB -) +if (Brotli_FOUND) + add_library(Brotli::brotlicommon UNKNOWN IMPORTED) + set_target_properties(Brotli::brotlicommon PROPERTIES + IMPORTED_LOCATION "${BROTLI_COMMON_LIBRARY}" + INTERFACE_INCLUDE_DIRECTORIES "${BROTLI_INCLUDE_DIR}" + ) + add_library(Brotli::brotlienc UNKNOWN IMPORTED) + set_target_properties(Brotli::brotlienc PROPERTIES + IMPORTED_LOCATION "${BROTLI_ENC_LIBRARY}" + INTERFACE_INCLUDE_DIRECTORIES "${BROTLI_INCLUDE_DIR}" + ) + add_library(Brotli::brotlidec UNKNOWN IMPORTED) + set_target_properties(Brotli::brotlidec PROPERTIES + IMPORTED_LOCATION "${BROTLI_DEC_LIBRARY}" + INTERFACE_INCLUDE_DIRECTORIES "${BROTLI_INCLUDE_DIR}" + ) +endif() diff --git a/cpp/cmake_modules/FindBz2.cmake b/cpp/cmake_modules/FindBz2.cmake deleted file mode 100644 index c26b8b7241e..00000000000 --- a/cpp/cmake_modules/FindBz2.cmake +++ /dev/null @@ -1,57 +0,0 @@ -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# Tries to find bz2 headers and libraries. -# -# Usage of this module as follows: -# -# find_package(Bz2) -# -# Variables used by this module, they can change the default behaviour and need -# to be set before calling find_package: -# -# BZ2_HOME - When set, this path is inspected instead of standard library -# locations as the root of the bz2 installation. -# -# This module defines -# BZ2_INCLUDE_DIR, directory containing headers -# BZ2_STATIC_LIB, path to libbz2 static library -# BZ2_FOUND, whether bz2 has been found - -if( NOT "${BZ2_HOME}" STREQUAL "") - file( TO_CMAKE_PATH "${BZ2_HOME}" _native_path ) - list( APPEND _bz2_roots ${_native_path} ) -elseif ( BZ2_HOME ) - list( APPEND _bz2_roots ${BZ2_HOME} ) -endif() - -if (MSVC) - set(BZ2_STATIC_LIB_NAME - ${CMAKE_STATIC_LIBRARY_PREFIX}bz2_static${CMAKE_STATIC_LIBRARY_SUFFIX}) -else() - set(BZ2_STATIC_LIB_NAME - ${CMAKE_STATIC_LIBRARY_PREFIX}bz2${CMAKE_STATIC_LIBRARY_SUFFIX}) -endif() - -find_path(BZ2_INCLUDE_DIR NAMES bzlib.h - PATHS ${_bz2_roots} - NO_DEFAULT_PATH - PATH_SUFFIXES "include" ) -find_library(BZ2_STATIC_LIB NAMES ${BZ2_STATIC_LIB_NAME} lib${BZ2_STATIC_LIB_NAME} - PATHS ${_bz2_roots} - NO_DEFAULT_PATH - PATH_SUFFIXES "lib" ) - -include(FindPackageHandleStandardArgs) -find_package_handle_standard_args(BZ2 REQUIRED_VARS - BZ2_STATIC_LIB BZ2_INCLUDE_DIR) diff --git a/cpp/cmake_modules/FindDoubleConversion.cmake b/cpp/cmake_modules/FindDoubleConversion.cmake new file mode 100644 index 00000000000..9097a315254 --- /dev/null +++ b/cpp/cmake_modules/FindDoubleConversion.cmake @@ -0,0 +1,42 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +if(DoubleConversion_ROOT) + find_library(DoubleConversion_LIB + NAMES double-conversion + PATHS ${DoubleConversion_ROOT} + NO_DEFAULT_PATH) + find_path(DoubleConversion_INCLUDE_DIR NAMES double-conversion/double-conversion.h + PATHS ${DoubleConversion_ROOT} NO_DEFAULT_PATH + PATH_SUFFIXES "include") +else() + find_library(DoubleConversion_LIB + NAMES double-conversion) + find_path(DoubleConversion_INCLUDE_DIR NAMES double-conversion/double-conversion.h + PATH_SUFFIXES "include") +endif() + +find_package_handle_standard_args(DoubleConversion + REQUIRED_VARS DoubleConversion_LIB DoubleConversion_INCLUDE_DIR) + +if (DoubleConversion_FOUND) + add_library(double-conversion::double-conversion UNKNOWN IMPORTED) + set_target_properties(double-conversion::double-conversion PROPERTIES + IMPORTED_LOCATION "${DoubleConversion_LIB}" + INTERFACE_INCLUDE_DIRECTORIES "${DoubleConversion_INCLUDE_DIR}" + ) +endif() diff --git a/cpp/cmake_modules/FindFlatbuffers.cmake b/cpp/cmake_modules/FindFlatbuffers.cmake deleted file mode 100644 index f97286fc8e6..00000000000 --- a/cpp/cmake_modules/FindFlatbuffers.cmake +++ /dev/null @@ -1,95 +0,0 @@ -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# Tries to find Flatbuffers headers and libraries. -# -# Usage of this module as follows: -# -# find_package(Flatbuffers) -# -# Variables used by this module, they can change the default behaviour and need -# to be set before calling find_package: -# -# Flatbuffers_HOME - -# When set, this path is inspected instead of standard library locations as -# the root of the Flatbuffers installation. The environment variable -# FLATBUFFERS_HOME overrides this veriable. -# -# This module defines -# FLATBUFFERS_INCLUDE_DIR, directory containing headers -# FLATBUFFERS_LIBS, directory containing flatbuffers libraries -# FLATBUFFERS_STATIC_LIB, path to libflatbuffers.a -# FLATBUFFERS_FOUND, whether flatbuffers has been found - -if( NOT "${FLATBUFFERS_HOME}" STREQUAL "") - file( TO_CMAKE_PATH "${FLATBUFFERS_HOME}" _native_path ) - list( APPEND _flatbuffers_roots "${_native_path}" ) -elseif ( Flatbuffers_HOME ) - list( APPEND _flatbuffers_roots "${Flatbuffers_HOME}" ) -endif() - -# Try the parameterized roots, if they exist -if ( _flatbuffers_roots ) - find_path( FLATBUFFERS_INCLUDE_DIR NAMES flatbuffers/flatbuffers.h - PATHS "${_flatbuffers_roots}" NO_DEFAULT_PATH - PATH_SUFFIXES "include" ) - find_library( FLATBUFFERS_LIBRARIES NAMES flatbuffers - PATHS "${_flatbuffers_roots}" NO_DEFAULT_PATH - PATH_SUFFIXES "lib" "lib64" "lib/x86_64-linux-gnu/") -else () - find_path( FLATBUFFERS_INCLUDE_DIR NAMES flatbuffers/flatbuffers.h ) - find_library( FLATBUFFERS_LIBRARIES NAMES flatbuffers ) -endif () - -find_program(FLATBUFFERS_COMPILER flatc - "${FLATBUFFERS_HOME}/bin" - /usr/local/bin - /usr/bin - NO_DEFAULT_PATH -) - -if (FLATBUFFERS_INCLUDE_DIR AND FLATBUFFERS_LIBRARIES) - set(FLATBUFFERS_FOUND TRUE) - get_filename_component( FLATBUFFERS_LIBS "${FLATBUFFERS_LIBRARIES}" PATH ) - set(FLATBUFFERS_LIB_NAME libflatbuffers) - set(FLATBUFFERS_STATIC_LIB "${FLATBUFFERS_LIBS}/${FLATBUFFERS_LIB_NAME}.a") -else () - set(FLATBUFFERS_FOUND FALSE) -endif () - -if (FLATBUFFERS_FOUND) - if (NOT Flatbuffers_FIND_QUIETLY) - message(STATUS "Found the Flatbuffers library: ${FLATBUFFERS_LIBRARIES}") - endif () -else () - if (NOT Flatbuffers_FIND_QUIETLY) - set(FLATBUFFERS_ERR_MSG "Could not find the Flatbuffers library. Looked in ") - if ( _flatbuffers_roots ) - set(FLATBUFFERS_ERR_MSG "${FLATBUFFERS_ERR_MSG} ${_flatbuffers_roots}.") - else () - set(FLATBUFFERS_ERR_MSG "${FLATBUFFERS_ERR_MSG} system search paths.") - endif () - if (Flatbuffers_FIND_REQUIRED) - message(FATAL_ERROR "${FLATBUFFERS_ERR_MSG}") - else (Flatbuffers_FIND_REQUIRED) - message(STATUS "${FLATBUFFERS_ERR_MSG}") - endif (Flatbuffers_FIND_REQUIRED) - endif () -endif () - -mark_as_advanced( - FLATBUFFERS_INCLUDE_DIR - FLATBUFFERS_LIBS - FLATBUFFERS_STATIC_LIB - FLATBUFFERS_COMPILER -) diff --git a/cpp/cmake_modules/FindGBenchmark.cmake b/cpp/cmake_modules/FindGBenchmark.cmake deleted file mode 100644 index 3e46a60f5e6..00000000000 --- a/cpp/cmake_modules/FindGBenchmark.cmake +++ /dev/null @@ -1,88 +0,0 @@ -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# Tries to find Google benchmark headers and libraries. -# -# Usage of this module as follows: -# -# find_package(GBenchark) -# -# Variables used by this module, they can change the default behaviour and need -# to be set before calling find_package: -# -# GBenchmark_HOME - When set, this path is inspected instead of standard library -# locations as the root of the benchark installation. -# The environment variable GBENCHMARK_HOME overrides this veriable. -# -# This module defines -# GBENCHMARK_INCLUDE_DIR, directory containing benchmark header directory -# GBENCHMARK_LIBS, directory containing benchmark libraries -# GBENCHMARK_STATIC_LIB, path to libbenchmark.a -# GBENCHMARK_FOUND, whether gbenchmark has been found - -if( NOT "$ENV{GBENCHMARK_HOME}" STREQUAL "") - file( TO_CMAKE_PATH "$ENV{GBENCHMARK_HOME}" _native_path ) - list( APPEND _gbenchmark_roots ${_native_path} ) -elseif ( GBenchmark_HOME ) - list( APPEND _gbenchmark_roots ${GBenchmark_HOME} ) -endif() - -# Try the parameterized roots, if they exist -if ( _gbenchmark_roots ) - find_path( GBENCHMARK_INCLUDE_DIR NAMES benchmark/benchmark.h - PATHS ${_gbenchmark_roots} NO_DEFAULT_PATH - PATH_SUFFIXES "include" ) - find_library( GBENCHMARK_LIBRARIES NAMES benchmark - PATHS ${_gbenchmark_roots} NO_DEFAULT_PATH - PATH_SUFFIXES "lib" ) -else () - find_path( GBENCHMARK_INCLUDE_DIR NAMES benchmark/benchmark.hh ) - find_library( GBENCHMARK_LIBRARIES NAMES benchmark ) -endif () - - -if (GBENCHMARK_INCLUDE_DIR AND GBENCHMARK_LIBRARIES) - set(GBENCHMARK_FOUND TRUE) - get_filename_component( GBENCHMARK_LIBS ${GBENCHMARK_LIBRARIES} PATH ) - set(GBENCHMARK_LIB_NAME libbenchmark) - set(GBENCHMARK_STATIC_LIB ${GBENCHMARK_LIBS}/${GBENCHMARK_LIB_NAME}.a) -else () - set(GBENCHMARK_FOUND FALSE) -endif () - -if (GBENCHMARK_FOUND) - if (NOT GBenchmark_FIND_QUIETLY) - message(STATUS "Found the GBenchmark library: ${GBENCHMARK_LIBRARIES}") - endif () -else () - if (NOT GBenchmark_FIND_QUIETLY) - set(GBENCHMARK_ERR_MSG "Could not find the GBenchmark library. Looked in ") - if ( _gbenchmark_roots ) - set(GBENCHMARK_ERR_MSG "${GBENCHMARK_ERR_MSG} in ${_gbenchmark_roots}.") - else () - set(GBENCHMARK_ERR_MSG "${GBENCHMARK_ERR_MSG} system search paths.") - endif () - if (GBenchmark_FIND_REQUIRED) - message(FATAL_ERROR "${GBENCHMARK_ERR_MSG}") - else (GBenchmark_FIND_REQUIRED) - message(STATUS "${GBENCHMARK_ERR_MSG}") - endif (GBenchmark_FIND_REQUIRED) - endif () -endif () - -mark_as_advanced( - GBENCHMARK_INCLUDE_DIR - GBENCHMARK_LIBS - GBENCHMARK_LIBRARIES - GBENCHMARK_STATIC_LIB -) diff --git a/cpp/cmake_modules/FindGFlags.cmake b/cpp/cmake_modules/FindGFlags.cmake deleted file mode 100644 index 3a049ec77fd..00000000000 --- a/cpp/cmake_modules/FindGFlags.cmake +++ /dev/null @@ -1,142 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# - Find GFLAGS (gflags.h, libgflags.a, libgflags.so, and libgflags.so.0) -# This module defines -# GFLAGS_INCLUDE_DIR, directory containing headers -# GFLAGS_SHARED_LIB, path to libgflags shared library -# GFLAGS_STATIC_LIB, path to libgflags static library -# GFLAGS_FOUND, whether gflags has been found - -find_package(gflags CONFIG) -set(GFLAGS_FOUND ${gflags_FOUND}) -if(GFLAGS_FOUND) - message(STATUS "GFlags include dir: ${GFLAGS_INCLUDE_DIR}") - string(TOUPPER "${CMAKE_BUILD_TYPE}" _CONFIG) - get_target_property(_GFLAGS_CONFIGURATIONS gflags IMPORTED_CONFIGURATIONS) - foreach(_GFLAGS_CONFIG IN LISTS _GFLAGS_CONFIGURATIONS) - if("${_GFLAGS_CONFIG}" STREQUAL "${_CONFIG}") - set(_GFLAGS_TARGET_CONFIG "${_GFLAGS_CONFIG}") - endif() - endforeach() - if(NOT _GFLAGS_TARGET_CONFIG) - list(GET _GFLAGS_CONFIGURATIONS 0 _GFLAGS_TARGET_CONFIG) - endif() - if(GFLAGS_SHARED) - get_target_property(GFLAGS_SHARED_LIB gflags_shared - IMPORTED_LOCATION_${_GFLAGS_TARGET_CONFIG}) - message(STATUS "GFlags shared library: ${GFLAGS_SHARED_LIB}") - endif() - if(TARGET gflags_static) - set(GFLAGS_STATIC TRUE) - get_target_property(GFLAGS_STATIC_LIB gflags_static - IMPORTED_LOCATION_${_GFLAGS_TARGET_CONFIG}) - message(STATUS "GFlags static library: ${GFLAGS_STATIC_LIB}") - endif() - unset(_CONFIG) - unset(_GFLAGS_CONFIGURATIONS) - unset(_GFLAGS_CONFIG) - unset(_GFLAGS_TARGET_CONFIG) - return() -endif() - -pkg_check_modules(GFLAGS gflags) -if(GFLAGS_FOUND) - set(GFLAGS_INCLUDE_DIR "${GFLAGS_INCLUDEDIR}") - message(STATUS "GFlags include dir: ${GFLAGS_INCLUDE_DIR}") - find_library(GFLAGS_SHARED_LIB - NAMES "${GFLAGS_LIBRARIES}" - PATHS "${GFLAGS_LIBDIR}" - NO_DEFAULT_PATH) - if(GFLAGS_SHARED_LIB) - message(STATUS "GFlags shared library: ${GFLAGS_SHARED_LIB}") - add_thirdparty_lib(gflags - SHARED_LIB - "${GFLAGS_SHARED_LIB}" - INCLUDE_DIRECTORIES - "${GFLAGS_INCLUDE_DIR}") - target_compile_definitions(gflags_shared INTERFACE "GFLAGS_IS_A_DLL=1") - set(GFLAGS_STATIC_LIB FLASE) - return() - else() - set(GFLAGS_FOUND FALSE) - endif() -endif() - -if(GFLAGS_HOME) - set(GFlags_ROOT "${GFLAGS_HOME}") -endif() -if(CMAKE_VERSION VERSION_LESS 3.12) - list(INSERT CMAKE_PREFIX_PATH 0 "${GFlags_ROOT}") -endif() - -if(MSVC AND NOT DEFINED GFLAGS_MSVC_STATIC_LIB_SUFFIX) - set(GFLAGS_MSVC_STATIC_LIB_SUFFIX "_static") -endif() - -set(GFLAGS_STATIC_LIB_SUFFIX - "${GFLAGS_MSVC_STATIC_LIB_SUFFIX}${CMAKE_STATIC_LIBRARY_SUFFIX}") - -set(GFLAGS_STATIC_LIB_NAME - "${CMAKE_STATIC_LIBRARY_PREFIX}gflags${GFLAGS_STATIC_LIB_SUFFIX}") - -find_path(GFLAGS_INCLUDE_DIR NAMES gflags/gflags.h) -find_library(GFLAGS_SHARED_LIB NAMES gflags) -find_library(GFLAGS_STATIC_LIB NAMES ${GFLAGS_STATIC_LIB_NAME}) - -if(GFLAGS_INCLUDE_DIR) - message(STATUS "GFlags include dir: ${GFLAGS_INCLUDE_DIR}") - if(GFLAGS_SHARED_LIB) - message(STATUS "GFlags shared library: ${GFLAGS_SHARED_LIB}") - set(GFLAGS_SHARED TRUE) - else() - set(GFLAGS_SHARED FALSE) - endif() - if(GFLAGS_STATIC_LIB) - message(STATUS "GFlags static library: ${GFLAGS_STATIC_LIB}") - set(GFLAGS_STATIC TRUE) - else() - set(GFLAGS_STATIC FALSE) - endif() - if(GFLAGS_SHARED OR GFLAGS_STATIC) - set(GFLAGS_FOUND TRUE) - if(MSVC) - set(GFLAGS_MSVC_DEPS "shlwapi.lib") - endif() - add_thirdparty_lib(gflags - SHARED_LIB - "${GFLAGS_SHARED_LIB}" - STATIC_LIB - "${GFLAGS_STATIC_LIB}" - INCLUDE_DIRECTORIES - "${GFLAGS_INCLUDE_DIR}" - DEPS - "${GFLAGS_MSVC_DEPS}") - if(GFLAGS_SHARED) - target_compile_definitions(gflags_shared INTERFACE "GFLAGS_IS_A_DLL=1") - endif() - if(GFLAGS_STATIC) - target_compile_definitions(gflags_static INTERFACE "GFLAGS_IS_A_DLL=0") - endif() - else() - set(GFLAGS_FOUND FALSE) - endif() -else() - set(GFLAGS_FOUND FALSE) -endif() - -mark_as_advanced(GFLAGS_INCLUDE_DIR GFLAGS_SHARED_LIB GFLAGS_STATIC_LIB) diff --git a/cpp/cmake_modules/FindGLOG.cmake b/cpp/cmake_modules/FindGLOG.cmake index f22e4df70dd..e3a1bce6683 100644 --- a/cpp/cmake_modules/FindGLOG.cmake +++ b/cpp/cmake_modules/FindGLOG.cmake @@ -16,77 +16,41 @@ # Usage of this module as follows: # # find_package(GLOG) -# -# Variables used by this module, they can change the default behaviour and need -# to be set before calling find_package: -# -# GLOG_HOME - When set, this path is inspected instead of standard library -# locations as the root of the GLog installation. -# The environment variable GLOG_HOME overrides this veriable. -# -# This module defines -# GLOG_INCLUDE_DIR, directory containing headers -# GLOG_STATIC_LIB, path to libglog.a -# GLOG_FOUND, whether glog has been found -if( NOT "${GLOG_HOME}" STREQUAL "") - file( TO_CMAKE_PATH "${GLOG_HOME}" _native_path ) - list( APPEND _glog_roots ${_native_path} ) +pkg_check_modules(GLOG_PC libglog) +if (GLOG_PC_FOUND) + set(GLOG_INCLUDE_DIR "${GLOG_PC_INCLUDEDIR}") + list(APPEND GLOG_PC_LIBRARY_DIRS "${GLOG_PC_LIBDIR}") + find_library(GLOG_LIB glog + PATHS ${GLOG_PC_LIBRARY_DIRS} + NO_DEFAULT_PATH) +elseif(GLOG_ROOT) + find_library(GLOG_LIB + NAMES + glog + PATHS ${GLOG_ROOT} "${GLOG_ROOT}/Library" + PATH_SUFFIXES "lib64" "lib" "bin" + NO_DEFAULT_PATH) + find_path(GLOG_INCLUDE_DIR NAMES glog/logging.h + PATHS ${GLOG_ROOT} "${GLOG_ROOT}/Library" + NO_DEFAULT_PATH + PATH_SUFFIXES "include") +else() + find_library(GLOG_LIB + NAMES + glog + PATH_SUFFIXES "lib64" "lib" "bin") + find_path(GLOG_INCLUDE_DIR NAMES glog/logging.h + PATH_SUFFIXES "include") endif() -message(STATUS "GLOG_HOME: ${GLOG_HOME}") -# Try the parameterized roots, if they exist -if ( _glog_roots ) - set (lib_dirs "lib") - if (EXISTS "${_glog_roots}/lib64") - set (lib_dirs "lib64" ${lib_dirs}) - endif () - - find_path( GLOG_INCLUDE_DIR NAMES glog/logging.h - PATHS ${_glog_roots} NO_DEFAULT_PATH - PATH_SUFFIXES "include" ) - find_library( GLOG_LIBRARIES NAMES glog - PATHS ${_glog_roots} NO_DEFAULT_PATH - PATH_SUFFIXES ${lib_dirs}) -else () - find_path( GLOG_INCLUDE_DIR NAMES glog/logging.h ) - find_library( GLOG_LIBRARIES NAMES glog ) -endif () - - -if (GLOG_INCLUDE_DIR AND GLOG_LIBRARIES) - set(GLOG_FOUND TRUE) - get_filename_component( GLOG_LIBS ${GLOG_LIBRARIES} PATH ) - set(GLOG_LIB_NAME glog) - set(GLOG_STATIC_LIB ${GLOG_LIBS}/${CMAKE_STATIC_LIBRARY_PREFIX}${GLOG_LIB_NAME}${CMAKE_STATIC_LIBRARY_SUFFIX}) - set(GLOG_SHARED_LIB ${GLOG_LIBS}/${CMAKE_SHARED_LIBRARY_PREFIX}${GLOG_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX}) -else () - set(GLOG_FOUND FALSE) -endif () +find_package_handle_standard_args(GLOG + REQUIRED_VARS GLOG_INCLUDE_DIR GLOG_LIB) if (GLOG_FOUND) - if (NOT GLOG_FIND_QUIETLY) - message(STATUS "Found the GLog library: ${GLOG_LIBRARIES}") - endif () -else () - if (NOT GLOG_FIND_QUIETLY) - set(GLOG_ERR_MSG "Could not find the GLog library. Looked in ") - if ( _glog_roots ) - set(GLOG_ERR_MSG "${GLOG_ERR_MSG} ${_glog_roots}.") - else () - set(GLOG_ERR_MSG "${GLOG_ERR_MSG} system search paths.") - endif () - if (GLOG_FIND_REQUIRED) - message(FATAL_ERROR "${GLOG_ERR_MSG}") - else (GLOG_FIND_REQUIRED) - message(STATUS "${GLOG_ERR_MSG}") - endif (GLOG_FIND_REQUIRED) - endif () -endif () - -mark_as_advanced( - GLOG_INCLUDE_DIR - GLOG_LIBS - GLOG_LIBRARIES - GLOG_STATIC_LIB -) + add_library(GLOG::glog UNKNOWN IMPORTED) + set_target_properties(GLOG::glog PROPERTIES + IMPORTED_LOCATION "${GLOG_LIB}" + INTERFACE_INCLUDE_DIRECTORIES "${GLOG_INCLUDE_DIR}" + ) +endif() diff --git a/cpp/cmake_modules/FindGTest.cmake b/cpp/cmake_modules/FindGTest.cmake index 6ddb14aa6fb..4086dd9ec34 100644 --- a/cpp/cmake_modules/FindGTest.cmake +++ b/cpp/cmake_modules/FindGTest.cmake @@ -1,4 +1,3 @@ -# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at @@ -11,200 +10,268 @@ # See the License for the specific language governing permissions and # limitations under the License. # -# Tries to find GTest headers and libraries. -# -# Usage of this module as follows: -# -# find_package(GTest) -# -# Variables used by this module, they can change the default behaviour and need -# to be set before calling find_package: +# Originally imported from the CMake project at commit +# df4ed1e9ffcdb6b99ccff9e6f44808fdd2abda56 with the following license header: # -# GTest_HOME - When set, this path is inspected instead of standard library -# locations as the root of the GTest/Gmock installation. -# The environment variable GTEST_HOME overrides this variable. -# -# This module defines -# GTEST_INCLUDE_DIR, directory containing headers -# GTEST_LIBS, directory containing gtest libraries -# GTEST_STATIC_LIB, path to libgtest.a -# GTEST_STATIC_MAIN_LIB, path to libgtest_main.a -# GTEST_SHARED_LIB, path to libgtest's shared library -# GTEST_SHARED_MAIN_LIB, path to libgtest_main's shared library -# GTEST_FOUND, whether gtest has been found +# Distributed under the OSI-approved BSD 3-Clause License. See accompanying +# file Copyright.txt or https://cmake.org/licensing for details. + + +#[=======================================================================[.rst: +FindGTest +--------- + +Locate the Google C++ Testing Framework. + +Imported targets +^^^^^^^^^^^^^^^^ + +This module defines the following :prop_tgt:`IMPORTED` targets: + +``GTest::GTest`` + The Google Test ``gtest`` library, if found; adds Thread::Thread + automatically +``GTest::Main`` + The Google Test ``gtest_main`` library, if found +``GMock::GMock`` + The Google Mock ``gmock`` library, if found + + +Result variables +^^^^^^^^^^^^^^^^ + +This module will set the following variables in your project: + +``GTEST_FOUND`` + Found the Google Testing framework +``GTEST_INCLUDE_DIRS`` + the directory containing the Google Test headers + +The library variables below are set as normal variables. These +contain debug/optimized keywords when a debugging library is found. + +``GTEST_LIBRARIES`` + The Google Test ``gtest`` library; note it also requires linking + with an appropriate thread library +``GTEST_MAIN_LIBRARIES`` + The Google Test ``gtest_main`` library +``GTEST_BOTH_LIBRARIES`` + Both ``gtest`` and ``gtest_main`` + +Cache variables +^^^^^^^^^^^^^^^ + +The following cache variables may also be set: + +``GTEST_ROOT`` + The root directory of the Google Test installation (may also be + set as an environment variable) +``GTEST_MSVC_SEARCH`` + If compiling with MSVC, this variable can be set to ``MT`` or + ``MD`` (the default) to enable searching a GTest build tree + + +Example usage +^^^^^^^^^^^^^ + +:: + + enable_testing() + find_package(GTest REQUIRED) + + add_executable(foo foo.cc) + target_link_libraries(foo GTest::GTest GTest::Main) + + add_test(AllTestsInFoo foo) + + +Deeper integration with CTest +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +See :module:`GoogleTest` for information on the :command:`gtest_add_tests` +and :command:`gtest_discover_tests` commands. +#]=======================================================================] + +# include(${CMAKE_CURRENT_LIST_DIR}/GoogleTest.cmake) + +function(__gtest_append_debugs _endvar _library) + if(${_library} AND ${_library}_DEBUG) + set(_output optimized ${${_library}} debug ${${_library}_DEBUG}) + else() + set(_output ${${_library}}) + endif() + set(${_endvar} ${_output} PARENT_SCOPE) +endfunction() + +function(__gtest_find_library _name) + find_library(${_name} + NAMES ${ARGN} + HINTS + ENV GTEST_ROOT + ${GTEST_ROOT} + PATH_SUFFIXES ${_gtest_libpath_suffixes} + ) + mark_as_advanced(${_name}) +endfunction() + +macro(__gtest_determine_windows_library_type _var) + if(EXISTS "${${_var}}") + file(TO_NATIVE_PATH "${${_var}}" _lib_path) + get_filename_component(_name "${${_var}}" NAME_WE) + file(STRINGS "${${_var}}" _match REGEX "${_name}\\.dll" LIMIT_COUNT 1) + if(NOT _match STREQUAL "") + set(${_var}_TYPE SHARED PARENT_SCOPE) + else() + set(${_var}_TYPE UNKNOWN PARENT_SCOPE) + endif() + return() + endif() +endmacro() + +function(__gtest_determine_library_type _var) + if(WIN32) + # For now, at least, only Windows really needs to know the library type + __gtest_determine_windows_library_type(${_var}) + __gtest_determine_windows_library_type(${_var}_RELEASE) + __gtest_determine_windows_library_type(${_var}_DEBUG) + endif() + # If we get here, no determination was made from the above checks + set(${_var}_TYPE UNKNOWN PARENT_SCOPE) +endfunction() + +function(__gtest_import_library _target _var _config) + if(_config) + set(_config_suffix "_${_config}") + else() + set(_config_suffix "") + endif() + + set(_lib "${${_var}${_config_suffix}}") + if(EXISTS "${_lib}") + if(_config) + set_property(TARGET ${_target} APPEND PROPERTY + IMPORTED_CONFIGURATIONS ${_config}) + endif() + set_target_properties(${_target} PROPERTIES + IMPORTED_LINK_INTERFACE_LANGUAGES${_config_suffix} "CXX") + if(WIN32 AND ${_var}_TYPE STREQUAL SHARED) + set_target_properties(${_target} PROPERTIES + IMPORTED_IMPLIB${_config_suffix} "${_lib}") + else() + set_target_properties(${_target} PROPERTIES + IMPORTED_LOCATION${_config_suffix} "${_lib}") + endif() + endif() +endfunction() + # -# GMOCK_INCLUDE_DIR, directory containing headers -# GMOCK_LIBS, directory containing gmock libraries -# GMOCK_STATIC_LIB, path to libgmock.a -# GMOCK_STATIC_MAIN_LIB, path to libgmock_main.a -# GMOCK_SHARED_LIB, path to libgmock's shared library -# GMOCK_SHARED_MAIN_LIB, path to libgmock_main's shared library -# GMOCK_FOUND, whether gmock has been found - -if( NOT "${GTEST_HOME}" STREQUAL "") - file( TO_CMAKE_PATH "${GTEST_HOME}" _native_path ) - list( APPEND _gtest_roots ${_native_path} ) -elseif ( GTest_HOME ) - list( APPEND _gtest_roots ${GTest_HOME} ) -endif() -set(GTEST_STATIC_LIB_NAME - ${CMAKE_STATIC_LIBRARY_PREFIX}gtest${CMAKE_STATIC_LIBRARY_SUFFIX}) -set(GTEST_MAIN_STATIC_LIB_NAME - ${CMAKE_STATIC_LIBRARY_PREFIX}gtest_main${CMAKE_STATIC_LIBRARY_SUFFIX}) -set(GTEST_SHARED_LIB_NAME - ${CMAKE_SHARED_LIBRARY_PREFIX}gtest${CMAKE_SHARED_LIBRARY_SUFFIX}) -set(GTEST_MAIN_SHARED_LIB_NAME - ${CMAKE_SHARED_LIBRARY_PREFIX}gtest_main${CMAKE_SHARED_LIBRARY_SUFFIX}) -set(GMOCK_STATIC_LIB_NAME - ${CMAKE_STATIC_LIBRARY_PREFIX}gmock${CMAKE_STATIC_LIBRARY_SUFFIX}) -set(GMOCK_MAIN_STATIC_LIB_NAME - ${CMAKE_STATIC_LIBRARY_PREFIX}gmock_main${CMAKE_STATIC_LIBRARY_SUFFIX}) -set(GMOCK_SHARED_LIB_NAME - ${CMAKE_SHARED_LIBRARY_PREFIX}gmock${CMAKE_SHARED_LIBRARY_SUFFIX}) -set(GMOCK_MAIN_SHARED_LIB_NAME - ${CMAKE_SHARED_LIBRARY_PREFIX}gmock_main${CMAKE_SHARED_LIBRARY_SUFFIX}) - - -# Try the parameterized roots, if they exist -if(_gtest_roots) - find_path(GTEST_INCLUDE_DIR NAMES gtest/gtest.h - PATHS ${_gtest_roots} NO_DEFAULT_PATH - PATH_SUFFIXES "include") - set(lib_dirs - "lib/${CMAKE_LIBRARY_ARCHITECTURE}" - "lib64" - "lib") - find_library(GTEST_STATIC_LIB NAMES ${GTEST_STATIC_LIB_NAME} - PATHS ${_gtest_roots} NO_DEFAULT_PATH - PATH_SUFFIXES ${lib_dirs}) - find_library(GTEST_MAIN_STATIC_LIB NAMES ${GTEST_MAIN_STATIC_LIB_NAME} - PATHS ${_gtest_roots} NO_DEFAULT_PATH - PATH_SUFFIXES ${lib_dirs}) - find_library(GTEST_SHARED_LIB NAMES ${GTEST_SHARED_LIB_NAME} - PATHS ${_gtest_roots} NO_DEFAULT_PATH - PATH_SUFFIXES ${lib_dirs}) - find_library(GTEST_MAIN_SHARED_LIB NAMES ${GTEST_MAIN_SHARED_LIB_NAME} - PATHS ${_gtest_roots} NO_DEFAULT_PATH - PATH_SUFFIXES ${lib_dirs}) -else() - find_path(GTEST_INCLUDE_DIR NAMES gtest/gtest.h) - find_library(GTEST_STATIC_LIB NAMES ${GTEST_STATIC_LIB_NAME}) - find_library(GTEST_MAIN_STATIC_LIB NAMES ${GTEST_MAIN_STATIC_LIB_NAME}) - find_library(GTEST_SHARED_LIB NAMES ${GTEST_SHARED_LIB_NAME}) - find_library(GTEST_MAIN_SHARED_LIB NAMES ${GTEST_MAIN_SHARED_LIB_NAME}) +if(NOT DEFINED GTEST_MSVC_SEARCH) + set(GTEST_MSVC_SEARCH MD) endif() -# gmock -# Try the parameterized roots, if they exist (reuse gtest because the -# libraries should be built together). -if(_gtest_roots) - find_path(GMOCK_INCLUDE_DIR NAMES gmock/gmock.h - PATHS ${_gtest_roots} NO_DEFAULT_PATH - PATH_SUFFIXES "include") - set(lib_dirs - "lib/${CMAKE_LIBRARY_ARCHITECTURE}" - "lib64" - "lib") - find_library(GMOCK_STATIC_LIB NAMES ${GMOCK_STATIC_LIB_NAME} - PATHS ${_gtest_roots} NO_DEFAULT_PATH - PATH_SUFFIXES ${lib_dirs}) - find_library(GMOCK_MAIN_STATIC_LIB NAMES ${GMOCK_MAIN_STATIC_LIB_NAME} - PATHS ${_gtest_roots} NO_DEFAULT_PATH - PATH_SUFFIXES ${lib_dirs}) - find_library(GMOCK_SHARED_LIB NAMES ${GMOCK_SHARED_LIB_NAME} - PATHS ${_gtest_roots} NO_DEFAULT_PATH - PATH_SUFFIXES ${lib_dirs}) - find_library(GMOCK_MAIN_SHARED_LIB NAMES ${GMOCK_MAIN_SHARED_LIB_NAME} - PATHS ${_gtest_roots} NO_DEFAULT_PATH - PATH_SUFFIXES ${lib_dirs}) -else() - find_path(GMOCK_INCLUDE_DIR NAMES gmock/gmock.h) - find_library(GMOCK_STATIC_LIB NAMES ${GMOCK_STATIC_LIB_NAME}) - find_library(GMOCK_MAIN_STATIC_LIB NAMES ${GMOCK_MAIN_STATIC_LIB_NAME}) - find_library(GMOCK_SHARED_LIB NAMES ${GMOCK_SHARED_LIB_NAME}) - find_library(GMOCK_MAIN_SHARED_LIB NAMES ${GMOCK_MAIN_SHARED_LIB_NAME}) +set(_gtest_libpath_suffixes lib) +if(MSVC) + if(GTEST_MSVC_SEARCH STREQUAL "MD") + list(APPEND _gtest_libpath_suffixes + msvc/gtest-md/Debug + msvc/gtest-md/Release + msvc/x64/Debug + msvc/x64/Release + ) + elseif(GTEST_MSVC_SEARCH STREQUAL "MT") + list(APPEND _gtest_libpath_suffixes + msvc/gtest/Debug + msvc/gtest/Release + msvc/x64/Debug + msvc/x64/Release + ) + endif() endif() -if(GTEST_INCLUDE_DIR AND - (GTEST_STATIC_LIB AND GTEST_MAIN_STATIC_LIB) OR - (GTEST_SHARED_LIB AND GTEST_MAIN_SHARED_LIB)) - set(GTEST_FOUND TRUE) -else() - set(GTEST_FOUND FALSE) -endif() -if(GMOCK_INCLUDE_DIR AND - (GMOCK_STATIC_LIB AND GMOCK_MAIN_STATIC_LIB) OR - (GMOCK_SHARED_LIB AND GMOCK_MAIN_SHARED_LIB)) - set(GMOCK_FOUND TRUE) +find_path(GTEST_INCLUDE_DIR gtest/gtest.h + HINTS + $ENV{GTEST_ROOT}/include + ${GTEST_ROOT}/include +) +mark_as_advanced(GTEST_INCLUDE_DIR) + +if(MSVC AND GTEST_MSVC_SEARCH STREQUAL "MD") + # The provided /MD project files for Google Test add -md suffixes to the + # library names. + __gtest_find_library(GTEST_LIBRARY gtest-md gtest) + __gtest_find_library(GTEST_LIBRARY_DEBUG gtest-mdd gtestd) + __gtest_find_library(GTEST_MAIN_LIBRARY gtest_main-md gtest_main) + __gtest_find_library(GTEST_MAIN_LIBRARY_DEBUG gtest_main-mdd gtest_maind) + __gtest_find_library(GMOCK_LIBRARY gmock-md gmock) + __gtest_find_library(GMOCK_LIBRARY_DEBUG gmock-mdd gmockd) else() - set(GMOCK_FOUND FALSE) + __gtest_find_library(GTEST_LIBRARY gtest) + __gtest_find_library(GTEST_LIBRARY_DEBUG gtestd) + __gtest_find_library(GTEST_MAIN_LIBRARY gtest_main) + __gtest_find_library(GTEST_MAIN_LIBRARY_DEBUG gtest_maind) + __gtest_find_library(GMOCK_LIBRARY gmock) + __gtest_find_library(GMOCK_LIBRARY_DEBUG gtestd) endif() -if (GTEST_FOUND) - if (NOT GTest_FIND_QUIETLY) - message(STATUS "Found the GTest library:") - message(STATUS "GTEST_STATIC_LIB: ${GTEST_STATIC_LIB}") - message(STATUS "GTEST_MAIN_STATIC_LIB: ${GTEST_MAIN_STATIC_LIB}") - message(STATUS "GTEST_SHARED_LIB: ${GTEST_SHARED_LIB}") - message(STATUS "GTEST_MAIN_SHARED_LIB: ${GTEST_MAIN_SHARED_LIB}") - endif () -else () - if (NOT GTest_FIND_QUIETLY) - set(GTEST_ERR_MSG "Could not find the GTest library. Looked in ") - if ( _gtest_roots ) - set(GTEST_ERR_MSG "${GTEST_ERR_MSG} in ${_gtest_roots}.") - else () - set(GTEST_ERR_MSG "${GTEST_ERR_MSG} system search paths.") - endif () - if (GTest_FIND_REQUIRED) - message(FATAL_ERROR "${GTEST_ERR_MSG}") - else (GTest_FIND_REQUIRED) - message(STATUS "${GTEST_ERR_MSG}") - endif (GTest_FIND_REQUIRED) - endif () -endif () - -if (GMOCK_FOUND) - if (NOT GTest_FIND_QUIETLY) - message(STATUS "Found the Gmock library:") - message(STATUS "GMOCK_STATIC_LIB: ${GMOCK_STATIC_LIB}") - message(STATUS "GMOCK_MAIN_STATIC_LIB: ${GMOCK_MAIN_STATIC_LIB}") - message(STATUS "GMOCK_SHARED_LIB: ${GMOCK_SHARED_LIB}") - message(STATUS "GMOCK_MAIN_SHARED_LIB: ${GMOCK_MAIN_SHARED_LIB}") - endif () -else () - # Reuse Gtest quietly and required flags because they should be found - # in tandem. - if (NOT GTest_FIND_QUIETLY) - set(GMOCK_ERR_MSG "Could not find the GMock library. Looked in ") - if ( _gtest_roots ) - set(GTEST_ERR_MSG "${GMOCK_ERR_MSG} in ${_gtest_roots}.") - else () - set(GTEST_ERR_MSG "${GMOCK_ERR_MSG} system search paths.") - endif () - if (GTest_FIND_REQUIRED) - message(FATAL_ERROR "${GMOCK_ERR_MSG}") - else (GTest_FIND_REQUIRED) - message(STATUS "${GMOCK_ERR_MSG}") - endif (GTest_FIND_REQUIRED) - endif () -endif () - - -mark_as_advanced( - GTEST_INCLUDE_DIR - GTEST_LIBS - GTEST_STATIC_LIB - GTEST_MAIN_STATIC_LIB - GTEST_SHARED_LIB - GTEST_MAIN_SHARED_LIB - - GMOCK_INCLUDE_DIR - GMOCK_LIBS - GMOCK_STATIC_LIB - GMOCK_MAIN_STATIC_LIB - GMOCK_SHARED_LIB - GMOCK_MAIN_SHARED_LIB -) +include(FindPackageHandleStandardArgs) +FIND_PACKAGE_HANDLE_STANDARD_ARGS(GTest DEFAULT_MSG GTEST_LIBRARY GTEST_INCLUDE_DIR GTEST_MAIN_LIBRARY GMOCK_LIBRARY) + +if(GTEST_FOUND) + set(GTEST_INCLUDE_DIRS ${GTEST_INCLUDE_DIR}) + __gtest_append_debugs(GTEST_LIBRARIES GTEST_LIBRARY) + __gtest_append_debugs(GTEST_MAIN_LIBRARIES GTEST_MAIN_LIBRARY) + __gtest_append_debugs(GMOCK_LIBRARIES GMOCK_LIBRARY) + set(GTEST_BOTH_LIBRARIES ${GTEST_LIBRARIES} ${GTEST_MAIN_LIBRARIES}) + + find_package(Threads QUIET) + + if(NOT TARGET GTest::GTest) + __gtest_determine_library_type(GTEST_LIBRARY) + add_library(GTest::GTest ${GTEST_LIBRARY_TYPE} IMPORTED) + if(TARGET Threads::Threads) + set_target_properties(GTest::GTest PROPERTIES + INTERFACE_LINK_LIBRARIES Threads::Threads) + endif() + if(GTEST_LIBRARY_TYPE STREQUAL "SHARED") + set_target_properties(GTest::GTest PROPERTIES + INTERFACE_COMPILE_DEFINITIONS "GTEST_LINKED_AS_SHARED_LIBRARY=1") + endif() + if(GTEST_INCLUDE_DIRS) + set_target_properties(GTest::GTest PROPERTIES + INTERFACE_INCLUDE_DIRECTORIES "${GTEST_INCLUDE_DIRS}") + endif() + __gtest_import_library(GTest::GTest GTEST_LIBRARY "") + __gtest_import_library(GTest::GTest GTEST_LIBRARY "RELEASE") + __gtest_import_library(GTest::GTest GTEST_LIBRARY "DEBUG") + endif() + if(NOT TARGET GTest::Main) + __gtest_determine_library_type(GTEST_MAIN_LIBRARY) + add_library(GTest::Main ${GTEST_MAIN_LIBRARY_TYPE} IMPORTED) + set_target_properties(GTest::Main PROPERTIES + INTERFACE_LINK_LIBRARIES "GTest::GTest") + __gtest_import_library(GTest::Main GTEST_MAIN_LIBRARY "") + __gtest_import_library(GTest::Main GTEST_MAIN_LIBRARY "RELEASE") + __gtest_import_library(GTest::Main GTEST_MAIN_LIBRARY "DEBUG") + endif() + if(NOT TARGET GTest::GMock) + __gtest_determine_library_type(GMOCK_LIBRARY) + add_library(GTest::GMock ${GMOCK_LIBRARY_TYPE} IMPORTED) + if(TARGET Threads::Threads) + set_target_properties(GTest::GMock PROPERTIES + INTERFACE_LINK_LIBRARIES Threads::Threads) + endif() + if(GMOCK_LIBRARY_TYPE STREQUAL "SHARED") + set_target_properties(GTest::GMock PROPERTIES + INTERFACE_COMPILE_DEFINITIONS "GTEST_LINKED_AS_SHARED_LIBRARY=1") + endif() + if(GTEST_INCLUDE_DIRS) + set_target_properties(GTest::GMock PROPERTIES + INTERFACE_INCLUDE_DIRECTORIES "${GTEST_INCLUDE_DIRS}") + endif() + __gtest_import_library(GTest::GMock GMOCK_LIBRARY "") + __gtest_import_library(GTest::GMock GMOCK_LIBRARY "RELEASE") + __gtest_import_library(GTest::GMock GMOCK_LIBRARY "DEBUG") + endif() +endif() diff --git a/cpp/cmake_modules/FindLLVM.cmake b/cpp/cmake_modules/FindLLVM.cmake index 5a9705de272..f57f661c604 100644 --- a/cpp/cmake_modules/FindLLVM.cmake +++ b/cpp/cmake_modules/FindLLVM.cmake @@ -33,6 +33,7 @@ if (APPLE) endif() find_package(LLVM ${ARROW_LLVM_VERSION} REQUIRED CONFIG HINTS + ${LLVM_ROOT} ${LLVM_DIR} /usr/lib /usr/local/opt/llvm diff --git a/cpp/cmake_modules/FindLz4.cmake b/cpp/cmake_modules/FindLz4.cmake index 3a89c8d0288..4cb3c0ec13f 100644 --- a/cpp/cmake_modules/FindLz4.cmake +++ b/cpp/cmake_modules/FindLz4.cmake @@ -15,39 +15,50 @@ # specific language governing permissions and limitations # under the License. -# - Find LZ4 (lz4.h, liblz4.a, liblz4.so, and liblz4.so.0) -# This module defines -# LZ4_INCLUDE_DIR, directory containing headers -# LZ4_SHARED_LIB, path to liblz4 shared library -# LZ4_STATIC_LIB, path to liblz4 static library -# LZ4_FOUND, whether lz4 has been found - -if( NOT "${LZ4_HOME}" STREQUAL "") - file( TO_CMAKE_PATH "${LZ4_HOME}" _native_path ) - list( APPEND _lz4_roots ${_native_path} ) -elseif ( Lz4_HOME ) - list( APPEND _lz4_roots ${Lz4_HOME} ) -endif() - if (MSVC AND NOT DEFINED LZ4_MSVC_STATIC_LIB_SUFFIX) set(LZ4_MSVC_STATIC_LIB_SUFFIX "_static") endif() set(LZ4_STATIC_LIB_SUFFIX - "${LZ4_MSVC_STATIC_LIB_SUFFIX}${CMAKE_STATIC_LIBRARY_SUFFIX}") + "${LZ4_MSVC_STATIC_LIB_SUFFIX}${CMAKE_STATIC_LIBRARY_SUFFIX}") set(LZ4_STATIC_LIB_NAME - ${CMAKE_STATIC_LIBRARY_PREFIX}lz4${LZ4_STATIC_LIB_SUFFIX}) + ${CMAKE_STATIC_LIBRARY_PREFIX}lz4${LZ4_STATIC_LIB_SUFFIX}) -find_path(LZ4_INCLUDE_DIR NAMES lz4.h - PATHS ${_lz4_roots} - NO_DEFAULT_PATH - PATH_SUFFIXES "include" ) -find_library(LZ4_STATIC_LIB NAMES ${LZ4_STATIC_LIB_NAME} lib${LZ4_STATIC_LIB_NAME} - PATHS ${_lz4_roots} - NO_DEFAULT_PATH - PATH_SUFFIXES "lib" ) +pkg_check_modules(LZ4_PC liblz4) +if (LZ4_PC_FOUND) + set(LZ4_INCLUDE_DIR "${LZ4_PC_INCLUDEDIR}") -include(FindPackageHandleStandardArgs) -find_package_handle_standard_args(LZ4 REQUIRED_VARS - LZ4_STATIC_LIB LZ4_INCLUDE_DIR) + list(APPEND LZ4_PC_LIBRARY_DIRS "${LZ4_PC_LIBDIR}") + find_library(LZ4_LIB lz4 + PATHS ${LZ4_PC_LIBRARY_DIRS} + NO_DEFAULT_PATH + PATH_SUFFIXES "${CMAKE_LIBRARY_ARCHITECTURE}") +elseif(LZ4_ROOT) + find_library(LZ4_LIB + NAMES lz4 ${LZ4_STATIC_LIB_NAME} lib${LZ4_STATIC_LIB_NAME} "${CMAKE_SHARED_LIBRARY_PREFIX}lz4_static${CMAKE_SHARED_LIBRARY_SUFFIX}" + PATHS ${LZ4_ROOT} "${LZ4_ROOT}/Library" + PATH_SUFFIXES "lib64" "lib" "bin" + NO_DEFAULT_PATH) + find_path(LZ4_INCLUDE_DIR NAMES lz4.h + PATHS ${LZ4_ROOT} "${LZ4_ROOT}/Library" + NO_DEFAULT_PATH + PATH_SUFFIXES "include") +else() + find_library(LZ4_LIB + NAMES lz4 ${LZ4_STATIC_LIB_NAME} lib${LZ4_STATIC_LIB_NAME} "${CMAKE_SHARED_LIBRARY_PREFIX}lz4_static${CMAKE_SHARED_LIBRARY_SUFFIX}" + PATH_SUFFIXES "lib64" "lib" "bin") + find_path(LZ4_INCLUDE_DIR NAMES lz4.h + PATH_SUFFIXES "include") +endif() + +find_package_handle_standard_args(Lz4 + REQUIRED_VARS LZ4_LIB LZ4_INCLUDE_DIR) + +if (Lz4_FOUND) + add_library(LZ4::lz4 UNKNOWN IMPORTED) + set_target_properties(LZ4::lz4 PROPERTIES + IMPORTED_LOCATION "${LZ4_LIB}" + INTERFACE_INCLUDE_DIRECTORIES "${LZ4_INCLUDE_DIR}" + ) +endif() diff --git a/cpp/cmake_modules/FindProtobuf.cmake b/cpp/cmake_modules/FindProtobuf.cmake deleted file mode 100644 index f53f48d6068..00000000000 --- a/cpp/cmake_modules/FindProtobuf.cmake +++ /dev/null @@ -1,102 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# PROTOBUF_HOME environmental variable is used to check for Protobuf headers and static library - -# PROTOBUF_INCLUDE_DIR: directory containing headers -# PROTOBUF_LIBS: directory containing Protobuf libraries -# PROTOBUF_STATIC_LIB: location of protobuf.a -# PROTOC_STATIC_LIB: location of protoc.a -# PROTOBUF_EXECUTABLE: location of protoc -# PROTOBUF_FOUND is set if Protobuf is found - - -if( NOT "${PROTOBUF_HOME}" STREQUAL "") - file (TO_CMAKE_PATH "${PROTOBUF_HOME}" _protobuf_path) -endif() - -message (STATUS "PROTOBUF_HOME: ${PROTOBUF_HOME}") - -find_path (PROTOBUF_INCLUDE_DIR google/protobuf/io/zero_copy_stream.h HINTS - ${_protobuf_path} - NO_DEFAULT_PATH - PATH_SUFFIXES "include") - -find_path (PROTOBUF_INCLUDE_DIR google/protobuf/io/coded_stream.h HINTS - ${_protobuf_path} - NO_DEFAULT_PATH - PATH_SUFFIXES "include") - -set (lib_dirs "lib") -if (EXISTS "${_protobuf_path}/lib64") - set (lib_dirs "lib64" ${lib_dirs}) -endif () -if (EXISTS "${_protobuf_path}/lib/${CMAKE_LIBRARY_ARCHITECTURE}") - set (lib_dirs "lib/${CMAKE_LIBRARY_ARCHITECTURE}" ${lib_dirs}) -endif () - -find_library (PROTOBUF_LIBRARY NAMES protobuf libprotobuf PATHS - ${_protobuf_path} - NO_DEFAULT_PATH - PATH_SUFFIXES ${lib_dirs}) - -find_library (PROTOC_LIBRARY NAMES protoc libprotoc PATHS - ${_protobuf_path} - NO_DEFAULT_PATH - PATH_SUFFIXES ${lib_dirs}) - -find_program(PROTOBUF_EXECUTABLE protoc HINTS - ${_protobuf_path} - NO_DEFAULT_PATH - PATH_SUFFIXES "bin") - -if (PROTOBUF_INCLUDE_DIR AND PROTOBUF_LIBRARY AND PROTOC_LIBRARY AND PROTOBUF_EXECUTABLE) - set (PROTOBUF_FOUND TRUE) - set (PROTOBUF_SHARED_LIB ${PROTOBUF_LIBRARY}) - set (PROTOC_SHARED_LIB ${PROTOC_LIBRARY}) - get_filename_component (PROTOBUF_LIBS ${PROTOBUF_LIBRARY} PATH) - set (PROTOBUF_LIB_NAME protobuf) - set (PROTOC_LIB_NAME protoc) - set (PROTOBUF_STATIC_LIB ${PROTOBUF_LIBS}/${CMAKE_STATIC_LIBRARY_PREFIX}${PROTOBUF_LIB_NAME}${PROTOBUF_MSVC_STATIC_LIB_SUFFIX}${CMAKE_STATIC_LIBRARY_SUFFIX}) - set (PROTOC_STATIC_LIB ${PROTOBUF_LIBS}/${CMAKE_STATIC_LIBRARY_PREFIX}${PROTOC_LIB_NAME}${CMAKE_STATIC_LIBRARY_SUFFIX}) -else () - set (PROTOBUF_FOUND FALSE) -endif () - -if (PROTOBUF_FOUND) - message (STATUS "Found the Protobuf headers: ${PROTOBUF_INCLUDE_DIR}") - message (STATUS "Found the Protobuf shared library: ${PROTOBUF_SHARED_LIB}") - message (STATUS "Found the Protobuf library: ${PROTOBUF_STATIC_LIB}") - message (STATUS "Found the Protoc shared library: ${PROTOC_SHARED_LIB}") - message (STATUS "Found the Protoc library: ${PROTOC_STATIC_LIB}") - message (STATUS "Found the Protoc executable: ${PROTOBUF_EXECUTABLE}") -else() - if (_protobuf_path) - set (PROTOBUF_ERR_MSG "Could not find Protobuf. Looked in ${_protobuf_path}") - else () - set (PROTOBUF_ERR_MSG "Could not find Protobuf in system search paths.") - endif() - - if (Protobuf_FIND_REQUIRED) - message (FATAL_ERROR "${PROTOBUF_ERR_MSG}") - else () - message (STATUS "${PROTOBUF_ERR_MSG}") - endif () -endif() - -mark_as_advanced ( - PROTOBUF_EXECUTABLE - PROTOBUF_INCLUDE_DIR - PROTOBUF_LIBS - PROTOBUF_STATIC_LIB - PROTOC_STATIC_LIB -) diff --git a/cpp/cmake_modules/FindRE2.cmake b/cpp/cmake_modules/FindRE2.cmake index 51b093fc977..a0536103ef6 100644 --- a/cpp/cmake_modules/FindRE2.cmake +++ b/cpp/cmake_modules/FindRE2.cmake @@ -1,4 +1,3 @@ -############################################################################## # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -15,92 +14,48 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -############################################################################## -# - Find re2 headers and lib. -# RE2_HOME hints the location -# This module defines -# RE2_INCLUDE_DIR, directory containing headers -# RE2_STATIC_LIB, path to libre2.a -# RE2_SHARED_LIB, path to libre2.so -# RE2_FOUND, whether re2 has been found +pkg_check_modules(RE2_PC re2) +if (RE2_PC_FOUND) + set(RE2_INCLUDE_DIR "${RE2_PC_INCLUDEDIR}") -if( NOT "${RE2_HOME}" STREQUAL "") - file (TO_CMAKE_PATH "${RE2_HOME}" _re2_path) -endif() -message (STATUS "RE2_HOME: ${RE2_HOME}") - -find_path(RE2_INCLUDE_DIR re2/re2.h - HINTS ${_re2_path} - NO_DEFAULT_PATH - PATH_SUFFIXES "include" - DOC "Google's re2 regex header path" -) - -set (lib_dirs "lib") -if (EXISTS "${_re2_path}/lib64") - set (lib_dirs "lib64" ${lib_dirs}) -endif () -if (EXISTS "${_re2_path}/lib/${CMAKE_LIBRARY_ARCHITECTURE}") - set (lib_dirs "lib/${CMAKE_LIBRARY_ARCHITECTURE}" ${lib_dirs}) -endif () - -set(RE2_LIB_NAME re2) -set(RE2_STATIC_LIB_NAME ${CMAKE_STATIC_LIBRARY_PREFIX}${RE2_LIB_NAME}${RE2_MSVC_STATIC_LIB_SUFFIX}${CMAKE_STATIC_LIBRARY_SUFFIX}) -set(RE2_SHARED_LIB_NAME ${CMAKE_SHARED_LIBRARY_PREFIX}${RE2_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX}) - -find_library(RE2_STATIC_LIB NAMES ${RE2_STATIC_LIB_NAME} - PATHS ${_re2_path} - NO_DEFAULT_PATH - PATH_SUFFIXES ${lib_dirs} - DOC "Google's re2 regex static library" -) - -find_library(RE2_SHARED_LIB NAMES ${RE2_SHARED_LIB_NAME} - PATHS ${_re2_path} - NO_DEFAULT_PATH - PATH_SUFFIXES ${lib_dirs} - DOC "Google's re2 regex static library" -) + list(APPEND RE2_PC_LIBRARY_DIRS "${RE2_PC_LIBDIR}") + find_library(RE2_LIB re2 PATHS ${RE2_PC_LIBRARY_DIRS} NO_DEFAULT_PATH) -if (ARROW_RE2_LINKAGE STREQUAL "static" AND (NOT RE2_STATIC_LIB)) - set(RE2_LIB_NOT_FOUND TRUE) - set(RE2_MISSING "static lib") -elseif(ARROW_RE2_LINKAGE STREQUAL "shared" AND (NOT RE2_SHARED_LIB)) - set(RE2_LIB_NOT_FOUND TRUE) - set(RE2_MISSING "shared lib") + # On Fedora, the reported prefix is wrong. As users likely run into this, + # workaround. + # https://bugzilla.redhat.com/show_bug.cgi?id=1652589 + if(UNIX AND NOT APPLE AND NOT RE2_LIB) + if (RE2_PC_PREFIX STREQUAL "/usr/local") + find_library(RE2_LIB re2) + endif() + endif() +elseif(RE2_ROOT) + find_library(RE2_LIB + NAMES re2 "${CMAKE_STATIC_LIBRARY_PREFIX}re2${RE2_MSVC_STATIC_LIB_SUFFIX}${CMAKE_STATIC_LIBRARY_SUFFIX}" "${CMAKE_SHARED_LIBRARY_PREFIX}re2${CMAKE_SHARED_LIBRARY_SUFFIX}" + PATHS ${RE2_ROOT} "${RE2_ROOT}/Library" + PATH_SUFFIXES "lib64" "lib" "bin" + NO_DEFAULT_PATH) + find_path(RE2_INCLUDE_DIR NAMES re2/re2.h + PATHS ${RE2_ROOT} "${RE2_ROOT}/Library" + NO_DEFAULT_PATH + PATH_SUFFIXES "include") +else() + find_library(RE2_LIB + NAMES re2 "${CMAKE_STATIC_LIBRARY_PREFIX}re2${RE2_MSVC_STATIC_LIB_SUFFIX}${CMAKE_STATIC_LIBRARY_SUFFIX}" "${CMAKE_SHARED_LIBRARY_PREFIX}re2${CMAKE_SHARED_LIBRARY_SUFFIX}" + PATH_SUFFIXES "lib64" "lib" "bin") + find_path(RE2_INCLUDE_DIR NAMES re2/re2.h + PATH_SUFFIXES "include") endif() -if (NOT RE2_INCLUDE_DIR OR RE2_LIB_NOT_FOUND) - if (NOT RE2_INCLUDE_DIR) - if (RE2_MISSING) - set(RE2_MISSING "${RE2_MISSING}, headers") - else () - set(RE2_MISSING "headers") - endif () - endif () +find_package_handle_standard_args(RE2 + REQUIRED_VARS RE2_LIB RE2_INCLUDE_DIR) - set(RE2_FOUND FALSE) - if (_re2_path) - set (RE2_ERR_MSG "Could not find re2 (missing ${RE2_MISSING}). Looked in ${_re2_path}.") - else () - set (RE2_ERR_MSG "Could not find re2 (missing ${RE2_MISSING}). Looked in system search paths.") - endif() - - if (RE2_FIND_REQUIRED) - message(FATAL_ERROR "${RE2_ERR_MSG})") - else () - message (STATUS "${RE2_ERR_MSG}") - endif () -else() - set(RE2_FOUND TRUE) - message(STATUS "RE2 headers : ${RE2_INCLUDE_DIR}") - message(STATUS "RE2 static library : ${RE2_STATIC_LIB}") - message(STATUS "RE2 shared library : ${RE2_SHARED_LIB}") +if (RE2_FOUND) + add_library(RE2::re2 UNKNOWN IMPORTED) + set_target_properties(RE2::re2 PROPERTIES + IMPORTED_LOCATION "${RE2_LIB}" + INTERFACE_INCLUDE_DIRECTORIES "${RE2_INCLUDE_DIR}" + ) endif() -mark_as_advanced( - RE2_INCLUDE_DIR - RE2_SHARED_LIB - RE2_STATIC_LIB -) diff --git a/cpp/cmake_modules/FindRapidJSONAlt.cmake b/cpp/cmake_modules/FindRapidJSONAlt.cmake new file mode 100644 index 00000000000..d62fe087caa --- /dev/null +++ b/cpp/cmake_modules/FindRapidJSONAlt.cmake @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +if(RapidJSON_ROOT) + find_path(RAPIDJSON_INCLUDE_DIR NAMES rapidjson/rapidjson.h + PATHS ${RapidJSON_ROOT} NO_DEFAULT_PATH + PATH_SUFFIXES "include") +else() + find_path(RAPIDJSON_INCLUDE_DIR NAMES rapidjson/rapidjson.h + PATH_SUFFIXES "include") +endif() + +find_package_handle_standard_args(RapidJSONAlt + REQUIRED_VARS RAPIDJSON_INCLUDE_DIR) diff --git a/cpp/cmake_modules/FindSnappy.cmake b/cpp/cmake_modules/FindSnappy.cmake deleted file mode 100644 index 867963c103c..00000000000 --- a/cpp/cmake_modules/FindSnappy.cmake +++ /dev/null @@ -1,94 +0,0 @@ -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# Tries to find Snappy headers and libraries. -# -# Usage of this module as follows: -# -# find_package(Snappy) -# -# Variables used by this module, they can change the default behaviour and need -# to be set before calling find_package: -# -# Snappy_HOME - When set, this path is inspected instead of standard library -# locations as the root of the Snappy installation. -# The environment variable SNAPPY_HOME overrides this variable. -# -# This module defines -# SNAPPY_INCLUDE_DIR, directory containing headers -# SNAPPY_LIBS, directory containing snappy libraries -# SNAPPY_STATIC_LIB, path to libsnappy.a -# SNAPPY_SHARED_LIB, path to libsnappy's shared library -# SNAPPY_FOUND, whether snappy has been found - -if( NOT "${SNAPPY_HOME}" STREQUAL "") - file( TO_CMAKE_PATH "${SNAPPY_HOME}" _native_path ) - list( APPEND _snappy_roots ${_native_path} ) -elseif ( Snappy_HOME ) - list( APPEND _snappy_roots ${Snappy_HOME} ) -endif() - -message(STATUS "SNAPPY_HOME: ${SNAPPY_HOME}") -find_path(SNAPPY_INCLUDE_DIR snappy.h HINTS - ${_snappy_roots} - NO_DEFAULT_PATH - PATH_SUFFIXES "include") - -find_library( SNAPPY_LIBRARIES NAMES snappy PATHS - ${_snappy_roots} - NO_DEFAULT_PATH - PATH_SUFFIXES "lib") - -if (SNAPPY_INCLUDE_DIR AND (PARQUET_MINIMAL_DEPENDENCY OR SNAPPY_LIBRARIES)) - set(SNAPPY_FOUND TRUE) - get_filename_component( SNAPPY_LIBS ${SNAPPY_LIBRARIES} PATH ) - set(SNAPPY_HEADER_NAME snappy.h) - set(SNAPPY_HEADER ${SNAPPY_INCLUDE_DIR}/${SNAPPY_HEADER_NAME}) - set(SNAPPY_LIB_NAME snappy) - set(SNAPPY_STATIC_LIB ${SNAPPY_LIBS}/${CMAKE_STATIC_LIBRARY_PREFIX}${SNAPPY_LIB_NAME}${SNAPPY_MSVC_STATIC_LIB_SUFFIX}${CMAKE_STATIC_LIBRARY_SUFFIX}) - set(SNAPPY_SHARED_LIB ${SNAPPY_LIBS}/${CMAKE_SHARED_LIBRARY_PREFIX}${SNAPPY_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX}) -else () - set(SNAPPY_FOUND FALSE) -endif () - -if (SNAPPY_FOUND) - if (NOT Snappy_FIND_QUIETLY) - if (PARQUET_MINIMAL_DEPENDENCY) - message(STATUS "Found the Snappy header: ${SNAPPY_HEADER}") - else () - message(STATUS "Found the Snappy library: ${SNAPPY_LIBRARIES}") - endif () - endif () -else () - if (NOT Snappy_FIND_QUIETLY) - set(SNAPPY_ERR_MSG "Could not find the Snappy library. Looked in ") - if ( _snappy_roots ) - set(SNAPPY_ERR_MSG "${SNAPPY_ERR_MSG} in ${_snappy_roots}.") - else () - set(SNAPPY_ERR_MSG "${SNAPPY_ERR_MSG} system search paths.") - endif () - if (Snappy_FIND_REQUIRED) - message(FATAL_ERROR "${SNAPPY_ERR_MSG}") - else (Snappy_FIND_REQUIRED) - message(STATUS "${SNAPPY_ERR_MSG}") - endif (Snappy_FIND_REQUIRED) - endif () -endif () - -mark_as_advanced( - SNAPPY_INCLUDE_DIR - SNAPPY_LIBS - SNAPPY_LIBRARIES - SNAPPY_STATIC_LIB - SNAPPY_SHARED_LIB -) diff --git a/cpp/cmake_modules/FindSnappyAlt.cmake b/cpp/cmake_modules/FindSnappyAlt.cmake new file mode 100644 index 00000000000..15f125d8e99 --- /dev/null +++ b/cpp/cmake_modules/FindSnappyAlt.cmake @@ -0,0 +1,42 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +if(Snappy_ROOT) + find_library(Snappy_LIB + NAMES snappy + PATHS ${Snappy_ROOT} + NO_DEFAULT_PATH) + find_path(Snappy_INCLUDE_DIR NAMES snappy.h + PATHS ${Snappy_ROOT} NO_DEFAULT_PATH + PATH_SUFFIXES "include") +else() + find_library(Snappy_LIB + NAMES snappy) + find_path(Snappy_INCLUDE_DIR NAMES snappy.h + PATH_SUFFIXES "include") +endif() + +find_package_handle_standard_args(SnappyAlt + REQUIRED_VARS Snappy_LIB Snappy_INCLUDE_DIR) + +if (SnappyAlt_FOUND) + add_library(Snappy::snappy UNKNOWN IMPORTED) + set_target_properties(Snappy::snappy PROPERTIES + IMPORTED_LOCATION "${Snappy_LIB}" + INTERFACE_INCLUDE_DIRECTORIES "${Snappy_INCLUDE_DIR}" + ) +endif() diff --git a/cpp/cmake_modules/FindThrift.cmake b/cpp/cmake_modules/FindThrift.cmake index cb0f819bd57..aa1cdfa44f9 100644 --- a/cpp/cmake_modules/FindThrift.cmake +++ b/cpp/cmake_modules/FindThrift.cmake @@ -24,17 +24,11 @@ # This module defines # THRIFT_VERSION, version string of ant if found # THRIFT_INCLUDE_DIR, where to find THRIFT headers -# THRIFT_CONTRIB_DIR, where contrib thrift files (e.g. fb303.thrift) are installed # THRIFT_STATIC_LIB, THRIFT static library # THRIFT_FOUND, If false, do not try to use ant -# prefer the thrift version supplied in THRIFT_HOME -if( NOT "${THRIFT_HOME}" STREQUAL "") - file( TO_CMAKE_PATH "${THRIFT_HOME}" _native_path ) - list( APPEND _thrift_roots ${_native_path} ) -elseif ( Thrift_HOME ) - list( APPEND _thrift_roots ${Thrift_HOME} ) -elseif (APPLE) +# TODO: Add this back to global +if (APPLE) # Also look in homebrew for a matching llvm version find_program(BREW_BIN brew) if (BREW_BIN) @@ -43,39 +37,9 @@ elseif (APPLE) OUTPUT_VARIABLE THRIFT_BREW_PREFIX OUTPUT_STRIP_TRAILING_WHITESPACE ) - list( APPEND _thrift_roots ${THRIFT_BREW_PREFIX} ) endif() endif() -message(STATUS "THRIFT_HOME: ${THRIFT_HOME}") -find_path(THRIFT_INCLUDE_DIR thrift/Thrift.h HINTS - ${_thrift_roots} - NO_DEFAULT_PATH - PATH_SUFFIXES "include" -) - -find_path(THRIFT_CONTRIB_DIR share/fb303/if/fb303.thrift HINTS - ${_thrift_roots} - NO_DEFAULT_PATH -) - -if (MSVC AND NOT THRIFT_MSVC_STATIC_LIB_SUFFIX) - set(THRIFT_MSVC_STATIC_LIB_SUFFIX md) -endif() - -find_library(THRIFT_STATIC_LIB NAMES - ${CMAKE_STATIC_LIBRARY_PREFIX}thrift${THRIFT_MSVC_STATIC_LIB_SUFFIX}${CMAKE_STATIC_LIBRARY_SUFFIX} - HINTS ${_thrift_roots} - NO_DEFAULT_PATH - PATH_SUFFIXES "lib/${CMAKE_LIBRARY_ARCHITECTURE}" "lib" -) - -find_program(THRIFT_COMPILER thrift HINTS - ${_thrift_roots} - NO_DEFAULT_PATH - PATH_SUFFIXES "bin" -) - function(EXTRACT_THRIFT_VERSION) exec_program(${THRIFT_COMPILER} ARGS -version OUTPUT_VARIABLE THRIFT_VERSION RETURN_VALUE THRIFT_RETURN) @@ -94,37 +58,43 @@ function(EXTRACT_THRIFT_VERSION) endif() endfunction(EXTRACT_THRIFT_VERSION) -if (THRIFT_STATIC_LIB) - set(THRIFT_FOUND TRUE) - EXTRACT_THRIFT_VERSION() -else () - set(THRIFT_FOUND FALSE) -endif () +if (MSVC AND NOT THRIFT_MSVC_STATIC_LIB_SUFFIX) + set(THRIFT_MSVC_STATIC_LIB_SUFFIX md) +endif() + +pkg_check_modules(THRIFT_PC thrift) -if (THRIFT_FOUND) - if (NOT Thrift_FIND_QUIETLY) - message(STATUS "Thrift version: ${THRIFT_VERSION}") - endif () -else () - if (NOT Thrift_FIND_QUIETLY) - set(THRIFT_ERR_MSG "Thrift compiler/libraries NOT found: ${THRIFT_RETURN}") - set(THRIFT_ERR_MSG "${THRIFT_ERR_MSG} (${THRIFT_INCLUDE_DIR}, ${THRIFT_STATIC_LIB}).") - if ( _thrift_roots ) - set(THRIFT_ERR_MSG "${THRIFT_ERR_MSG} Looked in ${_thrift_roots}.") - else () - set(THRIFT_ERR_MSG "${THRIFT_ERR_MSG} Looked in system search paths.") - endif () - if ( Thrift_FIND_REQUIRED ) - message(FATAL_ERROR "${THRIFT_ERR_MSG}") - else () - message(STATUS "${THRIFT_ERR_MSG}") - endif () - endif () -endif () +# THRIFT-4760: The pkgconfig files are currently only installed when using autotools. +# Starting with 0.13, they are also installed for the CMake-based installations of Thrift. +if (THRIFT_PC_FOUND) + set(THRIFT_INCLUDE_DIR "${THRIFT_PC_INCLUDEDIR}") + list(APPEND THRIFT_PC_LIBRARY_DIRS "${THRIFT_PC_LIBDIR}") + + find_library(THRIFT_STATIC_LIB thrift${THRIFT_MSVC_STATIC_LIB_SUFFIX} PATHS ${THRIFT_PC_LIBRARY_DIRS} NO_DEFAULT_PATH) + find_program(THRIFT_COMPILER thrift HINTS + ${THRIFT_PC_PREFIX} + NO_DEFAULT_PATH + PATH_SUFFIXES "bin" + ) +else() + find_library(THRIFT_STATIC_LIB thrift${THRIFT_MSVC_STATIC_LIB_SUFFIX} HINTS ${Thrift_ROOT} PATH_SUFFIXES "lib/${CMAKE_LIBRARY_ARCHITECTURE}" "lib") + find_path(THRIFT_INCLUDE_DIR thrift/Thrift.h HINTS ${Thrift_ROOT} PATH_SUFFIXES "include") + find_program(THRIFT_COMPILER thrift HINTS + ${Thrift_ROOT} + PATH_SUFFIXES "bin" + ) +endif() -mark_as_advanced( - THRIFT_STATIC_LIB - THRIFT_COMPILER - THRIFT_INCLUDE_DIR -) +find_package_handle_standard_args(Thrift + REQUIRED_VARS THRIFT_STATIC_LIB THRIFT_INCLUDE_DIR THRIFT_COMPILER) + +if (Thrift_FOUND) + add_library(Thrift::thrift STATIC IMPORTED) + set_target_properties(Thrift::thrift PROPERTIES + IMPORTED_LOCATION "${THRIFT_STATIC_LIB}" + INTERFACE_INCLUDE_DIRECTORIES "${THRIFT_INCLUDE_DIR}" + ) + + EXTRACT_THRIFT_VERSION() +endif() diff --git a/cpp/cmake_modules/FindZLIB.cmake b/cpp/cmake_modules/FindZLIB.cmake deleted file mode 100644 index ca6060b57cc..00000000000 --- a/cpp/cmake_modules/FindZLIB.cmake +++ /dev/null @@ -1,103 +0,0 @@ -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# Tries to find ZLIB headers and libraries. -# -# Usage of this module as follows: -# -# find_package(ZLIB) -# -# Variables used by this module, they can change the default behaviour and need -# to be set before calling find_package: -# -# ZLIB_HOME - When set, this path is inspected instead of standard library -# locations as the root of the ZLIB installation. -# The environment variable ZLIB_HOME overrides this variable. -# -# - Find ZLIB (zlib.h, libz.a, libz.so, and libz.so.1) -# This module defines -# ZLIB_INCLUDE_DIR, directory containing headers -# ZLIB_SHARED_LIB, path to libz's shared library -# ZLIB_FOUND, whether zlib has been found - -if( NOT "${ZLIB_HOME}" STREQUAL "") - file( TO_CMAKE_PATH "${ZLIB_HOME}" _native_path ) - list( APPEND _zlib_roots ${_native_path} ) -elseif ( ZLIB_HOME ) - list( APPEND _zlib_roots ${ZLIB_HOME} ) -endif() - -if (MSVC) - # zlib uses zlib.lib for Windows. - set(ZLIB_LIB_NAME zlib.lib) -else () - # zlib uses libz.so for non Windows. - set(ZLIB_LIB_NAME - ${CMAKE_SHARED_LIBRARY_PREFIX}z${CMAKE_SHARED_LIBRARY_SUFFIX}) -endif () - -# Try the parameterized roots, if they exist -if (_zlib_roots) - find_path(ZLIB_INCLUDE_DIR NAMES zlib.h - PATHS ${_zlib_roots} NO_DEFAULT_PATH - PATH_SUFFIXES "include") - find_library(ZLIB_SHARED_LIB - NAMES ${ZLIB_LIB_NAME} - PATHS ${_zlib_roots} NO_DEFAULT_PATH - PATH_SUFFIXES "lib") -else () - pkg_check_modules(PKG_ZLIB zlib) - if (PKG_ZLIB_FOUND) - set(ZLIB_INCLUDE_DIR ${PKG_ZLIB_INCLUDEDIR}) - find_library(ZLIB_SHARED_LIB - NAMES ${ZLIB_LIB_NAME} - PATHS ${PKG_ZLIB_LIBDIR} NO_DEFAULT_PATH) - else () - find_path(ZLIB_INCLUDE_DIR NAMES zlib.h) - find_library(ZLIB_SHARED_LIB NAMES ${ZLIB_LIB_NAME}) - endif () -endif () - -if (ZLIB_INCLUDE_DIR AND ZLIB_SHARED_LIB) - set(ZLIB_FOUND TRUE) -else () - set(ZLIB_FOUND FALSE) -endif () - -if (ZLIB_FOUND) - if (NOT ZLIB_FIND_QUIETLY) - if (ZLIB_SHARED_LIB) - message(STATUS "Found the ZLIB shared library: ${ZLIB_SHARED_LIB}") - endif () - endif () -else () - if (NOT ZLIB_FIND_QUIETLY) - set(ZLIB_ERR_MSG "Could not find the ZLIB library. Looked in ") - if ( _zlib_roots ) - set(ZLIB_ERR_MSG "${ZLIB_ERR_MSG} ${_zlib_roots}.") - else () - set(ZLIB_ERR_MSG "${ZLIB_ERR_MSG} system search paths.") - endif () - if (ZLIB_FIND_REQUIRED) - message(FATAL_ERROR "${ZLIB_ERR_MSG}") - else (ZLIB_FIND_REQUIRED) - message(STATUS "${ZLIB_ERR_MSG}") - endif (ZLIB_FIND_REQUIRED) - endif () -endif () - -mark_as_advanced( - ZLIB_INCLUDE_DIR - ZLIB_LIBRARIES - ZLIB_SHARED_LIB -) diff --git a/cpp/cmake_modules/FindZSTD.cmake b/cpp/cmake_modules/FindZSTD.cmake index 7f8d1319e64..7fb751a4d84 100644 --- a/cpp/cmake_modules/FindZSTD.cmake +++ b/cpp/cmake_modules/FindZSTD.cmake @@ -15,38 +15,47 @@ # specific language governing permissions and limitations # under the License. -# - Find ZSTD (zstd.h, libzstd.a, libzstd.so, and libzstd.so.0) -# This module defines -# ZSTD_INCLUDE_DIR, directory containing headers -# ZSTD_STATIC_LIB, path to libzstd static library -# ZSTD_FOUND, whether zstd has been found - -if( NOT "${ZSTD_HOME}" STREQUAL "") - file( TO_CMAKE_PATH "${ZSTD_HOME}" _native_path ) - list( APPEND _zstd_roots ${_native_path} ) -elseif ( ZStd_HOME ) - list( APPEND _zstd_roots ${ZStd_HOME} ) -endif() - if (MSVC AND NOT DEFINED ZSTD_MSVC_STATIC_LIB_SUFFIX) set(ZSTD_MSVC_STATIC_LIB_SUFFIX "_static") endif() -set(ZSTD_STATIC_LIB_SUFFIX - "${ZSTD_MSVC_STATIC_LIB_SUFFIX}${CMAKE_STATIC_LIBRARY_SUFFIX}") +set(ZSTD_STATIC_LIB_SUFFIX "${ZSTD_MSVC_STATIC_LIB_SUFFIX}${CMAKE_STATIC_LIBRARY_SUFFIX}") +set(ZSTD_STATIC_LIB_NAME ${CMAKE_STATIC_LIBRARY_PREFIX}zstd${ZSTD_STATIC_LIB_SUFFIX}) -set(ZSTD_STATIC_LIB_NAME - ${CMAKE_STATIC_LIBRARY_PREFIX}zstd${ZSTD_STATIC_LIB_SUFFIX}) +pkg_check_modules(ZSTD_PC libzstd) +if (ZSTD_PC_FOUND) + set(ZSTD_INCLUDE_DIR "${ZSTD_PC_INCLUDEDIR}") -find_path(ZSTD_INCLUDE_DIR NAMES zstd.h - PATHS ${_zstd_roots} - NO_DEFAULT_PATH - PATH_SUFFIXES "include" ) -find_library(ZSTD_STATIC_LIB NAMES ${ZSTD_STATIC_LIB_NAME} lib${ZSTD_STATIC_LIB_NAME} - PATHS ${_zstd_roots} - NO_DEFAULT_PATH - PATH_SUFFIXES "lib" ) + list(APPEND ZSTD_PC_LIBRARY_DIRS "${ZSTD_PC_LIBDIR}") + find_library(ZSTD_LIB zstd + PATHS ${ZSTD_PC_LIBRARY_DIRS} + NO_DEFAULT_PATH + PATH_SUFFIXES "${CMAKE_LIBRARY_ARCHITECTURE}") +elseif(ZSTD_ROOT) + message(STATUS "Using ZSTD_ROOT: ${ZSTD_ROOT}") + find_library(ZSTD_LIB + NAMES zstd "${ZSTD_STATIC_LIB_NAME}" "lib${ZSTD_STATIC_LIB_NAME}" "${CMAKE_SHARED_LIBRARY_PREFIX}zstd${CMAKE_SHARED_LIBRARY_SUFFIX}" + PATHS ${ZSTD_ROOT} "${ZSTD_ROOT}/Library" + PATH_SUFFIXES "lib64" "lib" "bin" + NO_DEFAULT_PATH) + find_path(ZSTD_INCLUDE_DIR NAMES zstd.h + PATHS ${ZSTD_ROOT} "${ZSTD_ROOT}/Library" NO_DEFAULT_PATH + PATH_SUFFIXES "include") +else() + find_library(ZSTD_LIB + NAMES zstd "${ZSTD_STATIC_LIB_NAME}" "lib${ZSTD_STATIC_LIB_NAME}" "${CMAKE_SHARED_LIBRARY_PREFIX}zstd${CMAKE_SHARED_LIBRARY_SUFFIX}" + PATH_SUFFIXES "lib64" "lib" "bin") + find_path(ZSTD_INCLUDE_DIR NAMES zstd.h + PATH_SUFFIXES "include") +endif() + +find_package_handle_standard_args(ZSTD + REQUIRED_VARS ZSTD_LIB ZSTD_INCLUDE_DIR) -include(FindPackageHandleStandardArgs) -find_package_handle_standard_args(ZSTD REQUIRED_VARS - ZSTD_STATIC_LIB ZSTD_INCLUDE_DIR) +if (ZSTD_FOUND) + add_library(ZSTD::zstd UNKNOWN IMPORTED) + set_target_properties(ZSTD::zstd PROPERTIES + IMPORTED_LOCATION "${ZSTD_LIB}" + INTERFACE_INCLUDE_DIRECTORIES "${ZSTD_INCLUDE_DIR}" + ) +endif() diff --git a/cpp/cmake_modules/Findc-ares.cmake b/cpp/cmake_modules/Findc-ares.cmake deleted file mode 100644 index 2b7b58916c6..00000000000 --- a/cpp/cmake_modules/Findc-ares.cmake +++ /dev/null @@ -1,109 +0,0 @@ -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# Tries to find c-ares headers and libraries. -# -# Usage of this module as follows: -# -# find_package(c-ares) -# -# Variables used by this module, they can change the default behaviour and need -# to be set before calling find_package: -# -# CARES_HOME - When set, this path is inspected instead of standard library -# locations as the root of the c-ares installation. -# The environment variable CARES_HOME overrides this variable. -# -# - Find CARES -# This module defines -# CARES_INCLUDE_DIR, directory containing headers -# CARES_SHARED_LIB, path to c-ares's shared library -# CARES_FOUND, whether c-ares has been found - -if( NOT "${CARES_HOME}" STREQUAL "") - file( TO_CMAKE_PATH "${CARES_HOME}" _native_path ) - list( APPEND _cares_roots ${_native_path} ) -elseif ( CARES_HOME ) - list( APPEND _cares_roots ${CARES_HOME} ) -endif() - -if (MSVC) - set(CARES_LIB_NAME cares.lib) -else () - set(CARES_LIB_NAME - ${CMAKE_SHARED_LIBRARY_PREFIX}cares${CMAKE_SHARED_LIBRARY_SUFFIX}) - set(CARES_STATIC_LIB_NAMES - ${CMAKE_STATIC_LIBRARY_PREFIX}cares${CMAKE_STATIC_LIBRARY_SUFFIX} - ${CMAKE_STATIC_LIBRARY_PREFIX}cares_static${CMAKE_STATIC_LIBRARY_SUFFIX}) -endif () - -# Try the parameterized roots, if they exist -if (_cares_roots) - find_path(CARES_INCLUDE_DIR NAMES ares.h - PATHS ${_cares_roots} NO_DEFAULT_PATH - PATH_SUFFIXES "include") - find_library(CARES_SHARED_LIB - NAMES ${CARES_LIB_NAME} - PATHS ${_cares_roots} NO_DEFAULT_PATH - PATH_SUFFIXES "lib") - find_library(CARES_STATIC_LIB - NAMES ${CARES_STATIC_LIB_NAMES} - PATHS ${_cares_roots} NO_DEFAULT_PATH - PATH_SUFFIXES "lib") -else () - pkg_check_modules(PKG_CARES cares) - if (PKG_CARES_FOUND) - set(CARES_INCLUDE_DIR ${PKG_CARES_INCLUDEDIR}) - find_library(CARES_SHARED_LIB - NAMES ${CARES_LIB_NAME} - PATHS ${PKG_CARES_LIBDIR} NO_DEFAULT_PATH) - else () - find_path(CARES_INCLUDE_DIR NAMES cares.h) - find_library(CARES_SHARED_LIB NAMES ${CARES_LIB_NAME}) - endif () -endif () - -if (CARES_INCLUDE_DIR AND CARES_SHARED_LIB) - set(CARES_FOUND TRUE) -else () - set(CARES_FOUND FALSE) -endif () - -if (CARES_FOUND) - if (NOT CARES_FIND_QUIETLY) - if (CARES_SHARED_LIB) - message(STATUS "Found the c-ares shared library: ${CARES_SHARED_LIB}") - endif () - endif () -else () - if (NOT CARES_FIND_QUIETLY) - set(CARES_ERR_MSG "Could not find the c-ares library. Looked in ") - if ( _cares_roots ) - set(CARES_ERR_MSG "${CARES_ERR_MSG} ${_cares_roots}.") - else () - set(CARES_ERR_MSG "${CARES_ERR_MSG} system search paths.") - endif () - if (CARES_FIND_REQUIRED) - message(FATAL_ERROR "${CARES_ERR_MSG}") - else (CARES_FIND_REQUIRED) - message(STATUS "${CARES_ERR_MSG}") - endif (CARES_FIND_REQUIRED) - endif () -endif () - -mark_as_advanced( - CARES_INCLUDE_DIR - CARES_LIBRARIES - CARES_SHARED_LIB - CARES_STATIC_LIB -) diff --git a/cpp/cmake_modules/Findc-aresAlt.cmake b/cpp/cmake_modules/Findc-aresAlt.cmake new file mode 100644 index 00000000000..c87a3e3aa3f --- /dev/null +++ b/cpp/cmake_modules/Findc-aresAlt.cmake @@ -0,0 +1,45 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +if(c-ares_ROOT) + find_library(CARES_LIB + NAMES cares + PATHS ${c-ares_ROOT} "${c-ares_ROOT}/Library" + PATH_SUFFIXES "lib64" "lib" "bin" + NO_DEFAULT_PATH) + find_path(CARES_INCLUDE_DIR NAMES ares.h + PATHS ${CARES_ROOT} "${CARES_ROOT}/Library" + NO_DEFAULT_PATH + PATH_SUFFIXES "include") +else() + find_library(CARES_LIB + NAMES cares + PATH_SUFFIXES "lib64" "lib" "bin") + find_path(CARES_INCLUDE_DIR NAMES ares.h + PATH_SUFFIXES "include") +endif() + +find_package_handle_standard_args(c-aresAlt + REQUIRED_VARS CARES_INCLUDE_DIR CARES_LIB) + +if (c-aresAlt_FOUND) + add_library(c-ares::cares UNKNOWN IMPORTED) + set_target_properties(c-ares::cares PROPERTIES + IMPORTED_LOCATION "${CARES_LIB}" + INTERFACE_INCLUDE_DIRECTORIES "${CARES_INCLUDE_DIR}" + ) +endif() diff --git a/cpp/cmake_modules/FindgRPC.cmake b/cpp/cmake_modules/FindgRPC.cmake deleted file mode 100644 index edf72864a72..00000000000 --- a/cpp/cmake_modules/FindgRPC.cmake +++ /dev/null @@ -1,101 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -if( NOT "${GRPC_HOME}" STREQUAL "") - file (TO_CMAKE_PATH "${GRPC_HOME}" _grpc_path) -endif() - -message (STATUS "GRPC_HOME: ${GRPC_HOME}") - -find_package(gRPC CONFIG) -if (gRPC_FOUND) - message (STATUS "Found CMake installation of gRPC") - get_property(GRPC_INCLUDE_DIR TARGET gRPC::gpr PROPERTY INTERFACE_INCLUDE_DIRECTORIES) - get_property(GPR_STATIC_LIB TARGET gRPC::gpr PROPERTY LOCATION) - get_property(GRPC_STATIC_LIB TARGET gRPC::grpc_unsecure PROPERTY LOCATION) - get_property(GRPCPP_STATIC_LIB TARGET gRPC::grpc++_unsecure PROPERTY LOCATION) - get_property(GRPC_ADDRESS_SORTING_STATIC_LIB - TARGET gRPC::address_sorting PROPERTY LOCATION) - # Get location of grpc_cpp_plugin so we can pass it to protoc - get_property(GRPC_CPP_PLUGIN TARGET gRPC::grpc_cpp_plugin PROPERTY LOCATION) -else() - find_path (GRPC_INCLUDE_DIR grpc/grpc.h HINTS - ${_grpc_path} - NO_DEFAULT_PATH - PATH_SUFFIXES "include") - - set (lib_dirs "lib") - if (EXISTS "${_grpc_path}/lib64") - set (lib_dirs "lib64" ${lib_dirs}) - endif () - if (EXISTS "${_grpc_path}/lib/${CMAKE_LIBRARY_ARCHITECTURE}") - set (lib_dirs "lib/${CMAKE_LIBRARY_ARCHITECTURE}" ${lib_dirs}) - endif () - - find_library (GPR_STATIC_LIB - NAMES "${CMAKE_STATIC_LIBRARY_PREFIX}gpr${CMAKE_STATIC_LIBRARY_SUFFIX}" - PATHS ${_grpc_path} - NO_DEFAULT_PATH - PATH_SUFFIXES ${lib_dirs}) - - # On Debian/Ubuntu, libaddress_sorting is statically linked. - find_library (GRPC_ADDRESS_SORTING_STATIC_LIB - NAMES "${CMAKE_STATIC_LIBRARY_PREFIX}address_sorting${CMAKE_STATIC_LIBRARY_SUFFIX}" - "${CMAKE_STATIC_LIBRARY_PREFIX}grpc++${CMAKE_STATIC_LIBRARY_SUFFIX}" - PATHS ${_grpc_path} - NO_DEFAULT_PATH - PATH_SUFFIXES ${lib_dirs}) - - find_library (GRPC_STATIC_LIB - NAMES "${CMAKE_STATIC_LIBRARY_PREFIX}grpc${CMAKE_STATIC_LIBRARY_SUFFIX}" - PATHS ${_grpc_path} - NO_DEFAULT_PATH - PATH_SUFFIXES ${lib_dirs}) - - find_library (GRPCPP_STATIC_LIB - NAMES "${CMAKE_STATIC_LIBRARY_PREFIX}grpc++${CMAKE_STATIC_LIBRARY_SUFFIX}" - PATHS ${_grpc_path} - NO_DEFAULT_PATH - PATH_SUFFIXES ${lib_dirs}) - - find_program(GRPC_CPP_PLUGIN grpc_cpp_plugin protoc-gen-grpc-cpp - HINTS ${_grpc_path} - NO_DEFAULT_PATH - PATH_SUFFIXES "bin") -endif() - -if (GRPC_INCLUDE_DIR AND GPR_STATIC_LIB AND GRPC_ADDRESS_SORTING_STATIC_LIB AND - GRPC_STATIC_LIB AND GRPCPP_STATIC_LIB AND GRPC_CPP_PLUGIN) - set (gRPC_FOUND TRUE) -else () - set (gRPC_FOUND FALSE) -endif () - -if (gRPC_FOUND) - message (STATUS "Found the gRPC headers: ${GRPC_INCLUDE_DIR}") -else() - if (_grpc_path) - set (GRPC_ERR_MSG "Could not find gRPC. Looked in ${_grpc_path}.") - else () - set (GRPC_ERR_MSG "Could not find gRPC in system search paths.") - endif() - - if (gRPC_FIND_REQUIRED) - message (FATAL_ERROR "${GRPC_ERR_MSG}") - else () - message (STATUS "${GRPC_ERR_MSG}") - endif () -endif() - -mark_as_advanced ( - GRPC_INCLUDE_DIR - ) diff --git a/cpp/cmake_modules/FindgRPCAlt.cmake b/cpp/cmake_modules/FindgRPCAlt.cmake new file mode 100644 index 00000000000..991dfff8fae --- /dev/null +++ b/cpp/cmake_modules/FindgRPCAlt.cmake @@ -0,0 +1,114 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +pkg_check_modules(GRPC_PC grpc) +if (GRPC_PC_FOUND) + set(GRPC_INCLUDE_DIR "${GRPC_PC_INCLUDEDIR}") + list(APPEND GRPC_PC_LIBRARY_DIRS "${GRPC_PC_LIBDIR}") + message(STATUS "${GRPC_PC_LIBRARY_DIRS}") + + find_library(GRPC_GPR_LIB gpr + PATHS ${GRPC_PC_LIBRARY_DIRS} + NO_DEFAULT_PATH) + find_library(GRPC_GRPC_LIB grpc + PATHS ${GRPC_PC_LIBRARY_DIRS} + NO_DEFAULT_PATH) + find_library(GRPC_GRPCPP_LIB grpc++ + PATHS ${GRPC_PC_LIBRARY_DIRS} + NO_DEFAULT_PATH) + find_library(GRPC_ADDRESS_SORTING_LIB address_sorting + PATHS ${GRPC_PC_LIBRARY_DIRS} + NO_DEFAULT_PATH) + find_program(GRPC_CPP_PLUGIN grpc_cpp_plugin + HINTS ${GRPC_PC_PREFIX} + NO_DEFAULT_PATH + PATH_SUFFIXES "bin" + ) +elseif(gRPC_ROOT) + find_library(GRPC_GPR_LIB gpr + PATHS ${GRPC_ROOT} "${GRPC_ROOT}/Library" + PATH_SUFFIXES "lib64" "lib" "bin" + NO_DEFAULT_PATH) + find_library(GRPC_GRPC_LIB grpc + PATHS ${GRPC_ROOT} "${GRPC_ROOT}/Library" + PATH_SUFFIXES "lib64" "lib" "bin" + NO_DEFAULT_PATH) + find_library(GRPC_GRPCPP_LIB grpc++ + PATHS ${GRPC_ROOT} "${GRPC_ROOT}/Library" + PATH_SUFFIXES "lib64" "lib" "bin" + NO_DEFAULT_PATH) + find_library(GRPC_ADDRESS_SORTING_LIB address_sorting + PATHS ${GRPC_ROOT} "${GRPC_ROOT}/Library" + PATH_SUFFIXES "lib64" "lib" "bin" + NO_DEFAULT_PATH) + find_program(GRPC_CPP_PLUGIN grpc_cpp_plugin + NO_DEFAULT_PATH + PATHS ${GRPC_ROOT} "${GRPC_ROOT}/Library" + PATH_SUFFIXES "bin" + ) + find_path(GRPC_INCLUDE_DIR NAMES grpc/grpc.h + PATHS ${GRPC_ROOT} "${GRPC_ROOT}/Library" + NO_DEFAULT_PATH + PATH_SUFFIXES "include") +else() + find_library(GRPC_GPR_LIB gpr + PATH_SUFFIXES "lib64" "lib" "bin") + find_library(GRPC_GRPC_LIB grpc + PATH_SUFFIXES "lib64" "lib" "bin") + find_library(GRPC_GRPCPP_LIB grpc++ + PATH_SUFFIXES "lib64" "lib" "bin") + find_library(GRPC_ADDRESS_SORTING_LIB address_sorting + PATH_SUFFIXES "lib64" "lib" "bin") + find_program(GRPC_CPP_PLUGIN grpc_cpp_plugin + PATH_SUFFIXES "bin" + ) + find_path(GRPC_INCLUDE_DIR NAMES grpc/grpc.h + PATH_SUFFIXES "include") +endif() + +find_package_handle_standard_args(gRPCAlt + REQUIRED_VARS GRPC_INCLUDE_DIR GRPC_GPR_LIB GRPC_GRPC_LIB GRPC_GRPCPP_LIB GRPC_CPP_PLUGIN) + +if (gRPCAlt_FOUND) + add_library(gRPC::gpr UNKNOWN IMPORTED) + set_target_properties(gRPC::gpr PROPERTIES + IMPORTED_LOCATION "${GRPC_GPR_LIB}" + INTERFACE_INCLUDE_DIRECTORIES "${GRPC_INCLUDE_DIR}" + ) + + add_library(gRPC::grpc UNKNOWN IMPORTED) + set_target_properties(gRPC::grpc PROPERTIES + IMPORTED_LOCATION "${GRPC_GRPC_LIB}" + INTERFACE_INCLUDE_DIRECTORIES "${GRPC_INCLUDE_DIR}" + ) + + add_library(gRPC::grpc++ UNKNOWN IMPORTED) + set_target_properties(gRPC::grpc++ PROPERTIES + IMPORTED_LOCATION "${GRPC_GRPCPP_LIB}" + INTERFACE_INCLUDE_DIRECTORIES "${GRPC_INCLUDE_DIR}" + ) + + if (GRPC_ADDRESS_SORTING_LIB) + # Address sorting is optional and not always requird. + add_library(gRPC::address_sorting UNKNOWN IMPORTED) + set_target_properties(gRPC::address_sorting PROPERTIES + IMPORTED_LOCATION "${GRPC_ADDRESS_SORTING_LIB}" + INTERFACE_INCLUDE_DIRECTORIES "${GRPC_INCLUDE_DIR}" + ) + endif() + + add_executable(gRPC::grpc_cpp_plugin IMPORTED) + set_target_properties(gRPC::grpc_cpp_plugin PROPERTIES + IMPORTED_LOCATION ${GRPC_CPP_PLUGIN} + ) +endif() diff --git a/cpp/cmake_modules/FindgflagsAlt.cmake b/cpp/cmake_modules/FindgflagsAlt.cmake new file mode 100644 index 00000000000..c7be783ca9f --- /dev/null +++ b/cpp/cmake_modules/FindgflagsAlt.cmake @@ -0,0 +1,43 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +if(gflags_ROOT) + find_library(gflags_LIB + NAMES gflags + PATHS ${gflags_ROOT} + NO_DEFAULT_PATH) + find_path(GFLAGS_INCLUDE_DIR NAMES gflags/gflags.h + PATHS ${gflags_ROOT} NO_DEFAULT_PATH + PATH_SUFFIXES "include") +else() + find_library(gflags_LIB + NAMES gflags) + find_path(GFLAGS_INCLUDE_DIR NAMES gflags/gflags.h + PATH_SUFFIXES "include") +endif() + +find_package_handle_standard_args(gflagsAlt + REQUIRED_VARS gflags_LIB GFLAGS_INCLUDE_DIR) + +if (gflagsAlt_FOUND) + add_library(gflags::gflags UNKNOWN IMPORTED) + set_target_properties(gflags::gflags PROPERTIES + IMPORTED_LOCATION "${gflags_LIB}" + INTERFACE_INCLUDE_DIRECTORIES "${GFLAGS_INCLUDE_DIR}" + ) + set(GFLAGS_LIBRARIES gflags::gflags) +endif() diff --git a/cpp/cmake_modules/SetupCxxFlags.cmake b/cpp/cmake_modules/SetupCxxFlags.cmake index 852ea96f5dc..d2148354e0b 100644 --- a/cpp/cmake_modules/SetupCxxFlags.cmake +++ b/cpp/cmake_modules/SetupCxxFlags.cmake @@ -128,10 +128,11 @@ if ("${BUILD_WARNING_LEVEL}" STREQUAL "CHECKIN") -Wno-global-constructors -Wno-weak-template-vtables -Wno-undefined-reinterpret-cast \ -Wno-implicit-fallthrough -Wno-unreachable-code-return \ -Wno-float-equal -Wno-missing-prototypes -Wno-documentation-unknown-command \ +-Wno-documentation \ -Wno-old-style-cast -Wno-covered-switch-default \ -Wno-cast-align -Wno-vla-extension -Wno-shift-sign-overflow \ -Wno-used-but-marked-unused -Wno-missing-variable-declarations \ --Wno-gnu-zero-variadic-macro-arguments -Wconversion -Wno-sign-conversion \ +-Wno-gnu-zero-variadic-macro-arguments -Wno-conversion -Wno-sign-conversion \ -Wno-disabled-macro-expansion -Wno-format-nonliteral -Wno-missing-noreturn") # Version numbers where warnings are introduced @@ -155,7 +156,7 @@ if ("${BUILD_WARNING_LEVEL}" STREQUAL "CHECKIN") set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -Wno-unknown-warning-option -Werror") elseif ("${COMPILER_FAMILY}" STREQUAL "gcc") set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -Wall \ --Wconversion -Wno-sign-conversion") +-Wno-conversion -Wno-sign-conversion") # Treat all compiler warnings as errors set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -Werror") diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index 50a7e971d5b..baefc6b9738 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -15,6 +15,7 @@ # specific language governing permissions and limitations # under the License. +add_custom_target(rapidjson) add_custom_target(toolchain) add_custom_target(toolchain-benchmarks) add_custom_target(toolchain-tests) @@ -25,134 +26,136 @@ add_custom_target(toolchain-tests) set(ARROW_RE2_LINKAGE "static" CACHE STRING "How to link the re2 library. static|shared (default static)") -# ---------------------------------------------------------------------- -# Thirdparty versions, environment variables, source URLs - -set(THIRDPARTY_DIR "${arrow_SOURCE_DIR}/thirdparty") +if (ARROW_PROTOBUF_USE_SHARED) + set(Protobuf_USE_STATIC_LIBS OFF) +else() + set(Protobuf_USE_STATIC_LIBS ON) +endif() +# ---------------------------------------------------------------------- +# Resolve the dependencies + +message(STATUS "Using ${ARROW_DEPENDENCY_SOURCE} approach to find dependencies") + +set(ARROW_ACTUAL_DEPENDENCY_SOURCE "${ARROW_DEPENDENCY_SOURCE}") +set(ARROW_THIRDPARTY_DEPENDENCIES + double-conversion + BROTLI + Snappy + gflags + Thrift + Protobuf + GTEST + benchmark + RapidJSON + Flatbuffers + ZLIB + BZip2 + LZ4 + ZSTD + RE2 + c-ares + LLVM + BOOST) + +# TODO: double-conversion check fails for conda, it should not +if(ARROW_DEPENDENCY_SOURCE STREQUAL "CONDA") + message(STATUS "Using CONDA_PREFIX: $ENV{CONDA_PREFIX}") + set(ARROW_PACKAGE_PREFIX $ENV{CONDA_PREFIX}) + set(ARROW_ACTUAL_DEPENDENCY_SOURCE "SYSTEM") +endif() + +if (ARROW_PACKAGE_PREFIX) + message(STATUS "Setting (unset) dependency *_ROOT variables: ${ARROW_PACKAGE_PREFIX}") + set(ENV{PKG_CONFIG_PATH} "${ARROW_PACKAGE_PREFIX}/lib/pkgconfig/") -if (NOT "$ENV{ARROW_BUILD_TOOLCHAIN}" STREQUAL "") - set(BROTLI_HOME "$ENV{ARROW_BUILD_TOOLCHAIN}") - set(BZ2_HOME "$ENV{ARROW_BUILD_TOOLCHAIN}") - set(CARES_HOME "$ENV{ARROW_BUILD_TOOLCHAIN}") - set(DOUBLE_CONVERSION_HOME "$ENV{ARROW_BUILD_TOOLCHAIN}") - set(FLATBUFFERS_HOME "$ENV{ARROW_BUILD_TOOLCHAIN}") - set(GFLAGS_HOME "$ENV{ARROW_BUILD_TOOLCHAIN}") - set(GLOG_HOME "$ENV{ARROW_BUILD_TOOLCHAIN}") - set(GRPC_HOME "$ENV{ARROW_BUILD_TOOLCHAIN}") - # Using gtest from the toolchain breaks AppVeyor and - # trusty builds if (NOT MSVC) - if (APPLE) - set(GTEST_HOME "$ENV{ARROW_BUILD_TOOLCHAIN}") - else() - #linux - execute_process(COMMAND lsb_release -cs - OUTPUT_VARIABLE RELEASE_CODENAME - OUTPUT_STRIP_TRAILING_WHITESPACE - ) - if (NOT RELEASE_CODENAME STREQUAL "trusty") - set(GTEST_HOME "$ENV{ARROW_BUILD_TOOLCHAIN}") - endif() + if (NOT ENV{BOOST_ROOT}) + set(ENV{BOOST_ROOT} ${ARROW_PACKAGE_PREFIX}) + endif() + if (NOT Boost_ROOT}) + set(ENV{Boost_ROOT} ${ARROW_PACKAGE_PREFIX}) endif() endif() - set(JEMALLOC_HOME "$ENV{ARROW_BUILD_TOOLCHAIN}") - set(LZ4_HOME "$ENV{ARROW_BUILD_TOOLCHAIN}") - # orc disabled as it's not in conda-forge (but in Anaconda with an incompatible ABI) - # set(ORC_HOME "$ENV{ARROW_BUILD_TOOLCHAIN}") - set(PROTOBUF_HOME "$ENV{ARROW_BUILD_TOOLCHAIN}") - set(RAPIDJSON_HOME "$ENV{ARROW_BUILD_TOOLCHAIN}") - set(RE2_HOME "$ENV{ARROW_BUILD_TOOLCHAIN}") - set(SNAPPY_HOME "$ENV{ARROW_BUILD_TOOLCHAIN}") - set(THRIFT_HOME "$ENV{ARROW_BUILD_TOOLCHAIN}") - set(ZLIB_HOME "$ENV{ARROW_BUILD_TOOLCHAIN}") - set(ZSTD_HOME "$ENV{ARROW_BUILD_TOOLCHAIN}") - - if (NOT DEFINED ENV{BOOST_ROOT}) - # Since we have to set this in the environment, we check whether - # $BOOST_ROOT is defined inside here - set(ENV{BOOST_ROOT} "$ENV{ARROW_BUILD_TOOLCHAIN}") - endif() -endif() - -# Home path for each third-party lib can be overriden with env vars - -if (DEFINED ENV{BROTLI_HOME}) - set(BROTLI_HOME "$ENV{BROTLI_HOME}") -endif() - -if (DEFINED ENV{BZ2_HOME}) - set(BZ2_HOME "$ENV{BZ2_HOME}") -endif() - -if (DEFINED ENV{CARES_HOME}) - set(CARES_HOME "$ENV{CARES_HOME}") -endif() - -if (DEFINED ENV{DOUBLE_CONVERSION_HOME}) - set(DOUBLE_CONVERSION_HOME "$ENV{DOUBLE_CONVERSION_HOME}") -endif() -if (DEFINED ENV{FLATBUFFERS_HOME}) - set(FLATBUFFERS_HOME "$ENV{FLATBUFFERS_HOME}") + foreach(DEPENDENCY ${ARROW_THIRDPARTY_DEPENDENCIES}) + if (NOT ${DEPENDENCY}_ROOT) + set(${DEPENDENCY}_ROOT ${ARROW_PACKAGE_PREFIX}) + endif() + endforeach() endif() -if (DEFINED ENV{GFLAGS_HOME}) - set(GFLAGS_HOME "$ENV{GFLAGS_HOME}") -endif() +function(get_dependency_source DEPENDENCY_NAME) + if ("${${DEPENDENCY_NAME}_SOURCE}" STREQUAL "") + set(${DEPENDENCY_NAME}_SOURCE ${ARROW_ACTUAL_DEPENDENCY_SOURCE} PARENT_SCOPE) + endif() -if (DEFINED ENV{GLOG_HOME}) - set(GLOG_HOME "$ENV{GLOG_HOME}") -endif() + # If AUTO was specified and a ROOT was supplied, use this root. + if (${DEPENDENCY_NAME}_ROOT AND ${DEPENDENCY_NAME}_SOURCE STREQUAL "AUTO") + set(${DEPENDENCY_NAME}_SOURCE "SYSTEM" PARENT_SCOPE) + endif() +endfunction() + +macro(build_dependency DEPENDENCY_NAME) + if ("${DEPENDENCY_NAME}" STREQUAL "Brotli") + build_brotli() + elseif ("${DEPENDENCY_NAME}" STREQUAL "GLOG") + build_glog() + elseif ("${DEPENDENCY_NAME}" STREQUAL "gflags") + build_gflags() + elseif ("${DEPENDENCY_NAME}" STREQUAL "Thrift") + build_thrift() + elseif ("${DEPENDENCY_NAME}" STREQUAL "Protobuf") + build_protobuf() + elseif ("${DEPENDENCY_NAME}" STREQUAL "GTest") + build_gtest() + elseif ("${DEPENDENCY_NAME}" STREQUAL "benchmark") + build_benchmark() + elseif ("${DEPENDENCY_NAME}" STREQUAL "Flatbuffers") + build_flatbuffers() + elseif ("${DEPENDENCY_NAME}" STREQUAL "ZLIB") + build_ZLIB() + elseif ("${DEPENDENCY_NAME}" STREQUAL "Lz4") + build_lz4() + elseif ("${DEPENDENCY_NAME}" STREQUAL "ZSTD") + build_zstd() + elseif ("${DEPENDENCY_NAME}" STREQUAL "RE2") + build_re2() + elseif ("${DEPENDENCY_NAME}" STREQUAL "c-ares") + build_cares() + elseif ("${DEPENDENCY_NAME}" STREQUAL "gRPC") + build_grpc() + elseif ("${DEPENDENCY_NAME}" STREQUAL "BZip2") + build_bzip2() + else() + message(FATAL_ERROR "Unknown thirdparty dependency to build: ${DEPENDENCY_NAME}") + endif() +endmacro() -if (DEFINED ENV{GRPC_HOME}) - set(GRPC_HOME "$ENV{GRPC_HOME}") -endif() +macro(resolve_dependency DEPENDENCY_NAME) + get_dependency_source(${DEPENDENCY_NAME}) -if (DEFINED ENV{GTEST_HOME}) - set(GTEST_HOME "$ENV{GTEST_HOME}") -endif() + if (${DEPENDENCY_NAME}_SOURCE STREQUAL "AUTO") + find_package(${DEPENDENCY_NAME} QUIET) + if (NOT ${DEPENDENCY_NAME}_FOUND) + build_dependency(${DEPENDENCY_NAME}) + endif() + elseif(${DEPENDENCY_NAME}_SOURCE STREQUAL "BUNDLED") + build_dependency(${DEPENDENCY_NAME}) + elseif(${DEPENDENCY_NAME}_SOURCE STREQUAL "SYSTEM") + find_package(${DEPENDENCY_NAME} REQUIRED) + endif() +endmacro() -if (DEFINED ENV{JEMALLOC_HOME}) - set(JEMALLOC_HOME "$ENV{JEMALLOC_HOME}") -endif() +# ---------------------------------------------------------------------- +# Thirdparty versions, environment variables, source URLs -if (DEFINED ENV{LZ4_HOME}) - set(LZ4_HOME "$ENV{LZ4_HOME}") -endif() +set(THIRDPARTY_DIR "${arrow_SOURCE_DIR}/thirdparty") if (DEFINED ENV{ORC_HOME}) set(ORC_HOME "$ENV{ORC_HOME}") endif() -if (DEFINED ENV{PROTOBUF_HOME}) - set(PROTOBUF_HOME "$ENV{PROTOBUF_HOME}") -endif() - -if (DEFINED ENV{RAPIDJSON_HOME}) - set(RAPIDJSON_HOME "$ENV{RAPIDJSON_HOME}") -endif() - -if (DEFINED ENV{RE2_HOME}) - set(RE2_HOME "$ENV{RAPIDJSON_HOME}") -endif() - -if (DEFINED ENV{SNAPPY_HOME}) - set(SNAPPY_HOME "$ENV{SNAPPY_HOME}") -endif() - -if (DEFINED ENV{THRIFT_HOME}) - set(THRIFT_HOME "$ENV{THRIFT_HOME}") -endif() - -if (DEFINED ENV{ZLIB_HOME}) - set(ZLIB_HOME "$ENV{ZLIB_HOME}") -endif() - -if (DEFINED ENV{ZSTD_HOME}) - set(ZSTD_HOME "$ENV{ZSTD_HOME}") -endif() - # ---------------------------------------------------------------------- # Some EP's require other EP's @@ -171,6 +174,7 @@ if (ARROW_FLIGHT) endif() if (ARROW_FLIGHT OR ARROW_IPC) + set(ARROW_WITH_FLATBUFFERS ON) set(ARROW_WITH_RAPIDJSON ON) endif() @@ -291,7 +295,6 @@ else() set(RE2_SOURCE_URL "https://github.com/google/re2/archive/${RE2_VERSION}.tar.gz") endif() -set(RAPIDJSON_SOURCE_MD5 "badd12c511e081fec6c89c43a7027bce") if (DEFINED ENV{ARROW_RAPIDJSON_URL}) set(RAPIDJSON_SOURCE_URL "$ENV{ARROW_RAPIDJSON_URL}") else() @@ -322,6 +325,12 @@ else() set(ZSTD_SOURCE_URL "https://github.com/facebook/zstd/archive/${ZSTD_VERSION}.tar.gz") endif() +if (DEFINED ENV{BZIP2_SOURCE_URL}) + set(BZIP2_SOURCE_URL "$ENV{BZIP2_SOURCE_URL}") +else() + set(BZIP2_SOURCE_URL "https://fossies.org/linux/misc/bzip2-${BZIP2_VERSION}.tar.gz") +endif() + # ---------------------------------------------------------------------- # ExternalProject options @@ -399,165 +408,13 @@ endif() set(THREADS_PREFER_PTHREAD_FLAG ON) find_package(Threads REQUIRED) -# ---------------------------------------------------------------------- -# Add Boost dependencies (code adapted from Apache Kudu (incubating)) - -set(Boost_USE_MULTITHREADED ON) -if (MSVC AND ARROW_USE_STATIC_CRT) - set(Boost_USE_STATIC_RUNTIME ON) -endif() -set(Boost_ADDITIONAL_VERSIONS - "1.70.0" "1.70" - "1.69.0" "1.69" - "1.68.0" "1.68" - "1.67.0" "1.67" - "1.66.0" "1.66" - "1.65.0" "1.65" - "1.64.0" "1.64" - "1.63.0" "1.63" - "1.62.0" "1.61" - "1.61.0" "1.62" - "1.60.0" "1.60") - -if (ARROW_BOOST_VENDORED) - set(BOOST_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/boost_ep-prefix/src/boost_ep") - set(BOOST_LIB_DIR "${BOOST_PREFIX}/stage/lib") - set(BOOST_BUILD_LINK "static") - set(BOOST_STATIC_SYSTEM_LIBRARY - "${BOOST_LIB_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}boost_system${CMAKE_STATIC_LIBRARY_SUFFIX}") - set(BOOST_STATIC_FILESYSTEM_LIBRARY - "${BOOST_LIB_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}boost_filesystem${CMAKE_STATIC_LIBRARY_SUFFIX}") - set(BOOST_STATIC_REGEX_LIBRARY - "${BOOST_LIB_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}boost_regex${CMAKE_STATIC_LIBRARY_SUFFIX}") - set(BOOST_SYSTEM_LIBRARY boost_system_static) - set(BOOST_FILESYSTEM_LIBRARY boost_filesystem_static) - set(BOOST_REGEX_LIBRARY boost_regex_static) - - if (ARROW_BOOST_HEADER_ONLY) - set(BOOST_BUILD_PRODUCTS) - set(BOOST_CONFIGURE_COMMAND "") - set(BOOST_BUILD_COMMAND "") - else() - set(BOOST_BUILD_PRODUCTS - ${BOOST_STATIC_SYSTEM_LIBRARY} - ${BOOST_STATIC_FILESYSTEM_LIBRARY} - ${BOOST_STATIC_REGEX_LIBRARY}) - set(BOOST_CONFIGURE_COMMAND - "./bootstrap.sh" - "--prefix=${BOOST_PREFIX}" - "--with-libraries=filesystem,regex,system") - if ("${CMAKE_BUILD_TYPE}" STREQUAL "DEBUG") - set(BOOST_BUILD_VARIANT "debug") - else() - set(BOOST_BUILD_VARIANT "release") - endif() - set(BOOST_BUILD_COMMAND - "./b2" - "link=${BOOST_BUILD_LINK}" - "variant=${BOOST_BUILD_VARIANT}" - "cxxflags=-fPIC") - endif() - ExternalProject_Add(boost_ep - URL ${BOOST_SOURCE_URL} - BUILD_BYPRODUCTS ${BOOST_BUILD_PRODUCTS} - BUILD_IN_SOURCE 1 - CONFIGURE_COMMAND ${BOOST_CONFIGURE_COMMAND} - BUILD_COMMAND ${BOOST_BUILD_COMMAND} - INSTALL_COMMAND "" - ${EP_LOG_OPTIONS}) - set(Boost_INCLUDE_DIR "${BOOST_PREFIX}") - set(Boost_INCLUDE_DIRS "${BOOST_INCLUDE_DIR}") - add_dependencies(toolchain boost_ep) -else() - if (MSVC) - # disable autolinking in boost - add_definitions(-DBOOST_ALL_NO_LIB) - endif() - - if (DEFINED ENV{BOOST_ROOT} OR DEFINED BOOST_ROOT) - # In older versions of CMake (such as 3.2), the system paths for Boost will - # be looked in first even if we set $BOOST_ROOT or pass -DBOOST_ROOT - set(Boost_NO_SYSTEM_PATHS ON) - endif() - - if (ARROW_BOOST_USE_SHARED) - # Find shared Boost libraries. - set(Boost_USE_STATIC_LIBS OFF) - - if (MSVC) - # force all boost libraries to dynamic link - add_definitions(-DBOOST_ALL_DYN_LINK) - endif() - - if (ARROW_BOOST_HEADER_ONLY) - find_package(Boost REQUIRED) - else() - find_package(Boost COMPONENTS regex system filesystem REQUIRED) - if ("${CMAKE_BUILD_TYPE}" STREQUAL "DEBUG") - set(BOOST_SHARED_SYSTEM_LIBRARY ${Boost_SYSTEM_LIBRARY_DEBUG}) - set(BOOST_SHARED_FILESYSTEM_LIBRARY ${Boost_FILESYSTEM_LIBRARY_DEBUG}) - set(BOOST_SHARED_REGEX_LIBRARY ${Boost_REGEX_LIBRARY_DEBUG}) - else() - set(BOOST_SHARED_SYSTEM_LIBRARY ${Boost_SYSTEM_LIBRARY_RELEASE}) - set(BOOST_SHARED_FILESYSTEM_LIBRARY ${Boost_FILESYSTEM_LIBRARY_RELEASE}) - set(BOOST_SHARED_REGEX_LIBRARY ${Boost_REGEX_LIBRARY_RELEASE}) - endif() - set(BOOST_SYSTEM_LIBRARY boost_system_shared) - set(BOOST_FILESYSTEM_LIBRARY boost_filesystem_shared) - set(BOOST_REGEX_LIBRARY boost_regex_shared) - endif() - else() - # Find static boost headers and libs - # TODO Differentiate here between release and debug builds - set(Boost_USE_STATIC_LIBS ON) - if (ARROW_BOOST_HEADER_ONLY) - find_package(Boost REQUIRED) - else() - find_package(Boost COMPONENTS regex system filesystem REQUIRED) - if ("${CMAKE_BUILD_TYPE}" STREQUAL "DEBUG") - set(BOOST_STATIC_SYSTEM_LIBRARY ${Boost_SYSTEM_LIBRARY_DEBUG}) - set(BOOST_STATIC_FILESYSTEM_LIBRARY ${Boost_FILESYSTEM_LIBRARY_DEBUG}) - set(BOOST_STATIC_REGEX_LIBRARY ${Boost_REGEX_LIBRARY_DEBUG}) - else() - set(BOOST_STATIC_SYSTEM_LIBRARY ${Boost_SYSTEM_LIBRARY_RELEASE}) - set(BOOST_STATIC_FILESYSTEM_LIBRARY ${Boost_FILESYSTEM_LIBRARY_RELEASE}) - set(BOOST_STATIC_REGEX_LIBRARY ${Boost_REGEX_LIBRARY_RELEASE}) - endif() - set(BOOST_SYSTEM_LIBRARY boost_system_static) - set(BOOST_FILESYSTEM_LIBRARY boost_filesystem_static) - set(BOOST_REGEX_LIBRARY boost_regex_static) - endif() - endif() -endif() - -message(STATUS "Boost include dir: " ${Boost_INCLUDE_DIR}) -message(STATUS "Boost libraries: " ${Boost_LIBRARIES}) - -if (NOT ARROW_BOOST_HEADER_ONLY) - ADD_THIRDPARTY_LIB(boost_system - STATIC_LIB "${BOOST_STATIC_SYSTEM_LIBRARY}" - SHARED_LIB "${BOOST_SHARED_SYSTEM_LIBRARY}") - - ADD_THIRDPARTY_LIB(boost_filesystem - STATIC_LIB "${BOOST_STATIC_FILESYSTEM_LIBRARY}" - SHARED_LIB "${BOOST_SHARED_FILESYSTEM_LIBRARY}") - - ADD_THIRDPARTY_LIB(boost_regex - STATIC_LIB "${BOOST_STATIC_REGEX_LIBRARY}" - SHARED_LIB "${BOOST_SHARED_REGEX_LIBRARY}") - - SET(ARROW_BOOST_LIBS ${BOOST_SYSTEM_LIBRARY} ${BOOST_FILESYSTEM_LIBRARY}) -endif() - -include_directories(SYSTEM ${Boost_INCLUDE_DIR}) - # ---------------------------------------------------------------------- # Google double-conversion -if("${DOUBLE_CONVERSION_HOME}" STREQUAL "") +macro(build_double_conversion) + message(STATUS "Building double-conversion from source") set(DOUBLE_CONVERSION_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/double-conversion_ep/src/double-conversion_ep") - set(DOUBLE_CONVERSION_HOME "${DOUBLE_CONVERSION_PREFIX}") - set(DOUBLE_CONVERSION_INCLUDE_DIR "${DOUBLE_CONVERSION_PREFIX}/include") + set(double-conversion_INCLUDE_DIRS "${DOUBLE_CONVERSION_PREFIX}/include") set(DOUBLE_CONVERSION_STATIC_LIB "${DOUBLE_CONVERSION_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}double-conversion${CMAKE_STATIC_LIBRARY_SUFFIX}") set(DOUBLE_CONVERSION_CMAKE_ARGS @@ -570,316 +427,585 @@ if("${DOUBLE_CONVERSION_HOME}" STREQUAL "") URL ${DOUBLE_CONVERSION_SOURCE_URL} CMAKE_ARGS ${DOUBLE_CONVERSION_CMAKE_ARGS} BUILD_BYPRODUCTS "${DOUBLE_CONVERSION_STATIC_LIB}") - set(DOUBLE_CONVERSION_VENDORED 1) + + add_library(double-conversion STATIC IMPORTED) + set_target_properties(double-conversion + PROPERTIES IMPORTED_LOCATION "${DOUBLE_CONVERSION_STATIC_LIB}") add_dependencies(toolchain double-conversion_ep) -else() - find_package(double-conversion REQUIRED - PATHS "${DOUBLE_CONVERSION_HOME}") - set(DOUBLE_CONVERSION_VENDORED 0) -endif() + add_dependencies(double-conversion double-conversion_ep) + set(double-conversion_LIBRARIES double-conversion) +endmacro() + +macro(double_conversion_config) + # Map the newer target to the old, simpler setting + if (TARGET double-conversion::double-conversion) + set(double-conversion_LIBRARIES double-conversion::double-conversion) + get_target_property(double-conversion_INCLUDE_DIRS double-conversion::double-conversion INTERFACE_INCLUDE_DIRECTORIES) + endif() +endmacro() + +macro(double_conversion_compability) + CHECK_CXX_SOURCE_COMPILES(" +#include +int main() { +const int flags_ = double_conversion::StringToDoubleConverter::ALLOW_CASE_INSENSIBILITY; + }" DOUBLE_CONVERSION_HAS_CASE_INSENSIBILITY) +endmacro() + +get_dependency_source(double-conversion) + +if (double-conversion_SOURCE STREQUAL "AUTO") + # Debian does not ship cmake configs for double-conversion + # TODO: Make upstream bug + find_package(double-conversion QUIET) + if (NOT double-conversion_FOUND) + find_package(DoubleConversion) + endif() + if (double-conversion_FOUND OR DoubleConversion_FOUND) + double_conversion_config() + else() + build_double_conversion() + endif() +elseif(double-conversion_SOURCE STREQUAL "BUNDLED") + build_double_conversion() +elseif(double-conversion_SOURCE STREQUAL "SYSTEM") + # Debian does not ship cmake configs for double-conversion + # TODO: Make upstream bug + find_package(double-conversion) + if (NOT double-conversion_FOUND) + find_package(DoubleConversion REQUIRED) + endif() -if (NOT DOUBLE_CONVERSION_VENDORED) - get_property(DOUBLE_CONVERSION_STATIC_LIB TARGET double-conversion::double-conversion - PROPERTY LOCATION) - get_property(DOUBLE_CONVERSION_INCLUDE_DIR TARGET double-conversion::double-conversion - PROPERTY INTERFACE_INCLUDE_DIRECTORIES) + double_conversion_config() endif() +# TODO: Don't use global includes but rather target_include_directories +include_directories(SYSTEM ${double-conversion_INCLUDE_DIRS}) -include_directories(SYSTEM ${DOUBLE_CONVERSION_INCLUDE_DIR}) - -ADD_THIRDPARTY_LIB(double-conversion - STATIC_LIB ${DOUBLE_CONVERSION_STATIC_LIB}) - -message(STATUS "double-conversion include dir: ${DOUBLE_CONVERSION_INCLUDE_DIR}") -message(STATUS "double-conversion static library: ${DOUBLE_CONVERSION_STATIC_LIB}") +double_conversion_compability() # ---------------------------------------------------------------------- -# gflags +# Snappy -if (ARROW_BUILD_TESTS OR - ARROW_BUILD_BENCHMARKS OR - (ARROW_USE_GLOG AND GLOG_HOME) OR - (ARROW_WITH_GRPC AND NOT GRPC_HOME)) - set(ARROW_NEED_GFLAGS 1) -else() - set(ARROW_NEED_GFLAGS 0) -endif() +macro(build_snappy) + message(STATUS "Building snappy from source") + set(SNAPPY_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/snappy_ep/src/snappy_ep-install") + if (MSVC) + set(SNAPPY_STATIC_LIB_NAME snappy_static) + else() + set(SNAPPY_STATIC_LIB_NAME snappy) + endif() + set(SNAPPY_STATIC_LIB "${SNAPPY_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}${SNAPPY_STATIC_LIB_NAME}${CMAKE_STATIC_LIBRARY_SUFFIX}") -if(ARROW_NEED_GFLAGS) - # gflags (formerly Googleflags) command line parsing - find_package(GFlags) - if(GFLAGS_FOUND) - set(GFLAGS_VENDORED FALSE) - get_filename_component(GFLAGS_HOME "${GFLAGS_INCLUDE_DIR}" DIRECTORY) - if(ARROW_GFLAGS_USE_SHARED AND GFLAGS_SHARED) - set(GFLAGS_LIBRARY gflags_shared) + if (${UPPERCASE_BUILD_TYPE} EQUAL "RELEASE") + if (APPLE) + set(SNAPPY_CXXFLAGS "CXXFLAGS='-DNDEBUG -O1'") else() - set(GFLAGS_LIBRARY gflags_static) + set(SNAPPY_CXXFLAGS "CXXFLAGS='-DNDEBUG -O2'") endif() - elseif(GFLAGS_HOME) - message(FATAL_ERROR "No static or shared library provided for gflags: ${GFLAGS_HOME}") + endif() + + if (WIN32) + set(SNAPPY_CMAKE_ARGS ${EP_COMMON_CMAKE_ARGS} + -DCMAKE_AR=${CMAKE_AR} + -DCMAKE_RANLIB=${CMAKE_RANLIB} + "-DCMAKE_INSTALL_PREFIX=${SNAPPY_PREFIX}") + set(SNAPPY_UPDATE_COMMAND ${CMAKE_COMMAND} -E copy + ${CMAKE_SOURCE_DIR}/cmake_modules/SnappyCMakeLists.txt + ./CMakeLists.txt && + ${CMAKE_COMMAND} -E copy + ${CMAKE_SOURCE_DIR}/cmake_modules/SnappyConfig.h + ./config.h) + ExternalProject_Add(snappy_ep + UPDATE_COMMAND ${SNAPPY_UPDATE_COMMAND} + ${EP_LOG_OPTIONS} + BUILD_IN_SOURCE 1 + BUILD_COMMAND ${MAKE} + INSTALL_DIR ${SNAPPY_PREFIX} + URL ${SNAPPY_SOURCE_URL} + CMAKE_ARGS ${SNAPPY_CMAKE_ARGS} + BUILD_BYPRODUCTS "${SNAPPY_STATIC_LIB}") else() - set(GFLAGS_VENDORED TRUE) - set(GFLAGS_CMAKE_CXX_FLAGS ${EP_CXX_FLAGS}) - - set(GFLAGS_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/gflags_ep-prefix/src/gflags_ep") - set(GFLAGS_HOME "${GFLAGS_PREFIX}") - set(GFLAGS_INCLUDE_DIR "${GFLAGS_PREFIX}/include") - if(MSVC) - set(GFLAGS_STATIC_LIB "${GFLAGS_PREFIX}/lib/gflags_static.lib") - else() - set(GFLAGS_STATIC_LIB "${GFLAGS_PREFIX}/lib/libgflags.a") - endif() - set(GFLAGS_VENDORED 1) - set(GFLAGS_CMAKE_ARGS ${EP_COMMON_CMAKE_ARGS} - "-DCMAKE_INSTALL_PREFIX=${GFLAGS_PREFIX}" - -DBUILD_SHARED_LIBS=OFF - -DBUILD_STATIC_LIBS=ON - -DBUILD_PACKAGING=OFF - -DBUILD_TESTING=OFF - -DBUILD_CONFIG_TESTS=OFF - -DINSTALL_HEADERS=ON) - - ExternalProject_Add(gflags_ep - URL ${GFLAGS_SOURCE_URL} + ExternalProject_Add(snappy_ep + CONFIGURE_COMMAND ./configure --with-pic "AR=${CMAKE_AR}" "RANLIB=${CMAKE_RANLIB}" "--prefix=${SNAPPY_PREFIX}" ${SNAPPY_CXXFLAGS} ${EP_LOG_OPTIONS} BUILD_IN_SOURCE 1 - BUILD_BYPRODUCTS "${GFLAGS_STATIC_LIB}" - CMAKE_ARGS ${GFLAGS_CMAKE_ARGS}) - - add_dependencies(toolchain gflags_ep) - - message(STATUS "GFlags include dir: ${GFLAGS_INCLUDE_DIR}") - message(STATUS "GFlags static library: ${GFLAGS_STATIC_LIB}") - include_directories(SYSTEM ${GFLAGS_INCLUDE_DIR}) - ADD_THIRDPARTY_LIB(gflags - STATIC_LIB ${GFLAGS_STATIC_LIB}) - set(GFLAGS_LIBRARY gflags_static) - if(WIN32) - set_target_properties(${GFLAGS_LIBRARY} - PROPERTIES - INTERFACE_LINK_LIBRARIES "shlwapi.lib" - INTERFACE_COMPILE_DEFINITIONS "GFLAGS_IS_A_DLL=0") - endif() - add_dependencies(${GFLAGS_LIBRARY} gflags_ep) + BUILD_COMMAND ${MAKE} + INSTALL_DIR ${SNAPPY_PREFIX} + URL ${SNAPPY_SOURCE_URL} + BUILD_BYPRODUCTS "${SNAPPY_STATIC_LIB}") endif() -endif() -# ---------------------------------------------------------------------- -# Google gtest + file(MAKE_DIRECTORY "${SNAPPY_PREFIX}/include") -if(ARROW_BUILD_TESTS OR ARROW_BUILD_BENCHMARKS) - if("${GTEST_HOME}" STREQUAL "") - set(GTEST_CMAKE_CXX_FLAGS ${EP_CXX_FLAGS}) + add_library(Snappy::snappy STATIC IMPORTED) + set_target_properties(Snappy::snappy PROPERTIES + IMPORTED_LOCATION "${SNAPPY_STATIC_LIB}" + INTERFACE_INCLUDE_DIRECTORIES "${SNAPPY_PREFIX}/include" + ) + add_dependencies(toolchain snappy_ep) + add_dependencies(Snappy::snappy snappy_ep) +endmacro() - if(CMAKE_BUILD_TYPE MATCHES DEBUG) - set(CMAKE_GTEST_DEBUG_EXTENSION "d") - else() - set(CMAKE_GTEST_DEBUG_EXTENSION "") +if (ARROW_WITH_SNAPPY) + get_dependency_source(Snappy) + if (Snappy_SOURCE STREQUAL "AUTO") + # Normally *Config.cmake files reside in /usr/lib/cmake but Snappy + # errornously places them in ${CMAKE_ROOT}/Modules/ + # This is fixed in 1.1.7 but fedora (30) still installs into the wrong + # location. + # https://bugzilla.redhat.com/show_bug.cgi?id=1679727 + # https://src.fedoraproject.org/rpms/snappy/pull-request/1 + find_package(Snappy QUIET HINTS "${CMAKE_ROOT}/Modules/") + if (NOT Snappy_FOUND) + find_package(SnappyAlt) endif() - - if(APPLE) - set(GTEST_CMAKE_CXX_FLAGS ${GTEST_CMAKE_CXX_FLAGS} - -DGTEST_USE_OWN_TR1_TUPLE=1 - -Wno-unused-value - -Wno-ignored-attributes) + if (NOT Snappy_FOUND AND NOT SnappyAlt_FOUND) + build_snappy() endif() - - set(GTEST_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/googletest_ep-prefix/src/googletest_ep") - set(GTEST_HOME ${GTEST_PREFIX}) - set(GTEST_INCLUDE_DIR "${GTEST_PREFIX}/include") - set(GTEST_STATIC_LIB - "${GTEST_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}gtest${CMAKE_GTEST_DEBUG_EXTENSION}${CMAKE_STATIC_LIBRARY_SUFFIX}") - set(GTEST_MAIN_STATIC_LIB - "${GTEST_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}gtest_main${CMAKE_GTEST_DEBUG_EXTENSION}${CMAKE_STATIC_LIBRARY_SUFFIX}") - set(GTEST_VENDORED 1) - set(GTEST_CMAKE_ARGS ${EP_COMMON_CMAKE_ARGS} - "-DCMAKE_INSTALL_PREFIX=${GTEST_PREFIX}" - "-DCMAKE_INSTALL_LIBDIR=lib" - -DCMAKE_CXX_FLAGS=${GTEST_CMAKE_CXX_FLAGS}) - set(GMOCK_INCLUDE_DIR "${GTEST_PREFIX}/include") - set(GMOCK_STATIC_LIB - "${GTEST_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}gmock${CMAKE_GTEST_DEBUG_EXTENSION}${CMAKE_STATIC_LIBRARY_SUFFIX}") - set(GMOCK_MAIN_STATIC_LIB - "${GTEST_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}gmock_main${CMAKE_GTEST_DEBUG_EXTENSION}${CMAKE_STATIC_LIBRARY_SUFFIX}") - - if (MSVC AND NOT ARROW_USE_STATIC_CRT) - set(GTEST_CMAKE_ARGS ${GTEST_CMAKE_ARGS} -Dgtest_force_shared_crt=ON) + elseif(Snappy_SOURCE STREQUAL "BUNDLED") + build_snappy() + elseif(Snappy_SOURCE STREQUAL "SYSTEM") + # SnappyConfig.cmake is not installed on Ubuntu/Debian + # TODO: Make a bug report upstream + find_package(Snappy HINTS "${CMAKE_ROOT}/Modules/") + if (NOT Snappy_FOUND) + find_package(SnappyAlt REQUIRED) endif() + endif() - ExternalProject_Add(googletest_ep - URL ${GTEST_SOURCE_URL} - BUILD_BYPRODUCTS ${GTEST_STATIC_LIB} ${GTEST_MAIN_STATIC_LIB} ${GMOCK_STATIC_LIB} ${GMOCK_MAIN_STATIC_LIB} - CMAKE_ARGS ${GTEST_CMAKE_ARGS} - ${EP_LOG_OPTIONS}) + # TODO: Don't use global includes but rather target_include_directories + get_target_property(SNAPPY_INCLUDE_DIRS Snappy::snappy INTERFACE_INCLUDE_DIRECTORIES) + include_directories(SYSTEM ${SNAPPY_INCLUDE_DIRS}) +endif() - add_dependencies(toolchain-tests googletest_ep) - else() - find_package(GTest REQUIRED) - set(GTEST_VENDORED 0) - endif() +# ---------------------------------------------------------------------- +# Brotli - message(STATUS "GTest include dir: ${GTEST_INCLUDE_DIR}") - message(STATUS "GMock include dir: ${GMOCK_INCLUDE_DIR}") - include_directories(SYSTEM ${GTEST_INCLUDE_DIR}) - if(GTEST_STATIC_LIB) - message(STATUS "GTest static library: ${GTEST_STATIC_LIB}") - message(STATUS "GMock static library: ${GMOCK_STATIC_LIB}") - ADD_THIRDPARTY_LIB(gtest - STATIC_LIB ${GTEST_STATIC_LIB}) - ADD_THIRDPARTY_LIB(gtest_main - STATIC_LIB ${GTEST_MAIN_STATIC_LIB}) - ADD_THIRDPARTY_LIB(gmock - STATIC_LIB ${GMOCK_STATIC_LIB}) - ADD_THIRDPARTY_LIB(gmock_main - STATIC_LIB ${GMOCK_MAIN_STATIC_LIB}) - set(GTEST_LIBRARY gtest_static) - set(GTEST_MAIN_LIBRARY gtest_main_static) - set(GMOCK_LIBRARY gmock_static) - set(GMOCK_MAIN_LIBRARY gmock_main_static) +macro(build_brotli) + message(STATUS "Building brotli from source") + set(BROTLI_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/brotli_ep/src/brotli_ep-install") + set(BROTLI_INCLUDE_DIR "${BROTLI_PREFIX}/include") + if (MSVC) + set(BROTLI_LIB_DIR bin) else() - message(STATUS "GTest shared library: ${GTEST_SHARED_LIB}") - message(STATUS "GMock shared library: ${GMOCK_SHARED_LIB}") - ADD_THIRDPARTY_LIB(gtest - SHARED_LIB ${GTEST_SHARED_LIB}) - ADD_THIRDPARTY_LIB(gtest_main - SHARED_LIB ${GTEST_MAIN_SHARED_LIB}) - ADD_THIRDPARTY_LIB(gmock - SHARED_LIB ${GMOCK_SHARED_LIB}) - ADD_THIRDPARTY_LIB(gmock_main - SHARED_LIB ${GMOCK_MAIN_SHARED_LIB}) - set(GTEST_LIBRARY gtest_shared) - set(GTEST_MAIN_LIBRARY gtest_main_shared) - set(GMOCK_LIBRARY gmock_shared) - set(GMOCK_MAIN_LIBRARY gmock_main_shared) - endif() - - if(GTEST_VENDORED) - add_dependencies(${GTEST_LIBRARY} googletest_ep) - add_dependencies(${GTEST_MAIN_LIBRARY} googletest_ep) - add_dependencies(${GMOCK_LIBRARY} googletest_ep) - add_dependencies(${GMOCK_MAIN_LIBRARY} googletest_ep) + set(BROTLI_LIB_DIR lib) endif() -endif() + set(BROTLI_STATIC_LIBRARY_ENC "${BROTLI_PREFIX}/${BROTLI_LIB_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}brotlienc${CMAKE_STATIC_LIBRARY_SUFFIX}") + set(BROTLI_STATIC_LIBRARY_DEC "${BROTLI_PREFIX}/${BROTLI_LIB_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}brotlidec${CMAKE_STATIC_LIBRARY_SUFFIX}") + set(BROTLI_STATIC_LIBRARY_COMMON "${BROTLI_PREFIX}/${BROTLI_LIB_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}brotlicommon${CMAKE_STATIC_LIBRARY_SUFFIX}") + set(BROTLI_CMAKE_ARGS ${EP_COMMON_CMAKE_ARGS} + "-DCMAKE_INSTALL_PREFIX=${BROTLI_PREFIX}" + -DCMAKE_INSTALL_LIBDIR=lib + -DBUILD_SHARED_LIBS=OFF) + + ExternalProject_Add(brotli_ep + URL ${BROTLI_SOURCE_URL} + BUILD_BYPRODUCTS "${BROTLI_STATIC_LIBRARY_ENC}" "${BROTLI_STATIC_LIBRARY_DEC}" "${BROTLI_STATIC_LIBRARY_COMMON}" + ${BROTLI_BUILD_BYPRODUCTS} + ${EP_LOG_OPTIONS} + CMAKE_ARGS ${BROTLI_CMAKE_ARGS} + STEP_TARGETS headers_copy) + if (MSVC) + ExternalProject_Get_Property(brotli_ep SOURCE_DIR) -if(ARROW_BUILD_BENCHMARKS) - if("$ENV{GBENCHMARK_HOME}" STREQUAL "") - if(CMAKE_VERSION VERSION_LESS 3.6) - message(FATAL_ERROR "Building gbenchmark from source requires at least CMake 3.6") - endif() + ExternalProject_Add_Step(brotli_ep headers_copy + COMMAND xcopy /E /I include ..\\..\\..\\brotli_ep\\src\\brotli_ep-install\\include /Y + DEPENDEES build + WORKING_DIRECTORY ${SOURCE_DIR}) + endif() - if(NOT MSVC) - set(GBENCHMARK_CMAKE_CXX_FLAGS "${EP_CXX_FLAGS} -std=c++11") - endif() + add_dependencies(toolchain brotli_ep) + file(MAKE_DIRECTORY "${BROTLI_INCLUDE_DIR}") - if(APPLE) - set(GBENCHMARK_CMAKE_CXX_FLAGS "${GBENCHMARK_CMAKE_CXX_FLAGS} -stdlib=libc++") - endif() + add_library(Brotli::brotlicommon STATIC IMPORTED) + set_target_properties(Brotli::brotlicommon PROPERTIES + IMPORTED_LOCATION "${BROTLI_STATIC_LIBRARY_COMMON}" + INTERFACE_INCLUDE_DIRECTORIES "${BROTLI_INCLUDE_DIR}" + ) + add_dependencies(Brotli::brotlicommon brotli_ep) - set(GBENCHMARK_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/gbenchmark_ep/src/gbenchmark_ep-install") - set(GBENCHMARK_INCLUDE_DIR "${GBENCHMARK_PREFIX}/include") - set(GBENCHMARK_STATIC_LIB "${GBENCHMARK_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}benchmark${CMAKE_STATIC_LIBRARY_SUFFIX}") - set(GBENCHMARK_VENDORED 1) - set(GBENCHMARK_CMAKE_ARGS ${EP_COMMON_CMAKE_ARGS} - "-DCMAKE_INSTALL_PREFIX=${GBENCHMARK_PREFIX}" - -DBENCHMARK_ENABLE_TESTING=OFF - -DCMAKE_CXX_FLAGS=${GBENCHMARK_CMAKE_CXX_FLAGS}) - if (APPLE) - set(GBENCHMARK_CMAKE_ARGS ${GBENCHMARK_CMAKE_ARGS} "-DBENCHMARK_USE_LIBCXX=ON") - endif() + add_library(Brotli::brotlienc STATIC IMPORTED) + set_target_properties(Brotli::brotlienc PROPERTIES + IMPORTED_LOCATION "${BROTLI_STATIC_LIBRARY_ENC}" + INTERFACE_INCLUDE_DIRECTORIES "${BROTLI_INCLUDE_DIR}" + ) + add_dependencies(Brotli::brotlienc brotli_ep) - ExternalProject_Add(gbenchmark_ep - URL ${GBENCHMARK_SOURCE_URL} - BUILD_BYPRODUCTS "${GBENCHMARK_STATIC_LIB}" - CMAKE_ARGS ${GBENCHMARK_CMAKE_ARGS} - ${EP_LOG_OPTIONS}) + add_library(Brotli::brotlidec STATIC IMPORTED) + set_target_properties(Brotli::brotlidec PROPERTIES + IMPORTED_LOCATION "${BROTLI_STATIC_LIBRARY_DEC}" + INTERFACE_INCLUDE_DIRECTORIES "${BROTLI_INCLUDE_DIR}" + ) + add_dependencies(Brotli::brotlidec brotli_ep) +endmacro() - add_dependencies(toolchain-benchmarks gbenchmark_ep) - else() - find_package(GBenchmark REQUIRED) - set(GBENCHMARK_VENDORED 0) - endif() - message(STATUS "GBenchmark include dir: ${GBENCHMARK_INCLUDE_DIR}") - message(STATUS "GBenchmark static library: ${GBENCHMARK_STATIC_LIB}") - include_directories(SYSTEM ${GBENCHMARK_INCLUDE_DIR}) - ADD_THIRDPARTY_LIB(gbenchmark - STATIC_LIB ${GBENCHMARK_STATIC_LIB}) +if (ARROW_WITH_BROTLI) + resolve_dependency(Brotli) + # TODO: Don't use global includes but rather target_include_directories + get_target_property(BROTLI_INCLUDE_DIR Brotli::brotlicommon INTERFACE_INCLUDE_DIRECTORIES) + include_directories(SYSTEM ${BROTLI_INCLUDE_DIR}) +endif() + +# ---------------------------------------------------------------------- +# GLOG + +macro(build_glog) + message(STATUS "Building glog from source") + set(GLOG_BUILD_DIR "${CMAKE_CURRENT_BINARY_DIR}/glog_ep-prefix/src/glog_ep") + set(GLOG_INCLUDE_DIR "${GLOG_BUILD_DIR}/include") + set(GLOG_STATIC_LIB "${GLOG_BUILD_DIR}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}glog${CMAKE_STATIC_LIBRARY_SUFFIX}") + set(GLOG_CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fPIC") + set(GLOG_CMAKE_C_FLAGS "${EP_C_FLAGS} -fPIC") + if (Threads::Threads) + set(GLOG_CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fPIC -pthread") + set(GLOG_CMAKE_C_FLAGS "${EP_C_FLAGS} -fPIC -pthread") + endif() - if(GBENCHMARK_VENDORED) - add_dependencies(gbenchmark_static gbenchmark_ep) + if(APPLE) + # If we don't set this flag, the binary built with 10.13 cannot be used in 10.12. + set(GLOG_CMAKE_CXX_FLAGS "${GLOG_CMAKE_CXX_FLAGS} -mmacosx-version-min=10.9") endif() + + set(GLOG_CMAKE_ARGS ${EP_COMMON_CMAKE_ARGS} + "-DCMAKE_INSTALL_PREFIX=${GLOG_BUILD_DIR}" + -DBUILD_SHARED_LIBS=OFF + -DBUILD_TESTING=OFF + -DWITH_GFLAGS=OFF + -DCMAKE_CXX_FLAGS_${UPPERCASE_BUILD_TYPE}=${GLOG_CMAKE_CXX_FLAGS} + -DCMAKE_C_FLAGS_${UPPERCASE_BUILD_TYPE}=${GLOG_CMAKE_C_FLAGS} + -DCMAKE_CXX_FLAGS=${GLOG_CMAKE_CXX_FLAGS}) + ExternalProject_Add(glog_ep + URL ${GLOG_SOURCE_URL} + BUILD_IN_SOURCE 1 + BUILD_BYPRODUCTS "${GLOG_STATIC_LIB}" + CMAKE_ARGS ${GLOG_CMAKE_ARGS} + ${EP_LOG_OPTIONS}) + + add_dependencies(toolchain glog_ep) + file(MAKE_DIRECTORY "${GLOG_INCLUDE_DIR}") + + add_library(GLOG::glog STATIC IMPORTED) + set_target_properties(GLOG::glog PROPERTIES + IMPORTED_LOCATION "${GLOG_STATIC_LIB}" + INTERFACE_INCLUDE_DIRECTORIES "${GLOG_INCLUDE_DIR}" + ) + add_dependencies(GLOG::glog glog_ep) +endmacro() + +if (ARROW_USE_GLOG) + resolve_dependency(GLOG) + # TODO: Don't use global includes but rather target_include_directories + get_target_property(GLOG_INCLUDE_DIR GLOG::glog INTERFACE_INCLUDE_DIRECTORIES) + include_directories(SYSTEM ${GLOG_INCLUDE_DIR}) endif() -if (ARROW_WITH_RAPIDJSON) - # RapidJSON, header only dependency - if("${RAPIDJSON_HOME}" STREQUAL "") - set(RAPIDJSON_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/rapidjson_ep/src/rapidjson_ep-install") - set(RAPIDJSON_HOME "${RAPIDJSON_PREFIX}") - set(RAPIDJSON_CMAKE_ARGS - -DRAPIDJSON_BUILD_DOC=OFF - -DRAPIDJSON_BUILD_EXAMPLES=OFF - -DRAPIDJSON_BUILD_TESTS=OFF - "-DCMAKE_INSTALL_PREFIX=${RAPIDJSON_PREFIX}") - - ExternalProject_Add(rapidjson_ep - ${EP_LOG_OPTIONS} - PREFIX "${CMAKE_BINARY_DIR}" - URL ${RAPIDJSON_SOURCE_URL} - URL_MD5 ${RAPIDJSON_SOURCE_MD5} - CMAKE_ARGS ${RAPIDJSON_CMAKE_ARGS}) +# ---------------------------------------------------------------------- +# gflags + +if (ARROW_BUILD_TESTS OR + ARROW_BUILD_BENCHMARKS OR + ARROW_USE_GLOG OR + ARROW_WITH_GRPC) + set(ARROW_NEED_GFLAGS 1) +else() + set(ARROW_NEED_GFLAGS 0) +endif() - set(RAPIDJSON_INCLUDE_DIR "${RAPIDJSON_HOME}/include") - set(RAPIDJSON_VENDORED 1) +macro(build_gflags) + message(STATUS "Building gflags from source") + set(GFLAGS_CMAKE_CXX_FLAGS ${EP_CXX_FLAGS}) - add_dependencies(toolchain rapidjson_ep) + set(GFLAGS_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/gflags_ep-prefix/src/gflags_ep") + set(GFLAGS_INCLUDE_DIR "${GFLAGS_PREFIX}/include") + if(MSVC) + set(GFLAGS_STATIC_LIB "${GFLAGS_PREFIX}/lib/gflags_static.lib") else() - set(RAPIDJSON_INCLUDE_DIR "${RAPIDJSON_HOME}/include") - set(RAPIDJSON_VENDORED 0) + set(GFLAGS_STATIC_LIB "${GFLAGS_PREFIX}/lib/libgflags.a") endif() - message(STATUS "RapidJSON include dir: ${RAPIDJSON_INCLUDE_DIR}") - include_directories(SYSTEM ${RAPIDJSON_INCLUDE_DIR}) + set(GFLAGS_CMAKE_ARGS ${EP_COMMON_CMAKE_ARGS} + "-DCMAKE_INSTALL_PREFIX=${GFLAGS_PREFIX}" + -DBUILD_SHARED_LIBS=OFF + -DBUILD_STATIC_LIBS=ON + -DBUILD_PACKAGING=OFF + -DBUILD_TESTING=OFF + -DBUILD_CONFIG_TESTS=OFF + -DINSTALL_HEADERS=ON) + + file(MAKE_DIRECTORY "${GFLAGS_INCLUDE_DIR}") + ExternalProject_Add(gflags_ep + URL ${GFLAGS_SOURCE_URL} + ${EP_LOG_OPTIONS} + BUILD_IN_SOURCE 1 + BUILD_BYPRODUCTS "${GFLAGS_STATIC_LIB}" + CMAKE_ARGS ${GFLAGS_CMAKE_ARGS}) + + add_dependencies(toolchain gflags_ep) + + ADD_THIRDPARTY_LIB(gflags + STATIC_LIB ${GFLAGS_STATIC_LIB}) + set(GFLAGS_LIBRARY gflags_static) + set_target_properties(${GFLAGS_LIBRARY} + PROPERTIES + INTERFACE_COMPILE_DEFINITIONS "GFLAGS_IS_A_DLL=0" + INTERFACE_INCLUDE_DIRECTORIES "${GFLAGS_INCLUDE_DIR}") + if(MSVC) + set_target_properties(${GFLAGS_LIBRARY} + PROPERTIES + INTERFACE_LINK_LIBRARIES "shlwapi.lib") + endif() + set(GFLAGS_LIBRARIES ${GFLAGS_LIBRARY}) +endmacro() + +if (ARROW_NEED_GFLAGS) + get_dependency_source(gflags) + if (gflags_SOURCE STREQUAL "AUTO") + find_package(gflags QUIET) + if (NOT gflags_FOUND) + find_package(gflagsAlt) + endif() + if (NOT gflags_FOUND AND NOT gflagsAlt_FOUND) + build_gflags() + endif() + elseif(gflags_SOURCE STREQUAL "BUNDLED") + build_gflags() + elseif(gflags_SOURCE STREQUAL "SYSTEM") + # gflagsConfig.cmake is not installed on Ubuntu/Debian + # TODO: Make a bug report upstream + find_package(gflags) + if (NOT gflags_FOUND) + find_package(gflagsAlt REQUIRED) + endif() + endif() + # TODO: Don't use global includes but rather target_include_directories + include_directories(SYSTEM ${GFLAGS_INCLUDE_DIR}) + + if (NOT TARGET ${GFLAGS_LIBRARIES}) + if (TARGET gflags-shared) + set(GFLAGS_LIBRARIES gflags-shared) + elseif (TARGET gflags_shared) + set(GFLAGS_LIBRARIES gflags_shared) + endif() + endif() +endif() - ## Flatbuffers - if("${FLATBUFFERS_HOME}" STREQUAL "") - set(FLATBUFFERS_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/flatbuffers_ep-prefix/src/flatbuffers_ep-install") - if (MSVC) - set(FLATBUFFERS_CMAKE_CXX_FLAGS /EHsc) +# ---------------------------------------------------------------------- +# Thrift + +macro(build_thrift) + message("Building Apache Thrift from source") + set(THRIFT_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/thrift_ep/src/thrift_ep-install") + set(THRIFT_INCLUDE_DIR "${THRIFT_PREFIX}/include") + set(THRIFT_COMPILER "${THRIFT_PREFIX}/bin/thrift") + set(THRIFT_CMAKE_ARGS ${EP_COMMON_CMAKE_ARGS} + "-DCMAKE_INSTALL_PREFIX=${THRIFT_PREFIX}" + "-DCMAKE_INSTALL_RPATH=${THRIFT_PREFIX}/lib" + -DBUILD_SHARED_LIBS=OFF + -DBUILD_TESTING=OFF + -DBUILD_EXAMPLES=OFF + -DBUILD_TUTORIALS=OFF + -DWITH_QT4=OFF + -DWITH_C_GLIB=OFF + -DWITH_JAVA=OFF + -DWITH_PYTHON=OFF + -DWITH_HASKELL=OFF + -DWITH_CPP=ON + -DWITH_STATIC_LIB=ON + -DWITH_LIBEVENT=OFF) + + # Thrift also uses boost. Forward important boost settings if there were ones passed. + if (DEFINED BOOST_ROOT) + set(THRIFT_CMAKE_ARGS ${THRIFT_CMAKE_ARGS} "-DBOOST_ROOT=${BOOST_ROOT}") + endif() + if (DEFINED Boost_NAMESPACE) + set(THRIFT_CMAKE_ARGS ${THRIFT_CMAKE_ARGS} "-DBoost_NAMESPACE=${Boost_NAMESPACE}") + endif() + + set(THRIFT_STATIC_LIB_NAME "${CMAKE_STATIC_LIBRARY_PREFIX}thrift") + if (MSVC) + if (ARROW_USE_STATIC_CRT) + set(THRIFT_STATIC_LIB_NAME "${THRIFT_STATIC_LIB_NAME}mt") + set(THRIFT_CMAKE_ARGS ${THRIFT_CMAKE_ARGS} "-DWITH_MT=ON") else() - set(FLATBUFFERS_CMAKE_CXX_FLAGS -fPIC) + set(THRIFT_STATIC_LIB_NAME "${THRIFT_STATIC_LIB_NAME}md") + set(THRIFT_CMAKE_ARGS ${THRIFT_CMAKE_ARGS} "-DWITH_MT=OFF") endif() - # We always need to do release builds, otherwise flatc will not be installed. - ExternalProject_Add(flatbuffers_ep - URL ${FLATBUFFERS_SOURCE_URL} - CMAKE_ARGS - ${EP_COMMON_CMAKE_ARGS} - -DCMAKE_BUILD_TYPE=RELEASE - "-DCMAKE_CXX_FLAGS=${FLATBUFFERS_CMAKE_CXX_FLAGS}" - "-DCMAKE_INSTALL_PREFIX:PATH=${FLATBUFFERS_PREFIX}" - "-DFLATBUFFERS_BUILD_TESTS=OFF" - ${EP_LOG_OPTIONS}) + endif() + if (${UPPERCASE_BUILD_TYPE} STREQUAL "DEBUG") + set(THRIFT_STATIC_LIB_NAME "${THRIFT_STATIC_LIB_NAME}d") + endif() + set(THRIFT_STATIC_LIB "${THRIFT_PREFIX}/lib/${THRIFT_STATIC_LIB_NAME}${CMAKE_STATIC_LIBRARY_SUFFIX}") - set(FLATBUFFERS_INCLUDE_DIR "${FLATBUFFERS_PREFIX}/include") - set(FLATBUFFERS_COMPILER "${FLATBUFFERS_PREFIX}/bin/flatc") - set(FLATBUFFERS_VENDORED 1) - add_dependencies(toolchain flatbuffers_ep) + if (ZLIB_SHARED_LIB) + set(THRIFT_CMAKE_ARGS "-DZLIB_LIBRARY=${ZLIB_SHARED_LIB}" + ${THRIFT_CMAKE_ARGS}) else() - find_package(Flatbuffers REQUIRED) - set(FLATBUFFERS_VENDORED 0) + set(THRIFT_CMAKE_ARGS "-DZLIB_LIBRARY=${ZLIB_STATIC_LIB}" + ${THRIFT_CMAKE_ARGS}) endif() + set(THRIFT_DEPENDENCIES ${THRIFT_DEPENDENCIES} ${ZLIB_LIBRARY}) - message(STATUS "Flatbuffers include dir: ${FLATBUFFERS_INCLUDE_DIR}") - message(STATUS "Flatbuffers compiler: ${FLATBUFFERS_COMPILER}") - include_directories(SYSTEM ${FLATBUFFERS_INCLUDE_DIR}) + if (MSVC) + set(WINFLEXBISON_VERSION 2.4.9) + set(WINFLEXBISON_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/winflexbison_ep/src/winflexbison_ep-install") + ExternalProject_Add(winflexbison_ep + URL https://github.com/lexxmark/winflexbison/releases/download/v.${WINFLEXBISON_VERSION}/win_flex_bison-${WINFLEXBISON_VERSION}.zip + URL_HASH MD5=a2e979ea9928fbf8567e995e9c0df765 + SOURCE_DIR ${WINFLEXBISON_PREFIX} + CONFIGURE_COMMAND "" + BUILD_COMMAND "" + INSTALL_COMMAND "" + ${EP_LOG_OPTIONS}) + set(THRIFT_DEPENDENCIES ${THRIFT_DEPENDENCIES} winflexbison_ep) + + set(THRIFT_CMAKE_ARGS "-DFLEX_EXECUTABLE=${WINFLEXBISON_PREFIX}/win_flex.exe" + "-DBISON_EXECUTABLE=${WINFLEXBISON_PREFIX}/win_bison.exe" + "-DZLIB_INCLUDE_DIR=${ZLIB_INCLUDE_DIR}" + "-DWITH_SHARED_LIB=OFF" + "-DWITH_PLUGIN=OFF" + ${THRIFT_CMAKE_ARGS}) + elseif (APPLE) + # Some other process always resets BISON_EXECUTABLE to the system default, + # thus we use our own variable here. + if (NOT DEFINED THRIFT_BISON_EXECUTABLE) + find_package(BISON 2.5.1) + + # In the case where we cannot find a system-wide installation, look for + # homebrew and ask for its bison installation. + if (NOT BISON_FOUND) + find_program(BREW_BIN brew) + if (BREW_BIN) + execute_process( + COMMAND ${BREW_BIN} --prefix bison + OUTPUT_VARIABLE BISON_PREFIX + OUTPUT_STRIP_TRAILING_WHITESPACE + ) + set(BISON_EXECUTABLE "${BISON_PREFIX}/bin/bison") + find_package(BISON 2.5.1) + set(THRIFT_BISON_EXECUTABLE "${BISON_EXECUTABLE}") + endif() + else() + set(THRIFT_BISON_EXECUTABLE "${BISON_EXECUTABLE}") + endif() + endif() + set(THRIFT_CMAKE_ARGS "-DBISON_EXECUTABLE=${THRIFT_BISON_EXECUTABLE}" + ${THRIFT_CMAKE_ARGS}) + endif() + + ExternalProject_Add(thrift_ep + URL ${THRIFT_SOURCE_URL} + BUILD_BYPRODUCTS "${THRIFT_STATIC_LIB}" "${THRIFT_COMPILER}" + CMAKE_ARGS ${THRIFT_CMAKE_ARGS} + DEPENDS ${THRIFT_DEPENDENCIES} + ${EP_LOG_OPTIONS}) + + add_library(Thrift::thrift STATIC IMPORTED) + # The include directory must exist before it is referenced by a target. + file(MAKE_DIRECTORY "${THRIFT_INCLUDE_DIR}") + set_target_properties(Thrift::thrift PROPERTIES + IMPORTED_LOCATION "${THRIFT_STATIC_LIB}" + INTERFACE_INCLUDE_DIRECTORIES "${THRIFT_INCLUDE_DIR}" + ) + add_dependencies(toolchain thrift_ep) + add_dependencies(Thrift::thrift thrift_ep) +endmacro() + +if (ARROW_WITH_THRIFT) + resolve_dependency(Thrift) + # TODO: Don't use global includes but rather target_include_directories + include_directories(SYSTEM ${THRIFT_INCLUDE_DIR}) + + if (THRIFT_VERSION VERSION_LESS "0.11.0") + add_definitions(-DPARQUET_THRIFT_USE_BOOST) + endif() +endif() + +# ---------------------------------------------------------------------- +# Protocol Buffers (required for ORC and Flight and Gandiva libraries) + +macro(build_protobuf) + message("Building Protocol Buffers from source") + set(PROTOBUF_PREFIX "${THIRDPARTY_DIR}/protobuf_ep-install") + set(PROTOBUF_INCLUDE_DIR "${PROTOBUF_PREFIX}/include") + set(PROTOBUF_STATIC_LIB "${PROTOBUF_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}protobuf${CMAKE_STATIC_LIBRARY_SUFFIX}") + set(PROTOC_STATIC_LIB "${PROTOBUF_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}protoc${CMAKE_STATIC_LIBRARY_SUFFIX}") + set(PROTOBUF_COMPILER "${PROTOBUF_PREFIX}/bin/protoc") + set(PROTOBUF_CONFIGURE_ARGS "AR=${CMAKE_AR}" + "RANLIB=${CMAKE_RANLIB}" + "CC=${CMAKE_C_COMPILER}" + "CXX=${CMAKE_CXX_COMPILER}" + "--disable-shared" + "--prefix=${PROTOBUF_PREFIX}" + "CFLAGS=${EP_C_FLAGS}" + "CXXFLAGS=${EP_CXX_FLAGS}") + + ExternalProject_Add(protobuf_ep + CONFIGURE_COMMAND "./configure" ${PROTOBUF_CONFIGURE_ARGS} + BUILD_IN_SOURCE 1 + URL ${PROTOBUF_SOURCE_URL} + BUILD_BYPRODUCTS "${PROTOBUF_STATIC_LIB}" "${PROTOBUF_COMPILER}" + ${EP_LOG_OPTIONS}) + + file(MAKE_DIRECTORY "${PROTOBUF_INCLUDE_DIR}") + + add_library(protobuf::libprotobuf STATIC IMPORTED) + set_target_properties(protobuf::libprotobuf PROPERTIES + IMPORTED_LOCATION "${PROTOBUF_STATIC_LIB}" + INTERFACE_INCLUDE_DIRECTORIES "${PROTOBUF_INCLUDE_DIR}" + ) + add_library(protobuf::libprotoc STATIC IMPORTED) + set_target_properties(protobuf::libprotoc PROPERTIES + IMPORTED_LOCATION "${PROTOC_STATIC_LIB}" + INTERFACE_INCLUDE_DIRECTORIES "${PROTOBUF_INCLUDE_DIR}" + ) + add_executable(protobuf::protoc IMPORTED) + set_target_properties(protobuf::protoc PROPERTIES + IMPORTED_LOCATION "${PROTOBUF_COMPILER}" + ) + + add_dependencies(toolchain protobuf_ep) + add_dependencies(protobuf::libprotobuf protobuf_ep) +endmacro() + +if (ARROW_WITH_PROTOBUF) + resolve_dependency(Protobuf) + # TODO: Don't use global includes but rather target_include_directories + include_directories(SYSTEM ${PROTOBUF_INCLUDE_DIR}) + + # Old CMake versions don't define the targets + if (NOT TARGET protobuf::libprotobuf) + add_library(protobuf::libprotobuf UNKNOWN IMPORTED) + set_target_properties(protobuf::libprotobuf PROPERTIES + IMPORTED_LOCATION "${PROTOBUF_LIBRARY}" + INTERFACE_INCLUDE_DIRECTORIES "${PROTOBUF_INCLUDE_DIR}" + ) + endif() + if (NOT TARGET protobuf::libprotoc) + if (NOT Protobuf_PROTOC_LIBRARY) + message(FATAL_ERROR "libprotoc was set to ${Protobuf_PROTOC_LIBRARY}") + endif() + add_library(protobuf::libprotoc UNKNOWN IMPORTED) + set_target_properties(protobuf::libprotoc PROPERTIES + IMPORTED_LOCATION "${Protobuf_PROTOC_LIBRARY}" + INTERFACE_INCLUDE_DIRECTORIES "${PROTOBUF_INCLUDE_DIR}" + ) + endif() + if (NOT TARGET protobuf::protoc) + add_executable(protobuf::protoc IMPORTED) + set_target_properties(protobuf::protoc PROPERTIES + IMPORTED_LOCATION "${PROTOBUF_PROTOC_EXECUTABLE}" + ) + endif() endif() -#---------------------------------------------------------------------- if (MSVC) # jemalloc is not supported on Windows set(ARROW_JEMALLOC off) endif() + if (ARROW_JEMALLOC) + message(STATUS "Building (vendored) jemalloc from source") # We only use a vendored jemalloc as we want to control its version. # Also our build of jemalloc is specially prefixed so that it will not # conflict with the default allocator as well as other jemalloc @@ -888,11 +1014,7 @@ if (ARROW_JEMALLOC) set(ARROW_JEMALLOC_USE_SHARED OFF) set(JEMALLOC_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/jemalloc_ep-prefix/src/jemalloc_ep/dist/") - set(JEMALLOC_HOME "${JEMALLOC_PREFIX}") - set(JEMALLOC_INCLUDE_DIR "${JEMALLOC_PREFIX}/include") - set(JEMALLOC_SHARED_LIB "${JEMALLOC_PREFIX}/lib/libjemalloc${CMAKE_SHARED_LIBRARY_SUFFIX}") set(JEMALLOC_STATIC_LIB "${JEMALLOC_PREFIX}/lib/libjemalloc_pic${CMAKE_STATIC_LIBRARY_SUFFIX}") - set(JEMALLOC_VENDORED 1) # We need to disable TLS or otherwise C++ exceptions won't work anymore. ExternalProject_Add(jemalloc_ep URL ${CMAKE_CURRENT_SOURCE_DIR}/thirdparty/jemalloc/${JEMALLOC_VERSION}.tar.gz @@ -901,604 +1023,955 @@ if (ARROW_JEMALLOC) ${EP_LOG_OPTIONS} BUILD_IN_SOURCE 1 BUILD_COMMAND ${MAKE} ${MAKE_BUILD_ARGS} - BUILD_BYPRODUCTS "${JEMALLOC_STATIC_LIB}" "${JEMALLOC_SHARED_LIB}" + BUILD_BYPRODUCTS "${JEMALLOC_STATIC_LIB}" INSTALL_COMMAND ${MAKE} install) # Don't use the include directory directly so that we can point to a path # that is unique to our codebase. include_directories(SYSTEM "${CMAKE_CURRENT_BINARY_DIR}/jemalloc_ep-prefix/src/") - - ADD_THIRDPARTY_LIB(jemalloc - STATIC_LIB ${JEMALLOC_STATIC_LIB} - SHARED_LIB ${JEMALLOC_SHARED_LIB} - DEPS Threads::Threads) - add_dependencies(jemalloc_static jemalloc_ep) - + # The include directory must exist before it is referenced by a target. + file(MAKE_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/jemalloc_ep-prefix/src/") + add_library(jemalloc::jemalloc STATIC IMPORTED) + set_target_properties(jemalloc::jemalloc PROPERTIES + INTERFACE_LINK_LIBRARIES Threads::Threads + IMPORTED_LOCATION "${JEMALLOC_STATIC_LIB}" + INTERFACE_INCLUDE_DIRECTORIES "${CMAKE_CURRENT_BINARY_DIR}/jemalloc_ep-prefix/src" + ) + add_dependencies(jemalloc::jemalloc jemalloc_ep) add_dependencies(toolchain jemalloc_ep) endif() -## Google PerfTools -## -## Disabled with TSAN/ASAN as well as with gold+dynamic linking (see comment -## near definition of ARROW_USING_GOLD). -# find_package(GPerf REQUIRED) -# if (NOT "${ARROW_USE_ASAN}" AND -# NOT "${ARROW_USE_TSAN}" AND -# NOT ("${ARROW_USING_GOLD}" AND "${ARROW_LINK}" STREQUAL "d")) -# ADD_THIRDPARTY_LIB(tcmalloc -# STATIC_LIB "${TCMALLOC_STATIC_LIB}" -# SHARED_LIB "${TCMALLOC_SHARED_LIB}") -# ADD_THIRDPARTY_LIB(profiler -# STATIC_LIB "${PROFILER_STATIC_LIB}" -# SHARED_LIB "${PROFILER_SHARED_LIB}") -# list(APPEND ARROW_BASE_LIBS tcmalloc profiler) -# add_definitions("-DTCMALLOC_ENABLED") -# set(ARROW_TCMALLOC_AVAILABLE 1) -# endif() +# ---------------------------------------------------------------------- +# Google gtest -######################################################################## -# HDFS thirdparty setup +macro(build_gtest) + message(STATUS "Building gtest from source") + set(GTEST_VENDORED TRUE) + set(GTEST_CMAKE_CXX_FLAGS ${EP_CXX_FLAGS}) -if (DEFINED ENV{HADOOP_HOME}) - set(HADOOP_HOME $ENV{HADOOP_HOME}) - if (NOT EXISTS "${HADOOP_HOME}/include/hdfs.h") - message(STATUS "Did not find hdfs.h in expected location, using vendored one") - set(HADOOP_HOME "${THIRDPARTY_DIR}/hadoop") + if(CMAKE_BUILD_TYPE MATCHES DEBUG) + set(CMAKE_GTEST_DEBUG_EXTENSION "d") + else() + set(CMAKE_GTEST_DEBUG_EXTENSION "") endif() -else() - set(HADOOP_HOME "${THIRDPARTY_DIR}/hadoop") -endif() -set(HDFS_H_PATH "${HADOOP_HOME}/include/hdfs.h") -if (NOT EXISTS ${HDFS_H_PATH}) - message(FATAL_ERROR "Did not find hdfs.h at ${HDFS_H_PATH}") -endif() -message(STATUS "Found hdfs.h at: " ${HDFS_H_PATH}) + if(APPLE) + set(GTEST_CMAKE_CXX_FLAGS ${GTEST_CMAKE_CXX_FLAGS} + -DGTEST_USE_OWN_TR1_TUPLE=1 + -Wno-unused-value + -Wno-ignored-attributes) + endif() -include_directories(SYSTEM "${HADOOP_HOME}/include") + set(GTEST_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/googletest_ep-prefix/src/googletest_ep") + set(GTEST_INCLUDE_DIR "${GTEST_PREFIX}/include") + set(GTEST_STATIC_LIB + "${GTEST_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}gtest${CMAKE_GTEST_DEBUG_EXTENSION}${CMAKE_STATIC_LIBRARY_SUFFIX}") + set(GTEST_MAIN_STATIC_LIB + "${GTEST_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}gtest_main${CMAKE_GTEST_DEBUG_EXTENSION}${CMAKE_STATIC_LIBRARY_SUFFIX}") + set(GTEST_CMAKE_ARGS ${EP_COMMON_CMAKE_ARGS} + "-DCMAKE_INSTALL_PREFIX=${GTEST_PREFIX}" + "-DCMAKE_INSTALL_LIBDIR=lib" + -DCMAKE_CXX_FLAGS=${GTEST_CMAKE_CXX_FLAGS}) + set(GMOCK_INCLUDE_DIR "${GTEST_PREFIX}/include") + set(GMOCK_STATIC_LIB + "${GTEST_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}gmock${CMAKE_GTEST_DEBUG_EXTENSION}${CMAKE_STATIC_LIBRARY_SUFFIX}") + + if (MSVC AND NOT ARROW_USE_STATIC_CRT) + set(GTEST_CMAKE_ARGS ${GTEST_CMAKE_ARGS} -Dgtest_force_shared_crt=ON) + endif() -if (ARROW_WITH_ZLIB) -# ---------------------------------------------------------------------- -# ZLIB + ExternalProject_Add(googletest_ep + URL ${GTEST_SOURCE_URL} + BUILD_BYPRODUCTS ${GTEST_STATIC_LIB} ${GTEST_MAIN_STATIC_LIB} ${GMOCK_STATIC_LIB} + CMAKE_ARGS ${GTEST_CMAKE_ARGS} + ${EP_LOG_OPTIONS}) - if("${ZLIB_HOME}" STREQUAL "") - find_package(ZLIB) - else() - find_package(ZLIB REQUIRED) - endif() - if(ZLIB_FOUND) - ADD_THIRDPARTY_LIB(zlib SHARED_LIB ${ZLIB_SHARED_LIB}) - set(ZLIB_LIBRARY zlib_shared) - else() - set(ZLIB_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/zlib_ep/src/zlib_ep-install") - set(ZLIB_HOME "${ZLIB_PREFIX}") - set(ZLIB_INCLUDE_DIR "${ZLIB_PREFIX}/include") - if (MSVC) - if (${UPPERCASE_BUILD_TYPE} STREQUAL "DEBUG") - set(ZLIB_STATIC_LIB_NAME zlibstaticd.lib) - else() - set(ZLIB_STATIC_LIB_NAME zlibstatic.lib) - endif() - else() - set(ZLIB_STATIC_LIB_NAME libz.a) - endif() - set(ZLIB_STATIC_LIB "${ZLIB_PREFIX}/lib/${ZLIB_STATIC_LIB_NAME}") - set(ZLIB_CMAKE_ARGS ${EP_COMMON_CMAKE_ARGS} - "-DCMAKE_INSTALL_PREFIX=${ZLIB_PREFIX}" - -DBUILD_SHARED_LIBS=OFF) - ADD_THIRDPARTY_LIB(zlib - STATIC_LIB ${ZLIB_STATIC_LIB}) - set(ZLIB_LIBRARY zlib_static) - - ExternalProject_Add(zlib_ep - URL ${ZLIB_SOURCE_URL} - ${EP_LOG_OPTIONS} - BUILD_BYPRODUCTS "${ZLIB_STATIC_LIB}" - CMAKE_ARGS ${ZLIB_CMAKE_ARGS}) - add_dependencies(${ZLIB_LIBRARY} zlib_ep) + # The include directory must exist before it is referenced by a target. + file(MAKE_DIRECTORY "${GTEST_INCLUDE_DIR}") + + add_library(GTest::GTest STATIC IMPORTED) + set_target_properties(GTest::GTest PROPERTIES + IMPORTED_LOCATION "${GTEST_STATIC_LIB}" + INTERFACE_INCLUDE_DIRECTORIES "${GTEST_INCLUDE_DIR}" + ) + + add_library(GTest::Main STATIC IMPORTED) + set_target_properties(GTest::Main PROPERTIES + IMPORTED_LOCATION "${GTEST_MAIN_STATIC_LIB}" + INTERFACE_INCLUDE_DIRECTORIES "${GTEST_INCLUDE_DIR}" + ) + + add_library(GTest::GMock STATIC IMPORTED) + set_target_properties(GTest::GMock PROPERTIES + IMPORTED_LOCATION "${GMOCK_STATIC_LIB}" + INTERFACE_INCLUDE_DIRECTORIES "${GTEST_INCLUDE_DIR}" + ) + add_dependencies(toolchain-tests googletest_ep) + add_dependencies(GTest::GTest googletest_ep) + add_dependencies(GTest::Main googletest_ep) + add_dependencies(GTest::GMock googletest_ep) +endmacro() - add_dependencies(toolchain zlib_ep) +if(ARROW_BUILD_TESTS OR ARROW_BUILD_BENCHMARKS) + resolve_dependency(GTest) + + if (NOT GTEST_VENDORED) +# TODO(wesm): This logic does not work correctly with the MSVC static libraries +# built for the shared crt + +# set(CMAKE_REQUIRED_LIBRARIES GTest::GTest GTest::Main GTest::GMock) +# CHECK_CXX_SOURCE_COMPILES(" +# #include +# #include + +# class A { +# public: +# int run() const { return 1; } +# }; + +# class B : public A { +# public: +# MOCK_CONST_METHOD0(run, int()); +# }; + +# TEST(Base, Test) { +# B b; +# }" GTEST_COMPILES_WITHOUT_MACRO) +# if (NOT GTEST_COMPILES_WITHOUT_MACRO) +# message(STATUS "Setting GTEST_LINKED_AS_SHARED_LIBRARY=1 on GTest::GTest") +# add_compile_definitions("GTEST_LINKED_AS_SHARED_LIBRARY=1") +# endif() +# set(CMAKE_REQUIRED_LIBRARIES) endif() - include_directories(SYSTEM ${ZLIB_INCLUDE_DIR}) -endif() - -if (ARROW_WITH_SNAPPY) -# ---------------------------------------------------------------------- -# Snappy + get_property(GTEST_LOCATION TARGET GTest::GTest + PROPERTY LOCATION) + get_property(GMOCK_LOCATION TARGET GTest::GMock + PROPERTY LOCATION) - if("${SNAPPY_HOME}" STREQUAL "") - set(SNAPPY_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/snappy_ep/src/snappy_ep-install") - set(SNAPPY_HOME "${SNAPPY_PREFIX}") - set(SNAPPY_INCLUDE_DIR "${SNAPPY_PREFIX}/include") - if (MSVC) - set(SNAPPY_STATIC_LIB_NAME snappy_static) - else() - set(SNAPPY_STATIC_LIB_NAME snappy) - endif() - set(SNAPPY_STATIC_LIB "${SNAPPY_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}${SNAPPY_STATIC_LIB_NAME}${CMAKE_STATIC_LIBRARY_SUFFIX}") + message(STATUS "Using gtest library at ${GTEST_LOCATION}") + message(STATUS "Using gmock library at ${GMOCK_LOCATION}") - if (${UPPERCASE_BUILD_TYPE} EQUAL "RELEASE") - if (APPLE) - set(SNAPPY_CXXFLAGS "CXXFLAGS='-DNDEBUG -O1'") - else() - set(SNAPPY_CXXFLAGS "CXXFLAGS='-DNDEBUG -O2'") - endif() - endif() + # TODO: Don't use global includes but rather target_include_directories + get_target_property(GTEST_INCLUDE_DIR GTest::GTest INTERFACE_INCLUDE_DIRECTORIES) + include_directories(SYSTEM ${GTEST_INCLUDE_DIR}) +endif() - if (WIN32) - set(SNAPPY_CMAKE_ARGS ${EP_COMMON_CMAKE_ARGS} - -DCMAKE_AR=${CMAKE_AR} - -DCMAKE_RANLIB=${CMAKE_RANLIB} - "-DCMAKE_INSTALL_PREFIX=${SNAPPY_PREFIX}") - set(SNAPPY_UPDATE_COMMAND ${CMAKE_COMMAND} -E copy - ${CMAKE_SOURCE_DIR}/cmake_modules/SnappyCMakeLists.txt - ./CMakeLists.txt && - ${CMAKE_COMMAND} -E copy - ${CMAKE_SOURCE_DIR}/cmake_modules/SnappyConfig.h - ./config.h) - ExternalProject_Add(snappy_ep - UPDATE_COMMAND ${SNAPPY_UPDATE_COMMAND} - ${EP_LOG_OPTIONS} - BUILD_IN_SOURCE 1 - BUILD_COMMAND ${MAKE} - INSTALL_DIR ${SNAPPY_PREFIX} - URL ${SNAPPY_SOURCE_URL} - CMAKE_ARGS ${SNAPPY_CMAKE_ARGS} - BUILD_BYPRODUCTS "${SNAPPY_STATIC_LIB}") - else() - ExternalProject_Add(snappy_ep - CONFIGURE_COMMAND ./configure --with-pic "AR=${CMAKE_AR}" "RANLIB=${CMAKE_RANLIB}" "--prefix=${SNAPPY_PREFIX}" ${SNAPPY_CXXFLAGS} - ${EP_LOG_OPTIONS} - BUILD_IN_SOURCE 1 - BUILD_COMMAND ${MAKE} - INSTALL_DIR ${SNAPPY_PREFIX} - URL ${SNAPPY_SOURCE_URL} - BUILD_BYPRODUCTS "${SNAPPY_STATIC_LIB}") - endif() +macro(build_benchmark) + message(STATUS "Building benchmark from source") + if(CMAKE_VERSION VERSION_LESS 3.6) + message(FATAL_ERROR "Building gbenchmark from source requires at least CMake 3.6") + endif() - set(SNAPPY_VENDORED 1) - add_dependencies(toolchain snappy_ep) - else() - find_package(Snappy REQUIRED) - set(SNAPPY_VENDORED 0) + if(NOT MSVC) + set(GBENCHMARK_CMAKE_CXX_FLAGS "${EP_CXX_FLAGS} -std=c++11") endif() - include_directories(SYSTEM ${SNAPPY_INCLUDE_DIR}) - ADD_THIRDPARTY_LIB(snappy - STATIC_LIB ${SNAPPY_STATIC_LIB}) + if(APPLE) + set(GBENCHMARK_CMAKE_CXX_FLAGS "${GBENCHMARK_CMAKE_CXX_FLAGS} -stdlib=libc++") + endif() - if (SNAPPY_VENDORED) - add_dependencies(snappy_static snappy_ep) + set(GBENCHMARK_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/gbenchmark_ep/src/gbenchmark_ep-install") + set(GBENCHMARK_INCLUDE_DIR "${GBENCHMARK_PREFIX}/include") + set(GBENCHMARK_STATIC_LIB "${GBENCHMARK_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}benchmark${CMAKE_STATIC_LIBRARY_SUFFIX}") + set(GBENCHMARK_MAIN_STATIC_LIB "${GBENCHMARK_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}benchmark_main${CMAKE_STATIC_LIBRARY_SUFFIX}") + set(GBENCHMARK_CMAKE_ARGS ${EP_COMMON_CMAKE_ARGS} + "-DCMAKE_INSTALL_PREFIX=${GBENCHMARK_PREFIX}" + -DBENCHMARK_ENABLE_TESTING=OFF + -DCMAKE_CXX_FLAGS=${GBENCHMARK_CMAKE_CXX_FLAGS}) + if (APPLE) + set(GBENCHMARK_CMAKE_ARGS ${GBENCHMARK_CMAKE_ARGS} "-DBENCHMARK_USE_LIBCXX=ON") endif() -endif() -if (ARROW_WITH_BROTLI) -# ---------------------------------------------------------------------- -# Brotli + ExternalProject_Add(gbenchmark_ep + URL ${GBENCHMARK_SOURCE_URL} + BUILD_BYPRODUCTS "${GBENCHMARK_STATIC_LIB}" "${GBENCHMARK_MAIN_STATIC_LIB}" + CMAKE_ARGS ${GBENCHMARK_CMAKE_ARGS} + ${EP_LOG_OPTIONS}) - if("${BROTLI_HOME}" STREQUAL "") - set(BROTLI_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/brotli_ep/src/brotli_ep-install") - set(BROTLI_HOME "${BROTLI_PREFIX}") - set(BROTLI_INCLUDE_DIR "${BROTLI_PREFIX}/include") - if (MSVC) - set(BROTLI_LIB_DIR bin) - else() - set(BROTLI_LIB_DIR lib) - endif() - set(BROTLI_STATIC_LIBRARY_ENC "${BROTLI_PREFIX}/${BROTLI_LIB_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}brotlienc${CMAKE_STATIC_LIBRARY_SUFFIX}") - set(BROTLI_STATIC_LIBRARY_DEC "${BROTLI_PREFIX}/${BROTLI_LIB_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}brotlidec${CMAKE_STATIC_LIBRARY_SUFFIX}") - set(BROTLI_STATIC_LIBRARY_COMMON "${BROTLI_PREFIX}/${BROTLI_LIB_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}brotlicommon${CMAKE_STATIC_LIBRARY_SUFFIX}") - set(BROTLI_CMAKE_ARGS ${EP_COMMON_CMAKE_ARGS} - "-DCMAKE_INSTALL_PREFIX=${BROTLI_PREFIX}" - -DCMAKE_INSTALL_LIBDIR=lib - -DBUILD_SHARED_LIBS=OFF) - - ExternalProject_Add(brotli_ep - URL ${BROTLI_SOURCE_URL} - BUILD_BYPRODUCTS "${BROTLI_STATIC_LIBRARY_ENC}" "${BROTLI_STATIC_LIBRARY_DEC}" "${BROTLI_STATIC_LIBRARY_COMMON}" - ${BROTLI_BUILD_BYPRODUCTS} - ${EP_LOG_OPTIONS} - CMAKE_ARGS ${BROTLI_CMAKE_ARGS} - STEP_TARGETS headers_copy) - if (MSVC) - ExternalProject_Get_Property(brotli_ep SOURCE_DIR) + # The include directory must exist before it is referenced by a target. + file(MAKE_DIRECTORY "${GBENCHMARK_INCLUDE_DIR}") - ExternalProject_Add_Step(brotli_ep headers_copy - COMMAND xcopy /E /I include ..\\..\\..\\brotli_ep\\src\\brotli_ep-install\\include /Y - DEPENDEES build - WORKING_DIRECTORY ${SOURCE_DIR}) - endif() - set(BROTLI_VENDORED 1) + add_library(benchmark::benchmark STATIC IMPORTED) + set_target_properties(benchmark::benchmark PROPERTIES + IMPORTED_LOCATION "${GBENCHMARK_STATIC_LIB}" + INTERFACE_INCLUDE_DIRECTORIES "${GBENCHMARK_INCLUDE_DIR}" + ) - add_dependencies(toolchain brotli_ep) - else() - find_package(Brotli REQUIRED) - set(BROTLI_VENDORED 0) + add_library(benchmark::benchmark_main STATIC IMPORTED) + set_target_properties(benchmark::benchmark_main PROPERTIES + IMPORTED_LOCATION "${GBENCHMARK_MAIN_STATIC_LIB}" + INTERFACE_INCLUDE_DIRECTORIES "${GBENCHMARK_INCLUDE_DIR}" + ) + + add_dependencies(toolchain-benchmarks gbenchmark_ep) + add_dependencies(benchmark::benchmark gbenchmark_ep) + add_dependencies(benchmark::benchmark_main gbenchmark_ep) +endmacro() + +if(ARROW_BUILD_BENCHMARKS) + resolve_dependency(benchmark) + # TODO: Don't use global includes but rather target_include_directories + get_target_property(BENCHMARK_INCLUDE_DIR benchmark::benchmark INTERFACE_INCLUDE_DIRECTORIES) + include_directories(SYSTEM ${BENCHMARK_INCLUDE_DIR}) +endif() + +macro(build_rapidjson) + message(STATUS "Building rapidjson from source") + set(RAPIDJSON_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/rapidjson_ep/src/rapidjson_ep-install") + set(RAPIDJSON_CMAKE_ARGS + -DRAPIDJSON_BUILD_DOC=OFF + -DRAPIDJSON_BUILD_EXAMPLES=OFF + -DRAPIDJSON_BUILD_TESTS=OFF + "-DCMAKE_INSTALL_PREFIX=${RAPIDJSON_PREFIX}") + + ExternalProject_Add(rapidjson_ep + ${EP_LOG_OPTIONS} + PREFIX "${CMAKE_BINARY_DIR}" + URL ${RAPIDJSON_SOURCE_URL} + CMAKE_ARGS ${RAPIDJSON_CMAKE_ARGS}) + + set(RAPIDJSON_INCLUDE_DIR "${RAPIDJSON_PREFIX}/include") + + add_dependencies(toolchain rapidjson_ep) + add_dependencies(rapidjson rapidjson_ep) +endmacro() + +# TODO: Check for 1.1.0+ +if(ARROW_WITH_RAPIDJSON) + get_dependency_source(RapidJSON) + if (RapidJSON_SOURCE STREQUAL "AUTO") + # Fedora packages place the package information at the wrong location. + # https://bugzilla.redhat.com/show_bug.cgi?id=1680400 + find_package(RapidJSON QUIET HINTS "${CMAKE_ROOT}") + if (NOT RapidJSON_FOUND) + # Ubuntu / Debian don't package the CMake config + find_package(RapidJSONAlt) + endif() + if (NOT RapidJSON_FOUND AND NOT RapidJSONAlt_FOUND) + build_rapidjson() + endif() + elseif(RapidJSON_SOURCE STREQUAL "BUNDLED") + build_rapidjson() + elseif(RapidJSON_SOURCE STREQUAL "SYSTEM") + # Fedora packages place the package information at the wrong location. + # https://bugzilla.redhat.com/show_bug.cgi?id=1680400 + find_package(RapidJSON HINTS "${CMAKE_ROOT}") + if (NOT RapidJSON_FOUND) + # Ubuntu / Debian don't package the CMake config + find_package(RapidJSONAlt REQUIRED) + endif() endif() - include_directories(SYSTEM ${BROTLI_INCLUDE_DIR}) - ADD_THIRDPARTY_LIB(brotli_enc - STATIC_LIB ${BROTLI_STATIC_LIBRARY_ENC}) - ADD_THIRDPARTY_LIB(brotli_dec - STATIC_LIB ${BROTLI_STATIC_LIBRARY_DEC}) - ADD_THIRDPARTY_LIB(brotli_common - STATIC_LIB ${BROTLI_STATIC_LIBRARY_COMMON}) - - if (BROTLI_VENDORED) - add_dependencies(brotli_enc_static brotli_ep) - add_dependencies(brotli_dec_static brotli_ep) - add_dependencies(brotli_common_static brotli_ep) + if (RapidJSON_INCLUDE_DIR) + set(RAPIDJSON_INCLUDE_DIR "${RapidJSON_INCLUDE_DIR}") endif() + + # TODO: Don't use global includes but rather target_include_directories + include_directories(SYSTEM ${RAPIDJSON_INCLUDE_DIR}) endif() -if (ARROW_WITH_BZ2) -# ---------------------------------------------------------------------- -# BZ2 - if("${BZ2_HOME}" STREQUAL "") - message(SEND_ERROR "a binary install of libbz2 must be present, please set the BZ2_HOME environment variable") +macro(build_flatbuffers) + message(STATUS "Building flatbuffers from source") + set(FLATBUFFERS_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/flatbuffers_ep-prefix/src/flatbuffers_ep-install") + if (MSVC) + set(FLATBUFFERS_CMAKE_CXX_FLAGS /EHsc) else() - find_package(Bz2 REQUIRED) + set(FLATBUFFERS_CMAKE_CXX_FLAGS -fPIC) endif() + set(FLATBUFFERS_COMPILER "${FLATBUFFERS_PREFIX}/bin/flatc") + set(FLATBUFFERS_STATIC_LIB "${FLATBUFFERS_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}flatbuffers${CMAKE_STATIC_LIBRARY_SUFFIX}") + # We always need to do release builds, otherwise flatc will not be installed. + ExternalProject_Add(flatbuffers_ep + URL ${FLATBUFFERS_SOURCE_URL} + BUILD_BYPRODUCTS ${FLATBUFFERS_COMPILER} ${FLATBUFFERS_STATIC_LIB} + CMAKE_ARGS + ${EP_COMMON_CMAKE_ARGS} + "-DCMAKE_BUILD_TYPE=RELEASE" + "-DCMAKE_CXX_FLAGS=${FLATBUFFERS_CMAKE_CXX_FLAGS}" + "-DCMAKE_INSTALL_PREFIX:PATH=${FLATBUFFERS_PREFIX}" + "-DFLATBUFFERS_BUILD_TESTS=OFF" + ${EP_LOG_OPTIONS}) - include_directories(SYSTEM ${BZ2_INCLUDE_DIR}) - ADD_THIRDPARTY_LIB(bz2 - STATIC_LIB ${BZ2_STATIC_LIB}) -endif() + file(MAKE_DIRECTORY "${FLATBUFFERS_PREFIX}/include") + + add_library(flatbuffers::flatbuffers STATIC IMPORTED) + set_target_properties(flatbuffers::flatbuffers PROPERTIES + IMPORTED_LOCATION "${FLATBUFFERS_STATIC_LIB}" + INTERFACE_INCLUDE_DIRECTORIES "${FLATBUFFERS_PREFIX}/include" + ) + add_executable(flatbuffers::flatc IMPORTED) + set_target_properties(flatbuffers::flatc PROPERTIES + IMPORTED_LOCATION "${FLATBUFFERS_COMPILER}" + ) + + add_dependencies(toolchain flatbuffers_ep) + add_dependencies(flatbuffers::flatbuffers flatbuffers_ep) + add_dependencies(flatbuffers::flatc flatbuffers_ep) +endmacro() + +if(ARROW_WITH_FLATBUFFERS) + resolve_dependency(Flatbuffers) + + if (TARGET flatbuffers::flatbuffers_shared AND NOT TARGET flatbuffers::flatbuffers) + get_target_property(FLATBUFFERS_INCLUDE_DIR flatbuffers::flatbuffers_shared INTERFACE_INCLUDE_DIRECTORIES) + get_target_property(FLATBUFFERS_SHARED_LIB flatbuffers::flatbuffers_shared IMPORTED_LOCATION) + add_library(flatbuffers::flatbuffers SHARED IMPORTED) + set_target_properties(flatbuffers::flatbuffers PROPERTIES + IMPORTED_LOCATION "${FLATBUFFERS_SHARED_LIB}" + INTERFACE_INCLUDE_DIRECTORIES "${FLATBUFFERS_INCLUDE_DIR}" + ) + endif() -if (ARROW_WITH_LZ4) -# ---------------------------------------------------------------------- -# Lz4 + # mingw-w64-flatbuffers doesn't set the flatc target + if (NOT TARGET flatbuffers::flatc) + get_target_property(FLATBUFFERS_INCLUDE_DIR flatbuffers::flatbuffers INTERFACE_INCLUDE_DIRECTORIES) + get_filename_component(FB_ROOT "${FLATBUFFERS_INCLUDE_DIR}" DIRECTORY) + find_program(FLATBUFFERS_COMPILER + NAMES flatc flatc.exe + PATH ${FB_ROOT} + PATH_SUFFIXES "bin") + add_executable(flatbuffers::flatc IMPORTED) + set_target_properties(flatbuffers::flatc PROPERTIES + IMPORTED_LOCATION "${FLATBUFFERS_COMPILER}" + ) + endif() - if("${LZ4_HOME}" STREQUAL "") - set(LZ4_BUILD_DIR "${CMAKE_CURRENT_BINARY_DIR}/lz4_ep-prefix/src/lz4_ep") - set(LZ4_HOME "${LZ4_BUILD_DIR}") - set(LZ4_INCLUDE_DIR "${LZ4_BUILD_DIR}/lib") + # TODO: Don't use global includes but rather target_include_directories + get_target_property(FLATBUFFERS_INCLUDE_DIR flatbuffers::flatbuffers INTERFACE_INCLUDE_DIRECTORIES) + include_directories(SYSTEM ${FLATBUFFERS_INCLUDE_DIR}) +endif() - if (MSVC) - if (ARROW_USE_STATIC_CRT) - if (${UPPERCASE_BUILD_TYPE} STREQUAL "DEBUG") - set(LZ4_RUNTIME_LIBRARY_LINKAGE "/p:RuntimeLibrary=MultiThreadedDebug") - else() - set(LZ4_RUNTIME_LIBRARY_LINKAGE "/p:RuntimeLibrary=MultiThreaded") - endif() - endif() - set(LZ4_STATIC_LIB "${LZ4_BUILD_DIR}/visual/VS2010/bin/x64_${CMAKE_BUILD_TYPE}/liblz4_static.lib") - set(LZ4_BUILD_COMMAND BUILD_COMMAND msbuild.exe /m /p:Configuration=${CMAKE_BUILD_TYPE} /p:Platform=x64 /p:PlatformToolset=v140 - ${LZ4_RUNTIME_LIBRARY_LINKAGE} /t:Build ${LZ4_BUILD_DIR}/visual/VS2010/lz4.sln) +macro(build_zlib) + message(STATUS "Building ZLIB from source") + set(ZLIB_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/zlib_ep/src/zlib_ep-install") + if (MSVC) + if (${UPPERCASE_BUILD_TYPE} STREQUAL "DEBUG") + set(ZLIB_STATIC_LIB_NAME zlibstaticd.lib) else() - set(LZ4_STATIC_LIB "${LZ4_BUILD_DIR}/lib/liblz4.a") - set(LZ4_BUILD_COMMAND BUILD_COMMAND ${CMAKE_SOURCE_DIR}/build-support/build-lz4-lib.sh "AR=${CMAKE_AR}") + set(ZLIB_STATIC_LIB_NAME zlibstatic.lib) endif() - - ExternalProject_Add(lz4_ep - URL ${LZ4_SOURCE_URL} - ${EP_LOG_OPTIONS} - UPDATE_COMMAND "" - ${LZ4_PATCH_COMMAND} - CONFIGURE_COMMAND "" - INSTALL_COMMAND "" - BINARY_DIR ${LZ4_BUILD_DIR} - BUILD_BYPRODUCTS ${LZ4_STATIC_LIB} - ${LZ4_BUILD_COMMAND} - ) - - set(LZ4_VENDORED 1) - - add_dependencies(toolchain lz4_ep) else() - find_package(Lz4 REQUIRED) - set(LZ4_VENDORED 0) + set(ZLIB_STATIC_LIB_NAME libz.a) endif() + set(ZLIB_STATIC_LIB "${ZLIB_PREFIX}/lib/${ZLIB_STATIC_LIB_NAME}") + set(ZLIB_CMAKE_ARGS ${EP_COMMON_CMAKE_ARGS} + "-DCMAKE_INSTALL_PREFIX=${ZLIB_PREFIX}" + -DBUILD_SHARED_LIBS=OFF) - include_directories(SYSTEM ${LZ4_INCLUDE_DIR}) - ADD_THIRDPARTY_LIB(lz4 - STATIC_LIB ${LZ4_STATIC_LIB}) + ExternalProject_Add(zlib_ep + URL ${ZLIB_SOURCE_URL} + ${EP_LOG_OPTIONS} + BUILD_BYPRODUCTS "${ZLIB_STATIC_LIB}" + CMAKE_ARGS ${ZLIB_CMAKE_ARGS}) - if (LZ4_VENDORED) - add_dependencies(lz4_static lz4_ep) - endif() -endif() + file(MAKE_DIRECTORY "${ZLIB_PREFIX}/include") -if (ARROW_WITH_ZSTD) -# ---------------------------------------------------------------------- -# ZSTD + add_library(ZLIB::ZLIB STATIC IMPORTED) + set_target_properties(ZLIB::ZLIB PROPERTIES + IMPORTED_LOCATION "${ZLIB_STATIC_LIB}" + INTERFACE_INCLUDE_DIRECTORIES "${ZLIB_PREFIX}/include" + ) - if("${ZSTD_HOME}" STREQUAL "") - set(ZSTD_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/zstd_ep-install") - set(ZSTD_INCLUDE_DIR "${ZSTD_PREFIX}/include") + add_dependencies(toolchain zlib_ep) + add_dependencies(ZLIB::ZLIB zlib_ep) +endmacro() - set(ZSTD_CMAKE_ARGS ${EP_COMMON_CMAKE_ARGS} - "-DCMAKE_INSTALL_PREFIX=${ZSTD_PREFIX}" - -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} - -DCMAKE_INSTALL_LIBDIR=${CMAKE_INSTALL_LIBDIR} - -DZSTD_BUILD_PROGRAMS=off - -DZSTD_BUILD_SHARED=off - -DZSTD_BUILD_STATIC=on - -DZSTD_MULTITHREAD_SUPPORT=off) +if(ARROW_WITH_ZLIB) + resolve_dependency(ZLIB) +endif() - if (MSVC) - set(ZSTD_STATIC_LIB "${ZSTD_PREFIX}/${CMAKE_INSTALL_LIBDIR}/zstd_static.lib") - if (ARROW_USE_STATIC_CRT) - set(ZSTD_CMAKE_ARGS ${ZSTD_CMAKE_ARGS} "-DZSTD_USE_STATIC_RUNTIME=on") - endif() - else() - set(ZSTD_STATIC_LIB "${ZSTD_PREFIX}/${CMAKE_INSTALL_LIBDIR}/libzstd.a") - # Only pass our C flags on Unix as on MSVC it leads to a - # "incompatible command-line options" error - set(ZSTD_CMAKE_ARGS ${ZSTD_CMAKE_ARGS} - -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} - -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} - -DCMAKE_C_FLAGS=${EP_C_FLAGS} - -DCMAKE_CXX_FLAGS=${EP_CXX_FLAGS}) - endif() +macro(build_lz4) + message(STATUS "Building lz4 from source") + set(LZ4_BUILD_DIR "${CMAKE_CURRENT_BINARY_DIR}/lz4_ep-prefix/src/lz4_ep") + set(LZ4_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/lz4_ep-prefix") - if(CMAKE_VERSION VERSION_LESS 3.7) - message(FATAL_ERROR "Building zstd using ExternalProject requires \ -at least CMake 3.7") + if (MSVC) + if (ARROW_USE_STATIC_CRT) + if (${UPPERCASE_BUILD_TYPE} STREQUAL "DEBUG") + set(LZ4_RUNTIME_LIBRARY_LINKAGE "/p:RuntimeLibrary=MultiThreadedDebug") + else() + set(LZ4_RUNTIME_LIBRARY_LINKAGE "/p:RuntimeLibrary=MultiThreaded") + endif() endif() + set(LZ4_STATIC_LIB "${LZ4_BUILD_DIR}/visual/VS2010/bin/x64_${CMAKE_BUILD_TYPE}/liblz4_static.lib") + set(LZ4_BUILD_COMMAND BUILD_COMMAND msbuild.exe /m /p:Configuration=${CMAKE_BUILD_TYPE} /p:Platform=x64 /p:PlatformToolset=v140 + ${LZ4_RUNTIME_LIBRARY_LINKAGE} /t:Build ${LZ4_BUILD_DIR}/visual/VS2010/lz4.sln) + else() + set(LZ4_STATIC_LIB "${LZ4_BUILD_DIR}/lib/liblz4.a") + set(LZ4_BUILD_COMMAND BUILD_COMMAND ${CMAKE_SOURCE_DIR}/build-support/build-lz4-lib.sh "AR=${CMAKE_AR}") + endif() - ExternalProject_Add(zstd_ep + # We need to copy the header in lib to directory outside of the build + ExternalProject_Add(lz4_ep + URL ${LZ4_SOURCE_URL} ${EP_LOG_OPTIONS} - CMAKE_ARGS ${ZSTD_CMAKE_ARGS} - SOURCE_SUBDIR "build/cmake" - INSTALL_DIR ${ZSTD_PREFIX} - URL ${ZSTD_SOURCE_URL} - BUILD_BYPRODUCTS "${ZSTD_STATIC_LIB}") - - set(ZSTD_VENDORED 1) - add_dependencies(toolchain zstd_ep) + UPDATE_COMMAND ${CMAKE_COMMAND} -E copy_directory "${LZ4_BUILD_DIR}/lib" "${LZ4_PREFIX}/include" + ${LZ4_PATCH_COMMAND} + CONFIGURE_COMMAND "" + INSTALL_COMMAND "" + BINARY_DIR ${LZ4_BUILD_DIR} + BUILD_BYPRODUCTS ${LZ4_STATIC_LIB} + ${LZ4_BUILD_COMMAND} + ) + + file(MAKE_DIRECTORY "${LZ4_PREFIX}/include") + add_library(LZ4::lz4 STATIC IMPORTED) + set_target_properties(LZ4::lz4 PROPERTIES + IMPORTED_LOCATION "${LZ4_STATIC_LIB}" + INTERFACE_INCLUDE_DIRECTORIES "${LZ4_PREFIX}/include" + ) + add_dependencies(toolchain lz4_ep) + add_dependencies(LZ4::lz4 lz4_ep) +endmacro() + +if(ARROW_WITH_LZ4) + resolve_dependency(Lz4) + + # TODO: Don't use global includes but rather target_include_directories + get_target_property(LZ4_INCLUDE_DIR LZ4::lz4 INTERFACE_INCLUDE_DIRECTORIES) + include_directories(SYSTEM ${LZ4_INCLUDE_DIR}) +endif() + +macro(build_zstd) + message(STATUS "Building zstd from source") + set(ZSTD_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/zstd_ep-install") + + set(ZSTD_CMAKE_ARGS ${EP_COMMON_CMAKE_ARGS} + "-DCMAKE_INSTALL_PREFIX=${ZSTD_PREFIX}" + -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} + -DCMAKE_INSTALL_LIBDIR=${CMAKE_INSTALL_LIBDIR} + -DZSTD_BUILD_PROGRAMS=off + -DZSTD_BUILD_SHARED=off + -DZSTD_BUILD_STATIC=on + -DZSTD_MULTITHREAD_SUPPORT=off) + + if (MSVC) + set(ZSTD_STATIC_LIB "${ZSTD_PREFIX}/${CMAKE_INSTALL_LIBDIR}/zstd_static.lib") + if (ARROW_USE_STATIC_CRT) + set(ZSTD_CMAKE_ARGS ${ZSTD_CMAKE_ARGS} "-DZSTD_USE_STATIC_RUNTIME=on") + endif() else() - find_package(ZSTD REQUIRED) - set(ZSTD_VENDORED 0) + set(ZSTD_STATIC_LIB "${ZSTD_PREFIX}/${CMAKE_INSTALL_LIBDIR}/libzstd.a") + # Only pass our C flags on Unix as on MSVC it leads to a + # "incompatible command-line options" error + set(ZSTD_CMAKE_ARGS ${ZSTD_CMAKE_ARGS} + -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} + -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} + -DCMAKE_C_FLAGS=${EP_C_FLAGS} + -DCMAKE_CXX_FLAGS=${EP_CXX_FLAGS}) endif() - include_directories(SYSTEM ${ZSTD_INCLUDE_DIR}) - ADD_THIRDPARTY_LIB(zstd - STATIC_LIB ${ZSTD_STATIC_LIB}) - - if (ZSTD_VENDORED) - add_dependencies(zstd_static zstd_ep) + if(CMAKE_VERSION VERSION_LESS 3.7) + message(FATAL_ERROR "Building zstd using ExternalProject requires at least CMake 3.7") endif() + + ExternalProject_Add(zstd_ep + ${EP_LOG_OPTIONS} + CMAKE_ARGS ${ZSTD_CMAKE_ARGS} + SOURCE_SUBDIR "build/cmake" + INSTALL_DIR ${ZSTD_PREFIX} + URL ${ZSTD_SOURCE_URL} + BUILD_BYPRODUCTS "${ZSTD_STATIC_LIB}") + + file(MAKE_DIRECTORY "${ZSTD_PREFIX}/include") + + add_library(ZSTD::zstd STATIC IMPORTED) + set_target_properties(ZSTD::zstd PROPERTIES + IMPORTED_LOCATION "${ZSTD_STATIC_LIB}" + INTERFACE_INCLUDE_DIRECTORIES "${ZSTD_PREFIX}/include" + ) + + add_dependencies(toolchain zstd_ep) + add_dependencies(ZSTD::zstd zstd_ep) +endmacro() + +if(ARROW_WITH_ZSTD) + resolve_dependency(ZSTD) + + # TODO: Don't use global includes but rather target_include_directories + get_target_property(ZSTD_INCLUDE_DIR ZSTD::zstd INTERFACE_INCLUDE_DIRECTORIES) + include_directories(SYSTEM ${ZSTD_INCLUDE_DIR}) endif() # ---------------------------------------------------------------------- # RE2 (required for Gandiva) -if (ARROW_GANDIVA) - # re2 - if ("${RE2_HOME}" STREQUAL "") - set (RE2_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/re2_ep-install") - set (RE2_HOME "${RE2_PREFIX}") - set (RE2_INCLUDE_DIR "${RE2_PREFIX}/include") - set (RE2_STATIC_LIB "${RE2_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}re2${CMAKE_STATIC_LIBRARY_SUFFIX}") - - set(RE2_CMAKE_ARGS ${EP_COMMON_CMAKE_ARGS} - "-DCMAKE_INSTALL_PREFIX=${RE2_PREFIX}") - - ExternalProject_Add(re2_ep - ${EP_LOG_OPTIONS} - INSTALL_DIR ${RE2_PREFIX} - URL ${RE2_SOURCE_URL} - CMAKE_ARGS ${RE2_CMAKE_ARGS} - BUILD_BYPRODUCTS "${RE2_STATIC_LIB}") - set (RE2_VENDORED 1) - add_dependencies(toolchain re2_ep) - else () - find_package (RE2 REQUIRED) - set (RE2_VENDORED 0) - endif () - - include_directories (SYSTEM ${RE2_INCLUDE_DIR}) - - if (ARROW_RE2_LINKAGE STREQUAL "shared") - ADD_THIRDPARTY_LIB(re2 - SHARED_LIB ${RE2_SHARED_LIB}) - set(RE2_LIBRARY re2_shared) - else() - ADD_THIRDPARTY_LIB(re2 - STATIC_LIB ${RE2_STATIC_LIB}) - set(RE2_LIBRARY re2_static) + +macro(build_re2) + message(STATUS "Building re2 from source") + set (RE2_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/re2_ep-install") + set (RE2_STATIC_LIB "${RE2_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}re2${CMAKE_STATIC_LIBRARY_SUFFIX}") + + set(RE2_CMAKE_ARGS ${EP_COMMON_CMAKE_ARGS} + "-DCMAKE_INSTALL_PREFIX=${RE2_PREFIX}") + + ExternalProject_Add(re2_ep + ${EP_LOG_OPTIONS} + INSTALL_DIR ${RE2_PREFIX} + URL ${RE2_SOURCE_URL} + CMAKE_ARGS ${RE2_CMAKE_ARGS} + BUILD_BYPRODUCTS "${RE2_STATIC_LIB}") + + file(MAKE_DIRECTORY "${RE2_PREFIX}/include") + add_library(RE2::re2 STATIC IMPORTED) + set_target_properties(RE2::re2 PROPERTIES + IMPORTED_LOCATION "${RE2_STATIC_LIB}" + INTERFACE_INCLUDE_DIRECTORIES "${RE2_PREFIX}/include" + ) + + add_dependencies(toolchain re2_ep) + add_dependencies(RE2::re2 re2_ep) +endmacro() + +if(ARROW_GANDIVA) + resolve_dependency(RE2) + + # TODO: Don't use global includes but rather target_include_directories + get_target_property(RE2_INCLUDE_DIR RE2::re2 INTERFACE_INCLUDE_DIRECTORIES) + include_directories(SYSTEM ${RE2_INCLUDE_DIR}) +endif() + +macro(build_bzip2) + message(STATUS "Building BZip2 from source") + set(BZIP2_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/bzip2_ep-install") + set(BZIP2_STATIC_LIB "${BZIP2_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}bz2${CMAKE_STATIC_LIBRARY_SUFFIX}") + + ExternalProject_Add(bzip2_ep + ${EP_LOG_OPTIONS} + CONFIGURE_COMMAND "" + BUILD_IN_SOURCE 1 + BUILD_COMMAND ${MAKE} ${MAKE_BUILD_ARGS} CFLAGS=${EP_C_FLAGS} + INSTALL_COMMAND ${MAKE} install PREFIX=${BZIP2_PREFIX} CFLAGS=${EP_C_FLAGS} + INSTALL_DIR ${BZIP2_PREFIX} + URL ${BZIP2_SOURCE_URL} + BUILD_BYPRODUCTS "${BZIP2_STATIC_LIB}") + + file(MAKE_DIRECTORY "${BZIP2_PREFIX}/include") + add_library(BZip2::BZip2 STATIC IMPORTED) + set_target_properties(BZip2::BZip2 PROPERTIES + IMPORTED_LOCATION "${BZIP2_STATIC_LIB}" + INTERFACE_INCLUDE_DIRECTORIES "${BZIP2_PREFIX}/include" + ) + set(BZIP2_INCLUDE_DIR "${BZIP2_PREFIX}/include") + + add_dependencies(toolchain bzip2_ep) + add_dependencies(BZip2::BZip2 bzip2_ep) +endmacro() + +if (ARROW_WITH_BZ2) + resolve_dependency(BZip2) + + if (NOT TARGET BZip2::BZip2) + add_library(BZip2::BZip2 UNKNOWN IMPORTED) + set_target_properties(BZip2::BZip2 PROPERTIES + IMPORTED_LOCATION "${BZIP2_LIBRARIES}" + INTERFACE_INCLUDE_DIRECTORIES "${BZIP2_INCLUDE_DIR}" + ) endif() -endif () + include_directories(SYSTEM "${BZIP2_INCLUDE_DIR}") +endif() +macro(build_cares) + set(CARES_PREFIX "${THIRDPARTY_DIR}/cares_ep-install") + set(CARES_INCLUDE_DIR "${CARES_PREFIX}/include") -# ---------------------------------------------------------------------- -# Protocol Buffers (required for ORC and Flight and Gandiva libraries) + # If you set -DCARES_SHARED=ON then the build system names the library + # libcares_static.a + set(CARES_STATIC_LIB "${CARES_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}cares${CMAKE_STATIC_LIBRARY_SUFFIX}") -if (ARROW_WITH_PROTOBUF) - # protobuf - if ("${PROTOBUF_HOME}" STREQUAL "") - set (PROTOBUF_PREFIX "${THIRDPARTY_DIR}/protobuf_ep-install") - set (PROTOBUF_HOME "${PROTOBUF_PREFIX}") - set (PROTOBUF_INCLUDE_DIR "${PROTOBUF_PREFIX}/include") - set (PROTOBUF_STATIC_LIB "${PROTOBUF_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}protobuf${CMAKE_STATIC_LIBRARY_SUFFIX}") - set (PROTOBUF_EXECUTABLE "${PROTOBUF_PREFIX}/bin/protoc") - set (PROTOBUF_CONFIGURE_ARGS "AR=${CMAKE_AR}" - "RANLIB=${CMAKE_RANLIB}" - "CC=${CMAKE_C_COMPILER}" - "CXX=${CMAKE_CXX_COMPILER}" - "--disable-shared" - "--prefix=${PROTOBUF_PREFIX}" - "CFLAGS=${EP_C_FLAGS}" - "CXXFLAGS=${EP_CXX_FLAGS}") - - ExternalProject_Add(protobuf_ep - CONFIGURE_COMMAND "./configure" ${PROTOBUF_CONFIGURE_ARGS} - BUILD_IN_SOURCE 1 - URL ${PROTOBUF_SOURCE_URL} - BUILD_BYPRODUCTS "${PROTOBUF_STATIC_LIB}" "${PROTOBUF_EXECUTABLE}" - ${EP_LOG_OPTIONS}) + set(CARES_CMAKE_ARGS + -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} + -DCARES_STATIC=ON + -DCARES_SHARED=OFF + "-DCMAKE_C_FLAGS=${EP_C_FLAGS}" + "-DCMAKE_INSTALL_PREFIX=${CARES_PREFIX}") + + ExternalProject_Add(cares_ep + ${EP_LOG_OPTIONS} + URL ${CARES_SOURCE_URL} + CMAKE_ARGS ${CARES_CMAKE_ARGS} + BUILD_BYPRODUCTS "${CARES_STATIC_LIB}") + + add_dependencies(toolchain cares_ep) + add_library(c-ares::cares STATIC IMPORTED) + set_target_properties(c-ares::cares PROPERTIES + IMPORTED_LOCATION "${CARES_STATIC_LIB}" + INTERFACE_INCLUDE_DIRECTORIES "${CARES_INCLUDE_DIR}" + ) +endmacro() + +if (ARROW_WITH_GRPC) + get_dependency_source(c-ares) + if (c-ares_SOURCE STREQUAL "AUTO") + find_package(c-ares QUIET) + if (NOT c-ares_FOUND) + # Fedora doesn't package the CMake config + find_package(c-aresAlt) + endif() + if (NOT c-ares_FOUND AND NOT c-aresAlt_FOUND) + build_cares() + endif() + elseif(c-ares_SOURCE STREQUAL "BUNDLED") + build_cares() + elseif(c-ares_SOURCE STREQUAL "SYSTEM") + find_package(c-ares QUIET) + if (NOT c-ares_FOUND) + # Fedora doesn't package the CMake config + find_package(c-aresAlt REQUIRED) + endif() + endif() - set (PROTOBUF_VENDORED 1) - add_dependencies(toolchain protobuf_ep) - else () - find_package (Protobuf REQUIRED) - set (PROTOBUF_VENDORED 0) - endif () - - include_directories (SYSTEM ${PROTOBUF_INCLUDE_DIR}) - if (ARROW_PROTOBUF_USE_SHARED) - ADD_THIRDPARTY_LIB(protobuf - SHARED_LIB ${PROTOBUF_SHARED_LIB}) - set(PROTOBUF_LIBRARY protobuf_shared) - else () - ADD_THIRDPARTY_LIB(protobuf - STATIC_LIB ${PROTOBUF_STATIC_LIB}) - set(PROTOBUF_LIBRARY protobuf_static) - endif () - if (PROTOBUF_VENDORED) - add_dependencies (${PROTOBUF_LIBRARY} protobuf_ep) - endif () + # TODO: Don't use global includes but rather target_include_directories + get_target_property(CARES_INCLUDE_DIR c-ares::cares INTERFACE_INCLUDE_DIRECTORIES) + include_directories(SYSTEM ${CARES_INCLUDE_DIR}) endif() # ---------------------------------------------------------------------- # Dependencies for Arrow Flight RPC -if (ARROW_WITH_GRPC) - if ("${CARES_HOME}" STREQUAL "") - set(CARES_VENDORED 1) - set(CARES_PREFIX "${THIRDPARTY_DIR}/cares_ep-install") - set(CARES_HOME "${CARES_PREFIX}") - set(CARES_INCLUDE_DIR "${CARES_PREFIX}/include") - - # If you set -DCARES_SHARED=ON then the build system names the library - # libcares_static.a - set(CARES_STATIC_LIB "${CARES_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}cares${CMAKE_STATIC_LIBRARY_SUFFIX}") - - set(CARES_CMAKE_ARGS - -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} - -DCARES_STATIC=ON - -DCARES_SHARED=OFF - "-DCMAKE_C_FLAGS=${EP_C_FLAGS}" - "-DCMAKE_INSTALL_PREFIX=${CARES_PREFIX}") - - ExternalProject_Add(cares_ep - ${EP_LOG_OPTIONS} - URL ${CARES_SOURCE_URL} - CMAKE_ARGS ${CARES_CMAKE_ARGS} - BUILD_BYPRODUCTS "${CARES_STATIC_LIB}") +macro(build_grpc) + message(STATUS "Building gRPC from source") + set(GRPC_BUILD_DIR "${CMAKE_CURRENT_BINARY_DIR}/grpc_ep-prefix/src/grpc_ep-build") + set(GRPC_PREFIX "${THIRDPARTY_DIR}/grpc_ep-install") + set(GRPC_HOME "${GRPC_PREFIX}") + set(GRPC_INCLUDE_DIR "${GRPC_PREFIX}/include") + set(GRPC_CMAKE_ARGS ${EP_COMMON_CMAKE_ARGS} + "-DCMAKE_INSTALL_PREFIX=${GRPC_PREFIX}" + -DBUILD_SHARED_LIBS=OFF) - add_dependencies(toolchain cares_ep) - else() - set(CARES_VENDORED 0) - find_package(c-ares REQUIRED) + set(GRPC_STATIC_LIBRARY_GPR "${GRPC_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}gpr${CMAKE_STATIC_LIBRARY_SUFFIX}") + set(GRPC_STATIC_LIBRARY_GRPC "${GRPC_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}grpc${CMAKE_STATIC_LIBRARY_SUFFIX}") + set(GRPC_STATIC_LIBRARY_GRPCPP "${GRPC_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}grpc++${CMAKE_STATIC_LIBRARY_SUFFIX}") + set(GRPC_STATIC_LIBRARY_ADDRESS_SORTING "${GRPC_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}address_sorting${CMAKE_STATIC_LIBRARY_SUFFIX}") + set(GRPC_CPP_PLUGIN "${GRPC_PREFIX}/bin/grpc_cpp_plugin") + + set(GRPC_CMAKE_PREFIX) + + add_custom_target(grpc_dependencies) + + if (CARES_VENDORED) + add_dependencies(grpc_dependencies cares_ep) endif() - message(STATUS "c-ares library: ${CARES_STATIC_LIB}") - add_custom_target(grpc) + if (GFLAGS_VENDORED) + add_dependencies(grpc_dependencies gflags_ep) + endif() - if ("${GRPC_HOME}" STREQUAL "") - set(GRPC_VENDORED 1) - set(GRPC_BUILD_DIR "${CMAKE_CURRENT_BINARY_DIR}/grpc_ep-prefix/src/grpc_ep-build") - set(GRPC_PREFIX "${THIRDPARTY_DIR}/grpc_ep-install") - set(GRPC_HOME "${GRPC_PREFIX}") - set(GRPC_INCLUDE_DIR "${GRPC_PREFIX}/include") - set(GRPC_CMAKE_ARGS ${EP_COMMON_CMAKE_ARGS} - "-DCMAKE_INSTALL_PREFIX=${GRPC_PREFIX}" - -DBUILD_SHARED_LIBS=OFF) + add_dependencies(grpc_dependencies protobuf::libprotobuf c-ares::cares) - set(GRPC_STATIC_LIBRARY_GPR "${GRPC_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}gpr${CMAKE_STATIC_LIBRARY_SUFFIX}") - set(GRPC_STATIC_LIBRARY_GRPC "${GRPC_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}grpc${CMAKE_STATIC_LIBRARY_SUFFIX}") - set(GRPC_STATIC_LIBRARY_GRPCPP "${GRPC_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}grpc++${CMAKE_STATIC_LIBRARY_SUFFIX}") - set(GRPC_STATIC_LIBRARY_ADDRESS_SORTING "${GRPC_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}address_sorting${CMAKE_STATIC_LIBRARY_SUFFIX}") - set(GRPC_CPP_PLUGIN "${GRPC_PREFIX}/bin/grpc_cpp_plugin") + get_target_property(GRPC_PROTOBUF_INCLUDE_DIR protobuf::libprotobuf INTERFACE_INCLUDE_DIRECTORIES) + get_filename_component(GRPC_PB_ROOT "${GRPC_PROTOBUF_INCLUDE_DIR}" DIRECTORY) + get_target_property(GRPC_Protobuf_PROTOC_LIBRARY protobuf::libprotoc IMPORTED_LOCATION) + get_target_property(GRPC_CARES_INCLUDE_DIR c-ares::cares INTERFACE_INCLUDE_DIRECTORIES) + get_filename_component(GRPC_CARES_ROOT "${GRPC_CARES_INCLUDE_DIR}" DIRECTORY) + get_target_property(GRPC_GFLAGS_INCLUDE_DIR ${GFLAGS_LIBRARIES} INTERFACE_INCLUDE_DIRECTORIES) + get_filename_component(GRPC_GFLAGS_ROOT "${GRPC_GFLAGS_INCLUDE_DIR}" DIRECTORY) - set(GRPC_CMAKE_PREFIX) + set(GRPC_CMAKE_PREFIX "${GRPC_CMAKE_PREFIX};${GRPC_PB_ROOT}") + set(GRPC_CMAKE_PREFIX "${GRPC_CMAKE_PREFIX};${GRPC_GFLAGS_ROOT}") + set(GRPC_CMAKE_PREFIX "${GRPC_CMAKE_PREFIX};${GRPC_CARES_ROOT}") - add_custom_target(grpc_dependencies) + # ZLIB is never vendored + set(GRPC_CMAKE_PREFIX "${GRPC_CMAKE_PREFIX};${ZLIB_ROOT}") - if (CARES_VENDORED) - add_dependencies(grpc_dependencies cares_ep) - endif() + if (RAPIDJSON_VENDORED) + add_dependencies(grpc_dependencies rapidjson_ep) + endif() - if (GFLAGS_VENDORED) - add_dependencies(grpc_dependencies gflags_ep) - endif() + # Yuck, see https://stackoverflow.com/a/45433229/776560 + string(REPLACE ";" "|" GRPC_PREFIX_PATH_ALT_SEP "${GRPC_CMAKE_PREFIX}") + + set(GRPC_CMAKE_ARGS + -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} + -DCMAKE_PREFIX_PATH='${GRPC_PREFIX_PATH_ALT_SEP}' + -DgRPC_CARES_PROVIDER=package + -DgRPC_GFLAGS_PROVIDER=package + -DgRPC_PROTOBUF_PROVIDER=package + -DgRPC_SSL_PROVIDER=package + -DgRPC_ZLIB_PROVIDER=package + -DCMAKE_CXX_FLAGS=${EP_CXX_FLAGS} + -DCMAKE_C_FLAGS=${EP_C_FLAGS} + -DCMAKE_INSTALL_PREFIX=${GRPC_PREFIX} + -DCMAKE_INSTALL_LIBDIR=lib + "-DProtobuf_PROTOC_LIBRARY=${GRPC_Protobuf_PROTOC_LIBRARY}" + -DBUILD_SHARED_LIBS=OFF) + + # XXX the gRPC git checkout is huge and takes a long time + # Ideally, we should be able to use the tarballs, but they don't contain + # vendored dependencies such as c-ares... + ExternalProject_Add(grpc_ep + URL ${GRPC_SOURCE_URL} + LIST_SEPARATOR | + BUILD_BYPRODUCTS + ${GRPC_STATIC_LIBRARY_GPR} + ${GRPC_STATIC_LIBRARY_GRPC} + ${GRPC_STATIC_LIBRARY_GRPCPP} + ${GRPC_STATIC_LIBRARY_ADDRESS_SORTING} + ${GRPC_CPP_PLUGIN} + CMAKE_ARGS ${GRPC_CMAKE_ARGS} + ${EP_LOG_OPTIONS}) - if (PROTOBUF_VENDORED) - add_dependencies(grpc_dependencies protobuf_ep) + add_library(gRPC::gpr STATIC IMPORTED) + set_target_properties(gRPC::gpr PROPERTIES + IMPORTED_LOCATION "${GRPC_STATIC_LIBRARY_GPR}" + INTERFACE_INCLUDE_DIRECTORIES "${GRPC_INCLUDE_DIR}" + ) + + add_library(gRPC::grpc STATIC IMPORTED) + set_target_properties(gRPC::grpc PROPERTIES + IMPORTED_LOCATION "${GRPC_STATIC_LIBRARY_GRPC}" + INTERFACE_INCLUDE_DIRECTORIES "${GRPC_INCLUDE_DIR}" + ) + + add_library(gRPC::grpc++ STATIC IMPORTED) + set_target_properties(gRPC::grpc++ PROPERTIES + IMPORTED_LOCATION "${GRPC_STATIC_LIBRARY_GRPCPP}" + INTERFACE_INCLUDE_DIRECTORIES "${GRPC_INCLUDE_DIR}" + ) + + add_library(gRPC::address_sorting STATIC IMPORTED) + set_target_properties(gRPC::address_sorting PROPERTIES + IMPORTED_LOCATION "${GRPC_STATIC_LIBRARY_ADDRESS_SORTING}" + INTERFACE_INCLUDE_DIRECTORIES "${GRPC_INCLUDE_DIR}" + ) + + add_executable(gRPC::grpc_cpp_plugin IMPORTED) + set_target_properties(gRPC::grpc_cpp_plugin PROPERTIES + IMPORTED_LOCATION ${GRPC_CPP_PLUGIN} + ) + + add_dependencies(grpc_ep grpc_dependencies) + add_dependencies(toolchain grpc_ep) + add_dependencies(gRPC::gpr grpc_ep) + add_dependencies(gRPC::grpc grpc_ep) + add_dependencies(gRPC::grpc++ grpc_ep) + add_dependencies(gRPC::address_sorting grpc_ep) +endmacro() + +if (ARROW_WITH_GRPC) + get_dependency_source(gRPC) + if (gRPC_SOURCE STREQUAL "AUTO") + find_package(gRPC QUIET) + if (NOT gRPC_FOUND) + # Ubuntu doesn't package the CMake config + find_package(gRPCAlt) endif() + if (NOT gRPC_FOUND AND NOT gRPCAlt_FOUND) + build_grpc() + endif() + elseif(gRPC_SOURCE STREQUAL "BUNDLED") + build_grpc() + elseif(gRPC_SOURCE STREQUAL "SYSTEM") + find_package(gRPC QUIET) + if (NOT gRPC_FOUND) + # Ubuntu doesn't package the CMake config + find_package(gRPCAlt REQUIRED) + endif() + endif() - set(GRPC_CMAKE_PREFIX "${GRPC_CMAKE_PREFIX};${CARES_HOME}") - set(GRPC_CMAKE_PREFIX "${GRPC_CMAKE_PREFIX};${GFLAGS_HOME}") - set(GRPC_CMAKE_PREFIX "${GRPC_CMAKE_PREFIX};${PROTOBUF_HOME}") + get_target_property(GRPC_CPP_PLUGIN gRPC::grpc_cpp_plugin IMPORTED_LOCATION) + if (NOT GRPC_CPP_PLUGIN) + get_target_property(GRPC_CPP_PLUGIN gRPC::grpc_cpp_plugin IMPORTED_LOCATION_RELEASE) + endif() - # ZLIB is never vendored - set(GRPC_CMAKE_PREFIX "${GRPC_CMAKE_PREFIX};${ZLIB_HOME}") + if (TARGET gRPC::address_sorting) + set(GRPC_HAS_ADDRESS_SORTING TRUE) + else() + set(GRPC_HAS_ADDRESS_SORTING FALSE) + endif() - if (RAPIDJSON_VENDORED) - add_dependencies(grpc_dependencies rapidjson_ep) - endif() + # TODO: Don't use global includes but rather target_include_directories + get_target_property(GRPC_INCLUDE_DIR gRPC::grpc INTERFACE_INCLUDE_DIRECTORIES) + include_directories(SYSTEM ${GRPC_INCLUDE_DIR}) - # Yuck, see https://stackoverflow.com/a/45433229/776560 - string(REPLACE ";" "|" GRPC_PREFIX_PATH_ALT_SEP "${GRPC_CMAKE_PREFIX}") - - set(GRPC_CMAKE_ARGS - -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} - -DCMAKE_PREFIX_PATH='${GRPC_PREFIX_PATH_ALT_SEP}' - -DgRPC_CARES_PROVIDER=package - -DgRPC_GFLAGS_PROVIDER=package - -DgRPC_PROTOBUF_PROVIDER=package - -DgRPC_SSL_PROVIDER=package - -DgRPC_ZLIB_PROVIDER=package - -DCMAKE_CXX_FLAGS=${EP_CXX_FLAGS} - -DCMAKE_C_FLAGS=${EP_C_FLAGS} - -DCMAKE_INSTALL_PREFIX=${GRPC_PREFIX} - -DCMAKE_INSTALL_LIBDIR=lib - -DBUILD_SHARED_LIBS=OFF) - - # XXX the gRPC git checkout is huge and takes a long time - # Ideally, we should be able to use the tarballs, but they don't contain - # vendored dependencies such as c-ares... - ExternalProject_Add(grpc_ep - URL ${GRPC_SOURCE_URL} - LIST_SEPARATOR | - BUILD_BYPRODUCTS - ${GRPC_STATIC_LIBRARY_GPR} - ${GRPC_STATIC_LIBRARY_GRPC} - ${GRPC_STATIC_LIBRARY_GRPCPP} - ${GRPC_STATIC_LIBRARY_ADDRESS_SORTING} - ${GRPC_CPP_PLUGIN} - CMAKE_ARGS ${GRPC_CMAKE_ARGS} - ${EP_LOG_OPTIONS}) + # grpc++ headers may reside in ${GRPC_INCLUDE_DIR}/grpc++ or ${GRPC_INCLUDE_DIR}/grpcpp + # depending on the gRPC version. + if (EXISTS "${GRPC_INCLUDE_DIR}/grpcpp/impl/codegen/config_protobuf.h") + set(GRPCPP_PP_INCLUDE TRUE) + elseif (EXISTS "${GRPC_INCLUDE_DIR}/grpc++/impl/codegen/config_protobuf.h") + set(GRPCPP_PP_INCLUDE FALSE) + else() + message(FATAL_ERROR "Cannot find grpc++ headers in ${GRPC_INCLUDE_DIR}") + endif() +endif() - add_dependencies(grpc_ep grpc_dependencies) - set(GPR_STATIC_LIB "${GRPC_STATIC_LIBRARY_GPR}") - set(GRPC_STATIC_LIB "${GRPC_STATIC_LIBRARY_GRPC}") - set(GRPCPP_STATIC_LIB "${GRPC_STATIC_LIBRARY_GRPCPP}") - set(GRPC_ADDRESS_SORTING_STATIC_LIB "${GRPC_STATIC_LIBRARY_ADDRESS_SORTING}") +# ---------------------------------------------------------------------- +# Add Boost dependencies (code adapted from Apache Kudu (incubating)) - add_dependencies(grpc grpc_ep) - add_dependencies(toolchain grpc) +set(Boost_USE_MULTITHREADED ON) +if (MSVC AND ARROW_USE_STATIC_CRT) + set(Boost_USE_STATIC_RUNTIME ON) +endif() +set(Boost_ADDITIONAL_VERSIONS + "1.70.0" "1.70" + "1.69.0" "1.69" + "1.68.0" "1.68" + "1.67.0" "1.67" + "1.66.0" "1.66" + "1.65.0" "1.65" + "1.64.0" "1.64" + "1.63.0" "1.63" + "1.62.0" "1.61" + "1.61.0" "1.62" + "1.60.0" "1.60") + +if (ARROW_BOOST_VENDORED) + set(BOOST_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/boost_ep-prefix/src/boost_ep") + set(BOOST_LIB_DIR "${BOOST_PREFIX}/stage/lib") + set(BOOST_BUILD_LINK "static") + set(BOOST_STATIC_SYSTEM_LIBRARY + "${BOOST_LIB_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}boost_system${CMAKE_STATIC_LIBRARY_SUFFIX}") + set(BOOST_STATIC_FILESYSTEM_LIBRARY + "${BOOST_LIB_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}boost_filesystem${CMAKE_STATIC_LIBRARY_SUFFIX}") + set(BOOST_STATIC_REGEX_LIBRARY + "${BOOST_LIB_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}boost_regex${CMAKE_STATIC_LIBRARY_SUFFIX}") + set(BOOST_SYSTEM_LIBRARY boost_system_static) + set(BOOST_FILESYSTEM_LIBRARY boost_filesystem_static) + set(BOOST_REGEX_LIBRARY boost_regex_static) + + if (ARROW_BOOST_HEADER_ONLY) + set(BOOST_BUILD_PRODUCTS) + set(BOOST_CONFIGURE_COMMAND "") + set(BOOST_BUILD_COMMAND "") else() - find_package(gRPC REQUIRED) - set(GRPC_VENDORED 0) + set(BOOST_BUILD_PRODUCTS + ${BOOST_STATIC_SYSTEM_LIBRARY} + ${BOOST_STATIC_FILESYSTEM_LIBRARY} + ${BOOST_STATIC_REGEX_LIBRARY}) + set(BOOST_CONFIGURE_COMMAND + "./bootstrap.sh" + "--prefix=${BOOST_PREFIX}" + "--with-libraries=filesystem,regex,system") + if ("${CMAKE_BUILD_TYPE}" STREQUAL "DEBUG") + set(BOOST_BUILD_VARIANT "debug") + else() + set(BOOST_BUILD_VARIANT "release") + endif() + set(BOOST_BUILD_COMMAND + "./b2" + "link=${BOOST_BUILD_LINK}" + "variant=${BOOST_BUILD_VARIANT}" + "cxxflags=-fPIC") + endif() + ExternalProject_Add(boost_ep + URL ${BOOST_SOURCE_URL} + BUILD_BYPRODUCTS ${BOOST_BUILD_PRODUCTS} + BUILD_IN_SOURCE 1 + CONFIGURE_COMMAND ${BOOST_CONFIGURE_COMMAND} + BUILD_COMMAND ${BOOST_BUILD_COMMAND} + INSTALL_COMMAND "" + ${EP_LOG_OPTIONS}) + set(Boost_INCLUDE_DIR "${BOOST_PREFIX}") + set(Boost_INCLUDE_DIRS "${BOOST_INCLUDE_DIR}") + add_dependencies(toolchain boost_ep) +else() + if (MSVC) + # disable autolinking in boost + add_definitions(-DBOOST_ALL_NO_LIB) endif() - if ("${GRPC_CPP_PLUGIN}" STREQUAL "") - message(SEND_ERROR "Please set GRPC_CPP_PLUGIN.") + if (DEFINED ENV{BOOST_ROOT} OR DEFINED BOOST_ROOT) + # In older versions of CMake (such as 3.2), the system paths for Boost will + # be looked in first even if we set $BOOST_ROOT or pass -DBOOST_ROOT + set(Boost_NO_SYSTEM_PATHS ON) endif() - include_directories(SYSTEM ${GRPC_INCLUDE_DIR}) + if (ARROW_BOOST_USE_SHARED) + # Find shared Boost libraries. + set(Boost_USE_STATIC_LIBS OFF) - ADD_THIRDPARTY_LIB(grpc_gpr - STATIC_LIB ${GPR_STATIC_LIB}) + if (MSVC) + # force all boost libraries to dynamic link + add_definitions(-DBOOST_ALL_DYN_LINK) + endif() - ADD_THIRDPARTY_LIB(grpc_grpc - STATIC_LIB ${GRPC_STATIC_LIB}) + if (ARROW_BOOST_HEADER_ONLY) + find_package(Boost REQUIRED) + else() + find_package(Boost COMPONENTS regex system filesystem REQUIRED) + if ("${CMAKE_BUILD_TYPE}" STREQUAL "DEBUG") + set(BOOST_SHARED_SYSTEM_LIBRARY ${Boost_SYSTEM_LIBRARY_DEBUG}) + set(BOOST_SHARED_FILESYSTEM_LIBRARY ${Boost_FILESYSTEM_LIBRARY_DEBUG}) + set(BOOST_SHARED_REGEX_LIBRARY ${Boost_REGEX_LIBRARY_DEBUG}) + else() + set(BOOST_SHARED_SYSTEM_LIBRARY ${Boost_SYSTEM_LIBRARY_RELEASE}) + set(BOOST_SHARED_FILESYSTEM_LIBRARY ${Boost_FILESYSTEM_LIBRARY_RELEASE}) + set(BOOST_SHARED_REGEX_LIBRARY ${Boost_REGEX_LIBRARY_RELEASE}) + endif() + set(BOOST_SYSTEM_LIBRARY boost_system_shared) + set(BOOST_FILESYSTEM_LIBRARY boost_filesystem_shared) + set(BOOST_REGEX_LIBRARY boost_regex_shared) + endif() + else() + # Find static boost headers and libs + # TODO Differentiate here between release and debug builds + set(Boost_USE_STATIC_LIBS ON) + if (ARROW_BOOST_HEADER_ONLY) + find_package(Boost REQUIRED) + else() + find_package(Boost COMPONENTS regex system filesystem REQUIRED) + if ("${CMAKE_BUILD_TYPE}" STREQUAL "DEBUG") + set(BOOST_STATIC_SYSTEM_LIBRARY ${Boost_SYSTEM_LIBRARY_DEBUG}) + set(BOOST_STATIC_FILESYSTEM_LIBRARY ${Boost_FILESYSTEM_LIBRARY_DEBUG}) + set(BOOST_STATIC_REGEX_LIBRARY ${Boost_REGEX_LIBRARY_DEBUG}) + else() + set(BOOST_STATIC_SYSTEM_LIBRARY ${Boost_SYSTEM_LIBRARY_RELEASE}) + set(BOOST_STATIC_FILESYSTEM_LIBRARY ${Boost_FILESYSTEM_LIBRARY_RELEASE}) + set(BOOST_STATIC_REGEX_LIBRARY ${Boost_REGEX_LIBRARY_RELEASE}) + endif() + set(BOOST_SYSTEM_LIBRARY boost_system_static) + set(BOOST_FILESYSTEM_LIBRARY boost_filesystem_static) + set(BOOST_REGEX_LIBRARY boost_regex_static) + endif() + endif() +endif() - ADD_THIRDPARTY_LIB(grpc_grpcpp - STATIC_LIB ${GRPCPP_STATIC_LIB}) +message(STATUS "Boost include dir: " ${Boost_INCLUDE_DIR}) +message(STATUS "Boost libraries: " ${Boost_LIBRARIES}) - ADD_THIRDPARTY_LIB(grpc_address_sorting - STATIC_LIB ${GRPC_ADDRESS_SORTING_STATIC_LIB}) +if (NOT ARROW_BOOST_HEADER_ONLY) + ADD_THIRDPARTY_LIB(boost_system + STATIC_LIB "${BOOST_STATIC_SYSTEM_LIBRARY}" + SHARED_LIB "${BOOST_SHARED_SYSTEM_LIBRARY}") - ADD_THIRDPARTY_LIB(cares - STATIC_LIB ${CARES_STATIC_LIB}) + ADD_THIRDPARTY_LIB(boost_filesystem + STATIC_LIB "${BOOST_STATIC_FILESYSTEM_LIBRARY}" + SHARED_LIB "${BOOST_SHARED_FILESYSTEM_LIBRARY}") + + ADD_THIRDPARTY_LIB(boost_regex + STATIC_LIB "${BOOST_STATIC_REGEX_LIBRARY}" + SHARED_LIB "${BOOST_SHARED_REGEX_LIBRARY}") + + SET(ARROW_BOOST_LIBS ${BOOST_SYSTEM_LIBRARY} ${BOOST_FILESYSTEM_LIBRARY}) +endif() + +include_directories(SYSTEM ${Boost_INCLUDE_DIR}) + +######################################################################## +# HDFS thirdparty setup + +if (DEFINED ENV{HADOOP_HOME}) + set(HADOOP_HOME $ENV{HADOOP_HOME}) + if (NOT EXISTS "${HADOOP_HOME}/include/hdfs.h") + message(STATUS "Did not find hdfs.h in expected location, using vendored one") + set(HADOOP_HOME "${THIRDPARTY_DIR}/hadoop") + endif() +else() + set(HADOOP_HOME "${THIRDPARTY_DIR}/hadoop") endif() +set(HDFS_H_PATH "${HADOOP_HOME}/include/hdfs.h") +if (NOT EXISTS ${HDFS_H_PATH}) + message(FATAL_ERROR "Did not find hdfs.h at ${HDFS_H_PATH}") +endif() +message(STATUS "Found hdfs.h at: " ${HDFS_H_PATH}) + +include_directories(SYSTEM "${HADOOP_HOME}/include") + + + # ---------------------------------------------------------------------- # Apache ORC @@ -1519,9 +1992,18 @@ if (ARROW_ORC) set(ORC_CMAKE_CXX_FLAGS "${EP_CXX_FLAGS} ${ORC_CMAKE_CXX_FLAGS}") - # Since LZ4 isn't installed, the header file is in ${LZ4_HOME}/lib instead of - # ${LZ4_HOME}/include, which forces us to specify the include directory - # manually as well. + get_target_property(ORC_PROTOBUF_INCLUDE_DIR protobuf::libprotobuf INTERFACE_INCLUDE_DIRECTORIES) + get_filename_component(ORC_PB_ROOT "${ORC_PROTOBUF_INCLUDE_DIR}" DIRECTORY) + get_target_property(ORC_PROTOBUF_LIBRARY protobuf::libprotobuf IMPORTED_LOCATION) + + get_target_property(ORC_SNAPPY_INCLUDE_DIR Snappy::snappy INTERFACE_INCLUDE_DIRECTORIES) + get_filename_component(ORC_SNAPPY_ROOT "${ORC_SNAPPY_INCLUDE_DIR}" DIRECTORY) + + get_target_property(ORC_LZ4_ROOT LZ4::lz4 INTERFACE_INCLUDE_DIRECTORIES) + get_filename_component(ORC_LZ4_ROOT "${ORC_LZ4_ROOT}" DIRECTORY) + + # Weirdly passing in PROTOBUF_LIBRARY for PROTOC_LIBRARY still results in ORC finding + # the protoc library. set (ORC_CMAKE_ARGS ${EP_COMMON_CMAKE_ARGS} "-DCMAKE_INSTALL_PREFIX=${ORC_PREFIX}" -DCMAKE_CXX_FLAGS=${ORC_CMAKE_CXX_FLAGS} @@ -1530,11 +2012,17 @@ if (ARROW_ORC) -DBUILD_TOOLS=OFF -DBUILD_CPP_TESTS=OFF -DINSTALL_VENDORED_LIBS=OFF - -DPROTOBUF_HOME=${PROTOBUF_HOME} - -DLZ4_HOME=${LZ4_HOME} - -DLZ4_INCLUDE_DIR=${LZ4_INCLUDE_DIR} - -DSNAPPY_HOME=${SNAPPY_HOME} - -DZLIB_HOME=${ZLIB_HOME}) + "-DSNAPPY_HOME=${ORC_SNAPPY_ROOT}" + "-DSNAPPY_INCLUDE_DIR=${ORC_SNAPPY_INCLUDE_DIR}" + "-DPROTOBUF_HOME=${ORC_PB_ROOT}" + "-DPROTOBUF_INCLUDE_DIR=${ORC_PROTOBUF_INCLUDE_DIR}" + "-DPROTOBUF_LIBRARY=${ORC_PROTOBUF_LIBRARY}" + "-DPROTOC_LIBRARY=${ORC_PROTOBUF_LIBRARY}" + "-DLZ4_HOME=${LZ4_HOME}" + ) + if (ZLIB_ROOT) + set(ORC_CMAKE_ARGS ${ORC_CMAKE_ARGS} "-DZLIB_HOME=${ZLIB_ROOT}") + endif() ExternalProject_Add(orc_ep URL ${ORC_SOURCE_URL} @@ -1545,16 +2033,10 @@ if (ARROW_ORC) add_dependencies(toolchain orc_ep) set(ORC_VENDORED 1) - add_dependencies(orc_ep ${ZLIB_LIBRARY}) - if (LZ4_VENDORED) - add_dependencies(orc_ep lz4_static) - endif() - if (SNAPPY_VENDORED) - add_dependencies(orc_ep snappy_static) - endif() - if (PROTOBUF_VENDORED) - add_dependencies(orc_ep ${PROTOBUF_LIBRARY}) - endif() + add_dependencies(orc_ep ZLIB::ZLIB) + add_dependencies(orc_ep LZ4::lz4) + add_dependencies(orc_ep Snappy::snappy) + add_dependencies(orc_ep protobuf::libprotobuf) else() set(ORC_INCLUDE_DIR "${ORC_HOME}/include") set(ORC_STATIC_LIB "${ORC_HOME}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}orc${CMAKE_STATIC_LIBRARY_SUFFIX}") @@ -1564,210 +2046,14 @@ if (ARROW_ORC) include_directories(SYSTEM ${ORC_INCLUDE_DIR}) ADD_THIRDPARTY_LIB(orc STATIC_LIB ${ORC_STATIC_LIB} - DEPS ${PROTOBUF_LIBRARY}) + DEPS protobuf::libprotobuf) if (ORC_VENDORED) add_dependencies(orc_static orc_ep) endif() endif() -# ---------------------------------------------------------------------- -# Thrift - -if (ARROW_WITH_THRIFT) - -# find thrift headers and libs -find_package(Thrift) - -if (NOT THRIFT_FOUND) - set(THRIFT_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/thrift_ep/src/thrift_ep-install") - set(THRIFT_HOME "${THRIFT_PREFIX}") - set(THRIFT_INCLUDE_DIR "${THRIFT_PREFIX}/include") - set(THRIFT_COMPILER "${THRIFT_PREFIX}/bin/thrift") - set(THRIFT_CMAKE_ARGS ${EP_COMMON_CMAKE_ARGS} - "-DCMAKE_INSTALL_PREFIX=${THRIFT_PREFIX}" - -DCMAKE_INSTALL_RPATH=${THRIFT_PREFIX}/lib - -DBUILD_SHARED_LIBS=OFF - -DBUILD_TESTING=OFF - -DBUILD_EXAMPLES=OFF - -DBUILD_TUTORIALS=OFF - -DWITH_QT4=OFF - -DWITH_C_GLIB=OFF - -DWITH_JAVA=OFF - -DWITH_PYTHON=OFF - -DWITH_HASKELL=OFF - -DWITH_CPP=ON - -DWITH_STATIC_LIB=ON - -DWITH_LIBEVENT=OFF) - - # Thrift also uses boost. Forward important boost settings if there were ones passed. - if (DEFINED BOOST_ROOT) - set(THRIFT_CMAKE_ARGS ${THRIFT_CMAKE_ARGS} "-DBOOST_ROOT=${BOOST_ROOT}") - endif() - if (DEFINED Boost_NAMESPACE) - set(THRIFT_CMAKE_ARGS ${THRIFT_CMAKE_ARGS} "-DBoost_NAMESPACE=${Boost_NAMESPACE}") - endif() +# Write out the package configurations. - set(THRIFT_STATIC_LIB_NAME "${CMAKE_STATIC_LIBRARY_PREFIX}thrift") - if (MSVC) - if (ARROW_USE_STATIC_CRT) - set(THRIFT_STATIC_LIB_NAME "${THRIFT_STATIC_LIB_NAME}mt") - set(THRIFT_CMAKE_ARGS ${THRIFT_CMAKE_ARGS} "-DWITH_MT=ON") - else() - set(THRIFT_STATIC_LIB_NAME "${THRIFT_STATIC_LIB_NAME}md") - set(THRIFT_CMAKE_ARGS ${THRIFT_CMAKE_ARGS} "-DWITH_MT=OFF") - endif() - endif() - if (${UPPERCASE_BUILD_TYPE} STREQUAL "DEBUG") - set(THRIFT_STATIC_LIB_NAME "${THRIFT_STATIC_LIB_NAME}d") - endif() - set(THRIFT_STATIC_LIB "${THRIFT_PREFIX}/lib/${THRIFT_STATIC_LIB_NAME}${CMAKE_STATIC_LIBRARY_SUFFIX}") - - if (ZLIB_SHARED_LIB) - set(THRIFT_CMAKE_ARGS "-DZLIB_LIBRARY=${ZLIB_SHARED_LIB}" - ${THRIFT_CMAKE_ARGS}) - else() - set(THRIFT_CMAKE_ARGS "-DZLIB_LIBRARY=${ZLIB_STATIC_LIB}" - ${THRIFT_CMAKE_ARGS}) - endif() - set(THRIFT_DEPENDENCIES ${THRIFT_DEPENDENCIES} ${ZLIB_LIBRARY}) - - if (MSVC) - set(WINFLEXBISON_VERSION 2.4.9) - set(WINFLEXBISON_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/winflexbison_ep/src/winflexbison_ep-install") - ExternalProject_Add(winflexbison_ep - URL https://github.com/lexxmark/winflexbison/releases/download/v.${WINFLEXBISON_VERSION}/win_flex_bison-${WINFLEXBISON_VERSION}.zip - URL_HASH MD5=a2e979ea9928fbf8567e995e9c0df765 - SOURCE_DIR ${WINFLEXBISON_PREFIX} - CONFIGURE_COMMAND "" - BUILD_COMMAND "" - INSTALL_COMMAND "" - ${EP_LOG_OPTIONS}) - set(THRIFT_DEPENDENCIES ${THRIFT_DEPENDENCIES} winflexbison_ep) - - set(THRIFT_CMAKE_ARGS "-DFLEX_EXECUTABLE=${WINFLEXBISON_PREFIX}/win_flex.exe" - "-DBISON_EXECUTABLE=${WINFLEXBISON_PREFIX}/win_bison.exe" - "-DZLIB_INCLUDE_DIR=${ZLIB_INCLUDE_DIR}" - "-DWITH_SHARED_LIB=OFF" - "-DWITH_PLUGIN=OFF" - ${THRIFT_CMAKE_ARGS}) - elseif (APPLE) - # Some other process always resets BISON_EXECUTABLE to the system default, - # thus we use our own variable here. - if (NOT DEFINED THRIFT_BISON_EXECUTABLE) - find_package(BISON 2.5.1) - - # In the case where we cannot find a system-wide installation, look for - # homebrew and ask for its bison installation. - if (NOT BISON_FOUND) - find_program(BREW_BIN brew) - if (BREW_BIN) - execute_process( - COMMAND ${BREW_BIN} --prefix bison - OUTPUT_VARIABLE BISON_PREFIX - OUTPUT_STRIP_TRAILING_WHITESPACE - ) - set(BISON_EXECUTABLE "${BISON_PREFIX}/bin/bison") - find_package(BISON 2.5.1) - set(THRIFT_BISON_EXECUTABLE "${BISON_EXECUTABLE}") - endif() - else() - set(THRIFT_BISON_EXECUTABLE "${BISON_EXECUTABLE}") - endif() - endif() - set(THRIFT_CMAKE_ARGS "-DBISON_EXECUTABLE=${THRIFT_BISON_EXECUTABLE}" - ${THRIFT_CMAKE_ARGS}) - endif() - - ExternalProject_Add(thrift_ep - URL ${THRIFT_SOURCE_URL} - BUILD_BYPRODUCTS "${THRIFT_STATIC_LIB}" "${THRIFT_COMPILER}" - CMAKE_ARGS ${THRIFT_CMAKE_ARGS} - DEPENDS ${THRIFT_DEPENDENCIES} - ${EP_LOG_OPTIONS}) - - set(THRIFT_VENDORED 1) - - add_dependencies(toolchain thrift_ep) -else() - set(THRIFT_VENDORED 0) -endif() - -include_directories(SYSTEM ${THRIFT_INCLUDE_DIR} ${THRIFT_INCLUDE_DIR}/thrift) -message(STATUS "Thrift include dir: ${THRIFT_INCLUDE_DIR}") -message(STATUS "Thrift static library: ${THRIFT_STATIC_LIB}") -message(STATUS "Thrift compiler: ${THRIFT_COMPILER}") -add_library(thriftstatic STATIC IMPORTED) -set_target_properties(thriftstatic PROPERTIES IMPORTED_LOCATION ${THRIFT_STATIC_LIB}) - -if (THRIFT_VENDORED) - add_dependencies(thriftstatic thrift_ep) -endif() - -if (THRIFT_VERSION VERSION_LESS "0.11.0") - add_definitions(-DPARQUET_THRIFT_USE_BOOST) - message(STATUS "Using Boost in Thrift header") -endif() - -endif() # ARROW_HIVESERVER2 - -# ---------------------------------------------------------------------- -# GLOG - -if (ARROW_USE_GLOG) - if("${GLOG_HOME}" STREQUAL "") - set(GLOG_BUILD_DIR "${CMAKE_CURRENT_BINARY_DIR}/glog_ep-prefix/src/glog_ep") - set(GLOG_INCLUDE_DIR "${GLOG_BUILD_DIR}/include") - set(GLOG_STATIC_LIB "${GLOG_BUILD_DIR}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}glog${CMAKE_STATIC_LIBRARY_SUFFIX}") - set(GLOG_CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fPIC") - set(GLOG_CMAKE_C_FLAGS "${EP_C_FLAGS} -fPIC") - if (Threads::Threads) - set(GLOG_CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fPIC -pthread") - set(GLOG_CMAKE_C_FLAGS "${EP_C_FLAGS} -fPIC -pthread") - endif() - message(STATUS "GLOG_CMAKE_CXX_FLAGS: ${GLOG_CMAKE_CXX_FLAGS}") - message(STATUS "CMAKE_CXX_FLAGS in glog: ${GLOG_CMAKE_CXX_FLAGS}") - - if(APPLE) - # If we don't set this flag, the binary built with 10.13 cannot be used in 10.12. - set(GLOG_CMAKE_CXX_FLAGS "${GLOG_CMAKE_CXX_FLAGS} -mmacosx-version-min=10.9") - endif() - - set(GLOG_CMAKE_ARGS ${EP_COMMON_CMAKE_ARGS} - "-DCMAKE_INSTALL_PREFIX=${GLOG_BUILD_DIR}" - -DBUILD_SHARED_LIBS=OFF - -DBUILD_TESTING=OFF - -DWITH_GFLAGS=OFF - -DCMAKE_CXX_FLAGS_${UPPERCASE_BUILD_TYPE}=${GLOG_CMAKE_CXX_FLAGS} - -DCMAKE_C_FLAGS_${UPPERCASE_BUILD_TYPE}=${GLOG_CMAKE_C_FLAGS} - -DCMAKE_CXX_FLAGS=${GLOG_CMAKE_CXX_FLAGS}) - message(STATUS "Glog version: ${GLOG_VERSION}") - ExternalProject_Add(glog_ep - URL ${GLOG_SOURCE_URL} - BUILD_IN_SOURCE 1 - BUILD_BYPRODUCTS "${GLOG_STATIC_LIB}" - CMAKE_ARGS ${GLOG_CMAKE_ARGS} - ${EP_LOG_OPTIONS}) - - set(GLOG_VENDORED 1) - add_dependencies(toolchain glog_ep) - else() - find_package(GLOG REQUIRED) - set(GLOG_VENDORED 0) - endif() - - message(STATUS "Glog include dir: ${GLOG_INCLUDE_DIR}") - message(STATUS "Glog static library: ${GLOG_STATIC_LIB}") - - include_directories(SYSTEM ${GLOG_INCLUDE_DIR}) - - if (GLOG_VENDORED) - ADD_THIRDPARTY_LIB(glog - STATIC_LIB ${GLOG_STATIC_LIB}) - add_dependencies(glog_static glog_ep) - else() - ADD_THIRDPARTY_LIB(glog - STATIC_LIB ${GLOG_STATIC_LIB} - DEPS ${GFLAGS_LIBRARY}) - endif() -endif() +configure_file("src/arrow/util/config.h.cmake" "src/arrow/util/config.h") +install(FILES "${ARROW_BINARY_DIR}/src/arrow/util/config.h" DESTINATION "include/arrow/util") diff --git a/cpp/src/arrow/CMakeLists.txt b/cpp/src/arrow/CMakeLists.txt index b54157993fe..4ef60c9f168 100644 --- a/cpp/src/arrow/CMakeLists.txt +++ b/cpp/src/arrow/CMakeLists.txt @@ -165,7 +165,7 @@ if(ARROW_CUDA) endif() if(ARROW_JEMALLOC AND JEMALLOC_VENDORED) - add_dependencies(arrow_dependencies jemalloc_static) + add_dependencies(arrow_dependencies jemalloc::jemalloc) endif() if(ARROW_WITH_BROTLI) @@ -277,7 +277,7 @@ if(ARROW_BUILD_TESTS OR ARROW_BUILD_BENCHMARKS) arrow_test_dependencies SHARED_LINK_LIBS arrow_shared - ${GTEST_LIBRARY} + GTest::GTest STATIC_LINK_LIBS arrow_static) diff --git a/cpp/src/arrow/array-struct-test.cc b/cpp/src/arrow/array-struct-test.cc index a92a3e20585..1262be85e13 100644 --- a/cpp/src/arrow/array-struct-test.cc +++ b/cpp/src/arrow/array-struct-test.cc @@ -165,9 +165,9 @@ TEST_F(TestStructBuilder, TestBasics) { Int32Builder* int_vb = checked_cast(builder_->field_builder(1)); ASSERT_EQ(2, static_cast(builder_->num_fields())); - EXPECT_OK(builder_->Resize(list_lengths.size())); - EXPECT_OK(char_vb->Resize(list_values.size())); - EXPECT_OK(int_vb->Resize(int_values.size())); + ARROW_EXPECT_OK(builder_->Resize(list_lengths.size())); + ARROW_EXPECT_OK(char_vb->Resize(list_values.size())); + ARROW_EXPECT_OK(int_vb->Resize(int_values.size())); int pos = 0; for (size_t i = 0; i < list_lengths.size(); ++i) { diff --git a/cpp/src/arrow/buffer-test.cc b/cpp/src/arrow/buffer-test.cc index e503b286013..e7f60a5828d 100644 --- a/cpp/src/arrow/buffer-test.cc +++ b/cpp/src/arrow/buffer-test.cc @@ -38,14 +38,14 @@ namespace arrow { TEST(TestAllocate, Bitmap) { std::shared_ptr new_buffer; - EXPECT_OK(AllocateBitmap(default_memory_pool(), 100, &new_buffer)); + ARROW_EXPECT_OK(AllocateBitmap(default_memory_pool(), 100, &new_buffer)); EXPECT_GE(new_buffer->size(), 13); EXPECT_EQ(new_buffer->capacity() % 8, 0); } TEST(TestAllocate, EmptyBitmap) { std::shared_ptr new_buffer; - EXPECT_OK(AllocateEmptyBitmap(default_memory_pool(), 100, &new_buffer)); + ARROW_EXPECT_OK(AllocateEmptyBitmap(default_memory_pool(), 100, &new_buffer)); EXPECT_EQ(new_buffer->size(), 13); EXPECT_EQ(new_buffer->capacity() % 8, 0); EXPECT_TRUE(std::all_of(new_buffer->data(), new_buffer->data() + new_buffer->capacity(), diff --git a/cpp/src/arrow/compute/kernels/aggregate-benchmark.cc b/cpp/src/arrow/compute/kernels/aggregate-benchmark.cc index 8b6cd9059c1..b533b91e811 100644 --- a/cpp/src/arrow/compute/kernels/aggregate-benchmark.cc +++ b/cpp/src/arrow/compute/kernels/aggregate-benchmark.cc @@ -308,12 +308,26 @@ void BenchSum(benchmark::State& state) { state.SetBytesProcessed(state.iterations() * array_size * sizeof(T)); } +template +struct BenchmarkArgsType; + +template +struct BenchmarkArgsType&)> { + using type = Values; +}; + static void SetArgs(benchmark::internal::Benchmark* bench) { + // Benchmark changed its parameter type between releases from + // int to int64_t. As it doesn't have version macros, we need + // to apply C++ template magic. + using ArgsType = + typename BenchmarkArgsType::type; bench->Unit(benchmark::kMicrosecond); for (auto size : {kL1Size, kL2Size, kL3Size, kL3Size * 4}) - for (auto nulls : std::vector({0, 1, 10, 50})) - bench->Args({static_cast(size), nulls}); + for (auto nulls : std::vector({0, 1, 10, 50})) + bench->Args({static_cast(size), nulls}); } BENCHMARK_TEMPLATE(BenchSum, SumNoNulls)->Apply(SetArgs); diff --git a/cpp/src/arrow/dbi/hiveserver2/CMakeLists.txt b/cpp/src/arrow/dbi/hiveserver2/CMakeLists.txt index a1111cc1531..0c3926e4dc5 100644 --- a/cpp/src/arrow/dbi/hiveserver2/CMakeLists.txt +++ b/cpp/src/arrow/dbi/hiveserver2/CMakeLists.txt @@ -105,7 +105,7 @@ set(ARROW_HIVESERVER2_TEST_LINK_LIBS arrow_hiveserver2_static arrow_hiveserver2_thrift ${ARROW_TEST_LINK_LIBS} - thriftstatic) + Thrift::thrift) if(ARROW_BUILD_TESTS) add_test_case(hiveserver2-test diff --git a/cpp/src/arrow/flight/CMakeLists.txt b/cpp/src/arrow/flight/CMakeLists.txt index 8bd23e5acac..f3a3d80f976 100644 --- a/cpp/src/arrow/flight/CMakeLists.txt +++ b/cpp/src/arrow/flight/CMakeLists.txt @@ -21,12 +21,15 @@ add_custom_target(arrow_flight) arrow_install_all_headers("arrow/flight") set(ARROW_FLIGHT_STATIC_LINK_LIBS - ${PROTOBUF_LIBRARY} - grpc_grpcpp_static - grpc_grpc_static - grpc_gpr_static - grpc_address_sorting_static - cares_static) + protobuf::libprotobuf + gRPC::grpc++ + gRPC::grpc + gRPC::gpr + c-ares::cares) + +if(GRPC_HAS_ADDRESS_SORTING) + list(APPEND ARROW_FLIGHT_STATIC_LINK_LIBS gRPC::address_sorting) +endif() set(ARROW_FLIGHT_TEST_LINK_LIBS arrow_flight_shared @@ -45,14 +48,14 @@ set(FLIGHT_GENERATED_PROTO_FILES "${CMAKE_CURRENT_BINARY_DIR}/Flight.grpc.pb.cc" "${CMAKE_CURRENT_BINARY_DIR}/Flight.grpc.pb.h") -set(PROTO_DEPENDS ${FLIGHT_PROTO} ${PROTOBUF_LIBRARY} grpc) +set(PROTO_DEPENDS ${FLIGHT_PROTO} protobuf::libprotobuf gRPC::grpc) add_custom_command( OUTPUT ${FLIGHT_GENERATED_PROTO_FILES} - COMMAND ${PROTOBUF_EXECUTABLE} "-I${FLIGHT_PROTO_PATH}" + COMMAND protobuf::protoc "-I${FLIGHT_PROTO_PATH}" "--cpp_out=${CMAKE_CURRENT_BINARY_DIR}" "${FLIGHT_PROTO}" DEPENDS ${PROTO_DEPENDS} ARGS - COMMAND ${PROTOBUF_EXECUTABLE} "-I${FLIGHT_PROTO_PATH}" + COMMAND protobuf::protoc "-I${FLIGHT_PROTO_PATH}" "--grpc_out=${CMAKE_CURRENT_BINARY_DIR}" "--plugin=protoc-gen-grpc=${GRPC_CPP_PLUGIN}" "${FLIGHT_PROTO}") @@ -91,15 +94,16 @@ if(ARROW_BUILD_TESTS OR ARROW_BUILD_BENCHMARKS) SOURCES test-util.cc DEPENDENCIES - ${GTEST_LIBRARY} + GTest::GTest flight_grpc_gen arrow_dependencies + metadata_fbs SHARED_LINK_LIBS arrow_shared arrow_flight_shared ${BOOST_FILESYSTEM_LIBRARY} ${BOOST_SYSTEM_LIBRARY} - ${GTEST_LIBRARY} + GTest::GTest STATIC_LINK_LIBS arrow_static arrow_flight_static) @@ -115,15 +119,15 @@ add_arrow_test(flight-test if(ARROW_BUILD_TESTS OR ARROW_BUILD_BENCHMARKS) add_executable(flight-test-server test-server.cc) target_link_libraries(flight-test-server ${ARROW_FLIGHT_TEST_LINK_LIBS} - ${GFLAGS_LIBRARY} ${GTEST_LIBRARY}) + ${GFLAGS_LIBRARIES} GTest::GTest) add_executable(flight-test-integration-server test-integration-server.cc) target_link_libraries(flight-test-integration-server ${ARROW_FLIGHT_TEST_LINK_LIBS} - ${GFLAGS_LIBRARY} ${GTEST_LIBRARY}) + ${GFLAGS_LIBRARIES} GTest::GTest) add_executable(flight-test-integration-client test-integration-client.cc) target_link_libraries(flight-test-integration-client ${ARROW_FLIGHT_TEST_LINK_LIBS} - ${GFLAGS_LIBRARY} ${GTEST_LIBRARY}) + ${GFLAGS_LIBRARIES} GTest::GTest) # This is needed for the unit tests if(ARROW_BUILD_TESTS) @@ -140,7 +144,7 @@ if(ARROW_BUILD_BENCHMARKS) "${CMAKE_CURRENT_BINARY_DIR}/perf.pb.h") add_custom_command(OUTPUT ${PERF_PROTO_GENERATED_FILES} - COMMAND ${PROTOBUF_EXECUTABLE} "-I${CMAKE_CURRENT_SOURCE_DIR}" + COMMAND protobuf::protoc "-I${CMAKE_CURRENT_SOURCE_DIR}" "--cpp_out=${CMAKE_CURRENT_BINARY_DIR}" "perf.proto" DEPENDS ${PROTO_DEPENDS}) @@ -149,16 +153,16 @@ if(ARROW_BUILD_BENCHMARKS) arrow_flight_shared arrow_flight_testing_shared ${ARROW_FLIGHT_TEST_LINK_LIBS} - ${GFLAGS_LIBRARY} - ${GTEST_LIBRARY}) + ${GFLAGS_LIBRARIES} + GTest::GTest) add_executable(arrow-flight-benchmark flight-benchmark.cc perf.pb.cc) target_link_libraries(arrow-flight-benchmark arrow_flight_static arrow_testing_static ${ARROW_FLIGHT_TEST_LINK_LIBS} - ${GFLAGS_LIBRARY} - ${GTEST_LIBRARY}) + ${GFLAGS_LIBRARIES} + GTest::GTest) add_dependencies(arrow-flight-benchmark arrow-flight-perf-server) diff --git a/cpp/src/arrow/flight/client.cc b/cpp/src/arrow/flight/client.cc index f3420c4bced..5cebd9f62df 100644 --- a/cpp/src/arrow/flight/client.cc +++ b/cpp/src/arrow/flight/client.cc @@ -23,7 +23,12 @@ #include #include +#include "arrow/util/config.h" +#ifdef GRPCPP_PP_INCLUDE #include +#else +#include +#endif #include "arrow/ipc/dictionary.h" #include "arrow/ipc/metadata-internal.h" diff --git a/cpp/src/arrow/flight/customize_protobuf.h b/cpp/src/arrow/flight/customize_protobuf.h index 9f5dfab9571..58c168c23b9 100644 --- a/cpp/src/arrow/flight/customize_protobuf.h +++ b/cpp/src/arrow/flight/customize_protobuf.h @@ -20,7 +20,12 @@ #include #include +#include "arrow/util/config.h" +#ifdef GRPCPP_PP_INCLUDE #include +#else +#include +#endif // It is necessary to undefined this macro so that the protobuf // SerializationTraits specialization is not declared in proto_utils.h. We've @@ -29,7 +34,11 @@ // for our faster serialization-deserialization path #undef GRPC_OPEN_SOURCE_PROTO +#ifdef GRPCPP_PP_INCLUDE #include +#else +#include +#endif namespace grpc { diff --git a/cpp/src/arrow/flight/internal.cc b/cpp/src/arrow/flight/internal.cc index 1085e11a973..0c4ba4591af 100644 --- a/cpp/src/arrow/flight/internal.cc +++ b/cpp/src/arrow/flight/internal.cc @@ -23,7 +23,12 @@ #include #include +#include "arrow/util/config.h" +#ifdef GRPCPP_PP_INCLUDE #include +#else +#include +#endif #include "arrow/buffer.h" #include "arrow/io/memory.h" diff --git a/cpp/src/arrow/flight/serialization-internal.cc b/cpp/src/arrow/flight/serialization-internal.cc index d80c0c7411b..b7e566cf978 100644 --- a/cpp/src/arrow/flight/serialization-internal.cc +++ b/cpp/src/arrow/flight/serialization-internal.cc @@ -22,11 +22,18 @@ #include #include +#include "arrow/util/config.h" + #include #include #include +#ifdef GRPCPP_PP_INCLUDE #include #include +#else +#include +#include +#endif #include "arrow/buffer.h" #include "arrow/flight/server.h" diff --git a/cpp/src/arrow/flight/server.cc b/cpp/src/arrow/flight/server.cc index cc1c03d3258..d1977a34fba 100644 --- a/cpp/src/arrow/flight/server.cc +++ b/cpp/src/arrow/flight/server.cc @@ -25,7 +25,12 @@ #include #include +#include "arrow/util/config.h" +#ifdef GRPCPP_PP_INCLUDE #include +#else +#include +#endif #include "arrow/ipc/dictionary.h" #include "arrow/ipc/reader.h" diff --git a/cpp/src/arrow/flight/test-util.cc b/cpp/src/arrow/flight/test-util.cc index becad1bf42e..7ce8ef50dc2 100644 --- a/cpp/src/arrow/flight/test-util.cc +++ b/cpp/src/arrow/flight/test-util.cc @@ -139,9 +139,9 @@ std::vector ExampleFlightInfo() { auto schema2 = ExampleSchema2(); FlightInfo::Data flight1, flight2; - EXPECT_OK( + ARROW_EXPECT_OK( MakeFlightInfo(*schema1, descr1, {endpoint1, endpoint2}, 1000, 100000, &flight1)); - EXPECT_OK(MakeFlightInfo(*schema2, descr2, {endpoint3}, 1000, 100000, &flight2)); + ARROW_EXPECT_OK(MakeFlightInfo(*schema2, descr2, {endpoint3}, 1000, 100000, &flight2)); return {FlightInfo(flight1), FlightInfo(flight2)}; } diff --git a/cpp/src/arrow/gpu/CMakeLists.txt b/cpp/src/arrow/gpu/CMakeLists.txt index 3a2413b0150..334904dc8a0 100644 --- a/cpp/src/arrow/gpu/CMakeLists.txt +++ b/cpp/src/arrow/gpu/CMakeLists.txt @@ -76,7 +76,7 @@ endif() if(ARROW_BUILD_BENCHMARKS) cuda_add_executable(arrow-cuda-benchmark cuda-benchmark.cc) - target_link_libraries(arrow-cuda-benchmark arrow_cuda_shared ${GTEST_LIBRARY} + target_link_libraries(arrow-cuda-benchmark arrow_cuda_shared GTest::GTest ${ARROW_BENCHMARK_LINK_LIBS}) add_dependencies(arrow_cuda-benchmarks arrow-cuda-benchmark) endif() diff --git a/cpp/src/arrow/io/hdfs-test.cc b/cpp/src/arrow/io/hdfs-test.cc index 114cda9b27f..ba3f3285348 100644 --- a/cpp/src/arrow/io/hdfs-test.cc +++ b/cpp/src/arrow/io/hdfs-test.cc @@ -138,9 +138,9 @@ class TestHadoopFileSystem : public ::testing::Test { void TearDown() { if (client_) { if (client_->Exists(scratch_dir_)) { - EXPECT_OK(client_->Delete(scratch_dir_, true)); + ARROW_EXPECT_OK(client_->Delete(scratch_dir_, true)); } - EXPECT_OK(client_->Disconnect()); + ARROW_EXPECT_OK(client_->Disconnect()); } } @@ -208,9 +208,9 @@ TYPED_TEST(TestHadoopFileSystem, MakeDirectory) { ASSERT_OK(this->client_->MakeDirectory(path)); ASSERT_TRUE(this->client_->Exists(path)); std::vector listing; - EXPECT_OK(this->client_->ListDirectory(path, &listing)); + ARROW_EXPECT_OK(this->client_->ListDirectory(path, &listing)); ASSERT_EQ(0, listing.size()); - EXPECT_OK(this->client_->Delete(path, true)); + ARROW_EXPECT_OK(this->client_->Delete(path, true)); ASSERT_FALSE(this->client_->Exists(path)); ASSERT_RAISES(IOError, this->client_->ListDirectory(path, &listing)); } diff --git a/cpp/src/arrow/io/memory-benchmark.cc b/cpp/src/arrow/io/memory-benchmark.cc index c0b061a85fb..78389574b63 100644 --- a/cpp/src/arrow/io/memory-benchmark.cc +++ b/cpp/src/arrow/io/memory-benchmark.cc @@ -44,6 +44,8 @@ static const int64_t kL3Size = cpu_info->CacheSize(CpuInfo::L3_CACHE); constexpr size_t kMemoryPerCore = 32 * 1024 * 1024; using BufferPtr = std::shared_ptr; +#ifndef _MSC_VER + #ifdef ARROW_AVX512 using VectorType = __m512i; @@ -70,7 +72,7 @@ using VectorType = __m256i; asm volatile("vmovntdqa %[src], %[dst]" : [dst] "=v"(DST) : [src] "m"(SRC) :) #define VectorStreamWrite _mm256_stream_si256 -#else +#else // ARROW_AVX2 not set using VectorType = __m128i; #define VectorSet _mm_set1_epi32 @@ -82,8 +84,8 @@ using VectorType = __m128i; asm volatile("movntdqa %[src], %[dst]" : [dst] "=x"(DST) : [src] "m"(SRC) :) #define VectorStreamWrite _mm_stream_si128 -#endif -#endif +#endif // ARROW_AVX2 +#endif // ARROW_AVX512 static void Read(void* src, void* dst, size_t size) { const auto simd = static_cast(src); @@ -197,6 +199,8 @@ BENCHMARK_TEMPLATE(MemoryBandwidth, StreamWrite)->Apply(SetMemoryBandwidthArgs); BENCHMARK_TEMPLATE(MemoryBandwidth, StreamReadWrite)->Apply(SetMemoryBandwidthArgs); BENCHMARK_TEMPLATE(MemoryBandwidth, PlatformMemcpy)->Apply(SetMemoryBandwidthArgs); +#endif // _MSC_VER + static void ParallelMemoryCopy(benchmark::State& state) { // NOLINT non-const reference const int64_t n_threads = state.range(0); const int64_t buffer_size = kMemoryPerCore; diff --git a/cpp/src/arrow/io/memory-test.cc b/cpp/src/arrow/io/memory-test.cc index c879cda595c..6b75793262d 100644 --- a/cpp/src/arrow/io/memory-test.cc +++ b/cpp/src/arrow/io/memory-test.cc @@ -54,7 +54,7 @@ TEST_F(TestBufferOutputStream, DtorCloses) { const int K = 100; for (int i = 0; i < K; ++i) { - EXPECT_OK(stream_->Write(data)); + ARROW_EXPECT_OK(stream_->Write(data)); } stream_ = nullptr; @@ -66,7 +66,7 @@ TEST_F(TestBufferOutputStream, CloseResizes) { const int K = 100; for (int i = 0; i < K; ++i) { - EXPECT_OK(stream_->Write(data)); + ARROW_EXPECT_OK(stream_->Write(data)); } ASSERT_OK(stream_->Close()); diff --git a/cpp/src/arrow/ipc/CMakeLists.txt b/cpp/src/arrow/ipc/CMakeLists.txt index d4128b8165a..4b07d7d176c 100644 --- a/cpp/src/arrow/ipc/CMakeLists.txt +++ b/cpp/src/arrow/ipc/CMakeLists.txt @@ -28,7 +28,7 @@ add_arrow_test(json-simple-test PREFIX "arrow-ipc") add_arrow_test(json-test PREFIX "arrow-ipc") if(NOT ARROW_BOOST_HEADER_ONLY) - add_arrow_test(json-integration-test EXTRA_LINK_LIBS ${GFLAGS_LIBRARY}) + add_arrow_test(json-integration-test EXTRA_LINK_LIBS ${GFLAGS_LIBRARIES}) # Test is being built if(TARGET arrow-json-integration-test) @@ -66,15 +66,14 @@ foreach(FIL ${FBS_SRC}) list(APPEND ABS_FBS_SRC ${ABS_FIL}) endforeach() -if(FLATBUFFERS_VENDORED) - set(FBS_DEPENDS ${ABS_FBS_SRC} flatbuffers_ep) -else() - set(FBS_DEPENDS ${ABS_FBS_SRC}) +get_target_property(FLATC_EXECUTABLE flatbuffers::flatc IMPORTED_LOCATION) +if(NOT FLATC_EXECUTABLE) + get_target_property(FLATC_EXECUTABLE flatbuffers::flatc IMPORTED_LOCATION_RELEASE) endif() - +message(STATUS "flatc: ${FLATC_EXECUTABLE}") add_custom_command(OUTPUT ${FBS_OUTPUT_FILES} - COMMAND ${FLATBUFFERS_COMPILER} -c -o ${OUTPUT_DIR} ${ABS_FBS_SRC} - DEPENDS ${FBS_DEPENDS} + COMMAND ${FLATC_EXECUTABLE} -c -o ${OUTPUT_DIR} ${ABS_FBS_SRC} + DEPENDS flatbuffers::flatc COMMENT "Running flatc compiler on ${ABS_FBS_SRC}" VERBATIM) diff --git a/cpp/src/arrow/ipc/feather-test.cc b/cpp/src/arrow/ipc/feather-test.cc index 6ad2923543a..e7b699d79ad 100644 --- a/cpp/src/arrow/ipc/feather-test.cc +++ b/cpp/src/arrow/ipc/feather-test.cc @@ -260,8 +260,8 @@ void CheckArrays(const Array& expected, const Array& result) { std::stringstream pp_result; std::stringstream pp_expected; - EXPECT_OK(PrettyPrint(result, 0, &pp_result)); - EXPECT_OK(PrettyPrint(expected, 0, &pp_expected)); + ARROW_EXPECT_OK(PrettyPrint(result, 0, &pp_result)); + ARROW_EXPECT_OK(PrettyPrint(expected, 0, &pp_expected)); FAIL() << "Got: " << pp_result.str() << "\nExpected: " << pp_expected.str(); } } @@ -271,8 +271,8 @@ void CheckBatches(const RecordBatch& expected, const RecordBatch& result) { std::stringstream pp_result; std::stringstream pp_expected; - EXPECT_OK(PrettyPrint(result, 0, &pp_result)); - EXPECT_OK(PrettyPrint(expected, 0, &pp_expected)); + ARROW_EXPECT_OK(PrettyPrint(result, 0, &pp_result)); + ARROW_EXPECT_OK(PrettyPrint(expected, 0, &pp_expected)); FAIL() << "Got: " << pp_result.str() << "\nExpected: " << pp_expected.str(); } } diff --git a/cpp/src/arrow/python/CMakeLists.txt b/cpp/src/arrow/python/CMakeLists.txt index e1b0d7d9ad3..6f618367ed4 100644 --- a/cpp/src/arrow/python/CMakeLists.txt +++ b/cpp/src/arrow/python/CMakeLists.txt @@ -115,7 +115,7 @@ arrow_add_pkg_config("arrow-python") if(ARROW_BUILD_TESTS) add_library(arrow_python_test_main STATIC util/test_main.cc) - target_link_libraries(arrow_python_test_main ${GTEST_LIBRARY}) + target_link_libraries(arrow_python_test_main GTest::GTest) target_include_directories(arrow_python_test_main SYSTEM PUBLIC ${ARROW_PYTHON_INCLUDES}) diff --git a/cpp/src/arrow/python/util/CMakeLists.txt b/cpp/src/arrow/python/util/CMakeLists.txt index 204877ffe82..abf952c3bfc 100644 --- a/cpp/src/arrow/python/util/CMakeLists.txt +++ b/cpp/src/arrow/python/util/CMakeLists.txt @@ -23,10 +23,10 @@ if(PYARROW_BUILD_TESTS) add_library(arrow/python_test_main STATIC test_main.cc) if(APPLE) - target_link_libraries(arrow/python_test_main ${GTEST_LIBRARY} dl) + target_link_libraries(arrow/python_test_main GTest::GTest dl) set_target_properties(arrow/python_test_main PROPERTIES LINK_FLAGS "-undefined dynamic_lookup") else() - target_link_libraries(arrow/python_test_main ${GTEST_LIBRARY} pthread dl) + target_link_libraries(arrow/python_test_main GTest::GTest pthread dl) endif() endif() diff --git a/cpp/src/arrow/symbols.map b/cpp/src/arrow/symbols.map index 083b4b2a7a4..9ee0ff310ea 100644 --- a/cpp/src/arrow/symbols.map +++ b/cpp/src/arrow/symbols.map @@ -60,6 +60,9 @@ FSE_*; FSEv*; ZBUFFv*; + ZSTDMT*; + POOL_*; + HIST_*; ERR_getErrorString; # jemalloc je_arrow_*; diff --git a/cpp/src/arrow/testing/gtest_common.h b/cpp/src/arrow/testing/gtest_common.h index 200f0eb5f71..d0221de4b49 100644 --- a/cpp/src/arrow/testing/gtest_common.h +++ b/cpp/src/arrow/testing/gtest_common.h @@ -46,7 +46,7 @@ class TestBase : public ::testing::Test { const int64_t null_nbytes = BitUtil::BytesForBits(length); std::shared_ptr null_bitmap; - EXPECT_OK(AllocateBuffer(pool_, null_nbytes, &null_bitmap)); + ARROW_EXPECT_OK(AllocateBuffer(pool_, null_nbytes, &null_bitmap)); memset(null_bitmap->mutable_data(), 255, null_nbytes); for (int64_t i = 0; i < null_count; i++) { BitUtil::ClearBit(null_bitmap->mutable_data(), i * (length / null_count)); @@ -66,7 +66,7 @@ template std::shared_ptr TestBase::MakeRandomArray(int64_t length, int64_t null_count) { const int64_t data_nbytes = length * sizeof(typename ArrayType::value_type); std::shared_ptr data; - EXPECT_OK(AllocateBuffer(pool_, data_nbytes, &data)); + ARROW_EXPECT_OK(AllocateBuffer(pool_, data_nbytes, &data)); // Fill with random data random_bytes(data_nbytes, random_seed_++, data->mutable_data()); @@ -87,7 +87,7 @@ inline std::shared_ptr TestBase::MakeRandomArray( const int byte_width = 10; std::shared_ptr null_bitmap = MakeRandomNullBitmap(length, null_count); std::shared_ptr data; - EXPECT_OK(AllocateBuffer(pool_, byte_width * length, &data)); + ARROW_EXPECT_OK(AllocateBuffer(pool_, byte_width * length, &data)); ::arrow::random_bytes(data->size(), 0, data->mutable_data()); return std::make_shared(fixed_size_binary(byte_width), length, @@ -107,15 +107,15 @@ inline std::shared_ptr TestBase::MakeRandomArray(int64_t len uint8_t buffer[kBufferSize]; for (int64_t i = 0; i < length; i++) { if (!valid_bytes[i]) { - EXPECT_OK(builder.AppendNull()); + ARROW_EXPECT_OK(builder.AppendNull()); } else { ::arrow::random_bytes(kBufferSize, static_cast(i), buffer); - EXPECT_OK(builder.Append(buffer, kBufferSize)); + ARROW_EXPECT_OK(builder.Append(buffer, kBufferSize)); } } std::shared_ptr out; - EXPECT_OK(builder.Finish(&out)); + ARROW_EXPECT_OK(builder.Finish(&out)); return out; } diff --git a/cpp/src/arrow/testing/gtest_util.cc b/cpp/src/arrow/testing/gtest_util.cc index 9f8547256d9..1db631b5a0a 100644 --- a/cpp/src/arrow/testing/gtest_util.cc +++ b/cpp/src/arrow/testing/gtest_util.cc @@ -59,8 +59,8 @@ void AssertChunkedEqual(const ChunkedArray& expected, const ChunkedArray& actual auto c1 = actual.chunk(i); auto c2 = expected.chunk(i); if (!c1->Equals(*c2)) { - EXPECT_OK(::arrow::PrettyPrint(*c1, 0, &pp_result)); - EXPECT_OK(::arrow::PrettyPrint(*c2, 0, &pp_expected)); + ARROW_EXPECT_OK(::arrow::PrettyPrint(*c1, 0, &pp_result)); + ARROW_EXPECT_OK(::arrow::PrettyPrint(*c2, 0, &pp_expected)); FAIL() << "Chunk " << i << " Got: " << pp_result.str() << "\nExpected: " << pp_expected.str(); } @@ -73,7 +73,8 @@ void AssertChunkedEqual(const ChunkedArray& actual, const ArrayVector& expected) } void AssertBufferEqual(const Buffer& buffer, const std::vector& expected) { - ASSERT_EQ(buffer.size(), expected.size()) << "Mismatching buffer size"; + ASSERT_EQ(static_cast(buffer.size()), expected.size()) + << "Mismatching buffer size"; const uint8_t* buffer_data = buffer.data(); for (size_t i = 0; i < expected.size(); ++i) { ASSERT_EQ(buffer_data[i], expected[i]); @@ -81,7 +82,8 @@ void AssertBufferEqual(const Buffer& buffer, const std::vector& expecte } void AssertBufferEqual(const Buffer& buffer, const std::string& expected) { - ASSERT_EQ(buffer.size(), expected.length()) << "Mismatching buffer size"; + ASSERT_EQ(static_cast(buffer.size()), expected.length()) + << "Mismatching buffer size"; const uint8_t* buffer_data = buffer.data(); for (size_t i = 0; i < expected.size(); ++i) { ASSERT_EQ(buffer_data[i], expected[i]); @@ -114,7 +116,7 @@ void PrintColumn(const Column& col, std::stringstream* ss) { for (int i = 0; i < carr.num_chunks(); ++i) { auto c1 = carr.chunk(i); *ss << "Chunk " << i << std::endl; - EXPECT_OK(::arrow::PrettyPrint(*c1, 0, ss)); + ARROW_EXPECT_OK(::arrow::PrettyPrint(*c1, 0, ss)); *ss << std::endl; } } diff --git a/cpp/src/arrow/testing/gtest_util.h b/cpp/src/arrow/testing/gtest_util.h index 9e2fb916252..7f46dfdc933 100644 --- a/cpp/src/arrow/testing/gtest_util.h +++ b/cpp/src/arrow/testing/gtest_util.h @@ -77,7 +77,7 @@ #define ASSERT_OK_NO_THROW(expr) ASSERT_NO_THROW(ASSERT_OK(expr)) -#define EXPECT_OK(expr) \ +#define ARROW_EXPECT_OK(expr) \ do { \ ::arrow::Status s = (expr); \ EXPECT_TRUE(s.ok()); \ @@ -113,8 +113,8 @@ using ArrayVector = std::vector>; std::stringstream pp_result; \ std::stringstream pp_expected; \ \ - EXPECT_OK(PrettyPrint(RIGHT, 0, &pp_result)); \ - EXPECT_OK(PrettyPrint(LEFT, 0, &pp_expected)); \ + ARROW_EXPECT_OK(PrettyPrint(RIGHT, 0, &pp_result)); \ + ARROW_EXPECT_OK(PrettyPrint(LEFT, 0, &pp_expected)); \ FAIL() << "Got: \n" << pp_result.str() << "\nExpected: \n" << pp_expected.str(); \ } \ } while (false) diff --git a/cpp/src/arrow/util/CMakeLists.txt b/cpp/src/arrow/util/CMakeLists.txt index 4846c5be1b2..ba24f88e28b 100644 --- a/cpp/src/arrow/util/CMakeLists.txt +++ b/cpp/src/arrow/util/CMakeLists.txt @@ -26,19 +26,6 @@ arrow_install_all_headers("arrow/util") # arrow_test_main # -if(ARROW_BUILD_BENCHMARKS) - add_library(arrow_benchmark_main benchmark_main.cc) - if(APPLE) - target_link_libraries(arrow_benchmark_main gbenchmark_static) - elseif(MSVC) - target_link_libraries(arrow_benchmark_main gbenchmark_static Shlwapi.lib) - else() - target_link_libraries(arrow_benchmark_main gbenchmark_static Threads::Threads) - endif() - - add_dependencies(arrow_benchmark_main arrow_benchmark_dependencies) -endif() - add_arrow_test(bit-util-test) add_arrow_test(checked-cast-test) add_arrow_test(compression-test) diff --git a/cpp/src/arrow/util/compression_lz4.cc b/cpp/src/arrow/util/compression_lz4.cc index d157ba61760..238628b6be1 100644 --- a/cpp/src/arrow/util/compression_lz4.cc +++ b/cpp/src/arrow/util/compression_lz4.cc @@ -28,6 +28,10 @@ #include "arrow/util/logging.h" #include "arrow/util/macros.h" +#ifndef LZ4F_HEADER_SIZE_MAX +#define LZ4F_HEADER_SIZE_MAX 19 +#endif + namespace arrow { namespace util { @@ -83,7 +87,7 @@ class LZ4Decompressor : public Decompressor { bool IsFinished() override { return finished_; } protected: - LZ4F_dctx* ctx_ = nullptr; + LZ4F_decompressionContext_t ctx_ = nullptr; bool finished_; }; @@ -123,7 +127,7 @@ class LZ4Compressor : public Compressor { bool* should_retry) override; protected: - LZ4F_cctx* ctx_ = nullptr; + LZ4F_compressionContext_t ctx_ = nullptr; LZ4F_preferences_t prefs_; bool first_time_; }; diff --git a/cpp/src/arrow/util/config.h.cmake b/cpp/src/arrow/util/config.h.cmake new file mode 100644 index 00000000000..06732a3ed57 --- /dev/null +++ b/cpp/src/arrow/util/config.h.cmake @@ -0,0 +1,19 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#cmakedefine DOUBLE_CONVERSION_HAS_CASE_INSENSIBILITY +#cmakedefine GRPCPP_PP_INCLUDE diff --git a/cpp/src/arrow/util/parsing.h b/cpp/src/arrow/util/parsing.h index fc6ca040478..9bd102373af 100644 --- a/cpp/src/arrow/util/parsing.h +++ b/cpp/src/arrow/util/parsing.h @@ -34,6 +34,7 @@ #include "arrow/type.h" #include "arrow/type_traits.h" #include "arrow/util/checked_cast.h" +#include "arrow/util/config.h" #include "arrow/vendored/datetime.h" namespace arrow { @@ -119,8 +120,13 @@ class StringToFloatConverterMixin { } protected: +// This is only support in double-conversion 3.1+ +#ifdef DOUBLE_CONVERSION_HAS_CASE_INSENSIBILITY static const int flags_ = double_conversion::StringToDoubleConverter::ALLOW_CASE_INSENSIBILITY; +#else + static const int flags_ = double_conversion::StringToDoubleConverter::NO_FLAGS; +#endif // Two unlikely values to signal a parsing error static constexpr double main_junk_value_ = 0.7066424364107089; static constexpr double fallback_junk_value_ = 0.40088499148279166; diff --git a/cpp/src/gandiva/CMakeLists.txt b/cpp/src/gandiva/CMakeLists.txt index 1885c491867..02f2e70d46c 100644 --- a/cpp/src/gandiva/CMakeLists.txt +++ b/cpp/src/gandiva/CMakeLists.txt @@ -25,6 +25,9 @@ add_custom_target(gandiva-benchmarks) add_dependencies(gandiva-all gandiva gandiva-tests gandiva-benchmarks) +if(ARROW_DEPENDENCY_SOURCE STREQUAL "CONDA") + set(LLVM_ROOT $ENV{CONDA_PREFIX}) +endif() find_package(LLVM) # Set the path where the bitcode file generated, see precompiled/CMakeLists.txt @@ -73,9 +76,9 @@ set(SRC_FILES to_date_holder.cc ${GANDIVA_PRECOMPILED_CC_PATH}) -set(GANDIVA_SHARED_PRIVATE_LINK_LIBS arrow_shared LLVM::LLVM_INTERFACE ${RE2_LIBRARY}) +set(GANDIVA_SHARED_PRIVATE_LINK_LIBS arrow_shared LLVM::LLVM_INTERFACE RE2::re2) -set(GANDIVA_STATIC_LINK_LIBS arrow_static LLVM::LLVM_INTERFACE ${RE2_LIBRARY}) +set(GANDIVA_STATIC_LINK_LIBS arrow_static LLVM::LLVM_INTERFACE RE2::re2) if(ARROW_GANDIVA_STATIC_LIBSTDCPP AND (CMAKE_COMPILER_IS_GNUCC OR CMAKE_COMPILER_IS_GNUCXX)) @@ -130,9 +133,9 @@ arrow_install_all_headers("gandiva") # pkg-config support arrow_add_pkg_config("gandiva") -set(GANDIVA_STATIC_TEST_LINK_LIBS gandiva_static ${RE2_LIBRARY} ${ARROW_TEST_LINK_LIBS}) +set(GANDIVA_STATIC_TEST_LINK_LIBS gandiva_static RE2::re2 ${ARROW_TEST_LINK_LIBS}) -set(GANDIVA_SHARED_TEST_LINK_LIBS gandiva_shared ${RE2_LIBRARY} ${ARROW_TEST_LINK_LIBS}) +set(GANDIVA_SHARED_TEST_LINK_LIBS gandiva_shared RE2::re2 ${ARROW_TEST_LINK_LIBS}) function(ADD_GANDIVA_TEST REL_TEST_NAME) set(options USE_STATIC_LINKING) diff --git a/cpp/src/gandiva/decimal_type_util_test.cc b/cpp/src/gandiva/decimal_type_util_test.cc index a593990638a..98ea0bb16ac 100644 --- a/cpp/src/gandiva/decimal_type_util_test.cc +++ b/cpp/src/gandiva/decimal_type_util_test.cc @@ -29,7 +29,7 @@ namespace gandiva { Decimal128TypePtr DoOp(DecimalTypeUtil::Op op, Decimal128TypePtr d1, Decimal128TypePtr d2) { Decimal128TypePtr ret_type; - EXPECT_OK(DecimalTypeUtil::GetResultType(op, {d1, d2}, &ret_type)); + ARROW_EXPECT_OK(DecimalTypeUtil::GetResultType(op, {d1, d2}, &ret_type)); return ret_type; } diff --git a/cpp/src/gandiva/jni/CMakeLists.txt b/cpp/src/gandiva/jni/CMakeLists.txt index eb276a9d58e..af58efd05bd 100644 --- a/cpp/src/gandiva/jni/CMakeLists.txt +++ b/cpp/src/gandiva/jni/CMakeLists.txt @@ -32,11 +32,11 @@ get_filename_component(ABS_GANDIVA_PROTO ${CMAKE_SOURCE_DIR}/src/gandiva/proto/T ABSOLUTE) add_custom_command(OUTPUT ${PROTO_OUTPUT_FILES} - COMMAND ${PROTOBUF_EXECUTABLE} + COMMAND protobuf::protoc --proto_path ${CMAKE_SOURCE_DIR}/src/gandiva/proto --cpp_out ${PROTO_OUTPUT_DIR} ${CMAKE_SOURCE_DIR}/src/gandiva/proto/Types.proto - DEPENDS ${ABS_GANDIVA_PROTO} ${PROTOBUF_LIBRARY} + DEPENDS ${ABS_GANDIVA_PROTO} protobuf::libprotobuf COMMENT "Running PROTO compiler on Types.proto" VERBATIM) @@ -48,7 +48,7 @@ set(PROTO_HDRS "${PROTO_OUTPUT_DIR}/Types.pb.h") set(JNI_HEADERS_DIR "${CMAKE_CURRENT_BINARY_DIR}/java") add_subdirectory(../../../../java/gandiva ./java/gandiva) -set(GANDIVA_LINK_LIBS ${PROTOBUF_LIBRARY} gandiva_static) +set(GANDIVA_LINK_LIBS protobuf::libprotobuf gandiva_static) set(GANDIVA_JNI_SOURCES config_builder.cc diff --git a/cpp/src/gandiva/precompiled/CMakeLists.txt b/cpp/src/gandiva/precompiled/CMakeLists.txt index b2c3017845f..6d3daf22a43 100644 --- a/cpp/src/gandiva/precompiled/CMakeLists.txt +++ b/cpp/src/gandiva/precompiled/CMakeLists.txt @@ -101,7 +101,7 @@ function(add_precompiled_unit_test REL_TEST_NAME) add_executable(${TEST_NAME} ${REL_TEST_NAME} ${ARGN}) target_include_directories(${TEST_NAME} PRIVATE ${CMAKE_SOURCE_DIR}/src) - target_link_libraries(${TEST_NAME} PRIVATE ${ARROW_TEST_LINK_LIBS} ${RE2_LIBRARY}) + target_link_libraries(${TEST_NAME} PRIVATE ${ARROW_TEST_LINK_LIBS} RE2::re2) target_compile_definitions(${TEST_NAME} PRIVATE GANDIVA_UNIT_TEST=1 diff --git a/cpp/src/gandiva/precompiled/decimal_ops_test.cc b/cpp/src/gandiva/precompiled/decimal_ops_test.cc index 9672a25e150..05289e215f9 100644 --- a/cpp/src/gandiva/precompiled/decimal_ops_test.cc +++ b/cpp/src/gandiva/precompiled/decimal_ops_test.cc @@ -118,7 +118,7 @@ void TestDecimalSql::Verify(DecimalTypeUtil::Op op, const DecimalScalar128& x, int64_t context = 0; Decimal128TypePtr out_type; - EXPECT_OK(DecimalTypeUtil::GetResultType(op, {t1, t2}, &out_type)); + ARROW_EXPECT_OK(DecimalTypeUtil::GetResultType(op, {t1, t2}, &out_type)); arrow::BasicDecimal128 out_value; std::string op_name; diff --git a/cpp/src/gandiva/tests/decimal_single_test.cc b/cpp/src/gandiva/tests/decimal_single_test.cc index c28c47e2217..666ee4a68d5 100644 --- a/cpp/src/gandiva/tests/decimal_single_test.cc +++ b/cpp/src/gandiva/tests/decimal_single_test.cc @@ -100,7 +100,7 @@ void TestDecimalOps::Verify(DecimalTypeUtil::Op op, const std::string& function, Decimal128TypePtr output_type; auto status = DecimalTypeUtil::GetResultType(op, {x_type, y_type}, &output_type); - EXPECT_OK(status); + ARROW_EXPECT_OK(status); // output fields auto res = field("res", output_type); @@ -111,7 +111,7 @@ void TestDecimalOps::Verify(DecimalTypeUtil::Op op, const std::string& function, // Build a projector for the expression. std::shared_ptr projector; status = Projector::Make(schema, {expr}, TestConfiguration(), &projector); - EXPECT_OK(status); + ARROW_EXPECT_OK(status); // Create a row-batch with some sample data auto array_a = MakeDecimalVector(x); @@ -123,7 +123,7 @@ void TestDecimalOps::Verify(DecimalTypeUtil::Op op, const std::string& function, // Evaluate expression arrow::ArrayVector outputs; status = projector->Evaluate(*in_batch, pool_, &outputs); - EXPECT_OK(status); + ARROW_EXPECT_OK(status); // Validate results auto out_array = dynamic_cast(outputs[0].get()); diff --git a/cpp/src/gandiva/tests/generate_data.h b/cpp/src/gandiva/tests/generate_data.h index 398057510cb..244a473e767 100644 --- a/cpp/src/gandiva/tests/generate_data.h +++ b/cpp/src/gandiva/tests/generate_data.h @@ -39,8 +39,11 @@ class Random { explicit Random(uint32_t seed = 100) : seed_(seed) {} // This is 3 times faster than random_device +#ifndef _MSC_VER int32_t next() { return rand_r(&seed_); } - // int32_t next() { return random_dev_(); } +#else + int32_t next() { return random_dev_(); } +#endif private: uint32_t seed_; diff --git a/cpp/src/parquet/CMakeLists.txt b/cpp/src/parquet/CMakeLists.txt index 26269665de2..f448064f78c 100644 --- a/cpp/src/parquet/CMakeLists.txt +++ b/cpp/src/parquet/CMakeLists.txt @@ -96,7 +96,7 @@ if(MSVC) set(PARQUET_BOOST_LINK_LIBS ${PARQUET_BOOST_LINK_LIBS} ${BOOST_SYSTEM_LIBRARY}) endif() -set(PARQUET_MIN_TEST_LIBS ${GTEST_MAIN_LIBRARY} ${GTEST_LIBRARY}) +set(PARQUET_MIN_TEST_LIBS GTest::Main GTest::GTest) if(APPLE) set(PARQUET_MIN_TEST_LIBS ${PARQUET_MIN_TEST_LIBS} ${CMAKE_DL_LIBS}) @@ -108,14 +108,15 @@ set(PARQUET_SHARED_TEST_LINK_LIBS arrow_testing_shared ${PARQUET_MIN_TEST_LIBS} parquet_shared - thriftstatic) + Thrift::thrift) set(PARQUET_STATIC_TEST_LINK_LIBS ${PARQUET_MIN_TEST_LIBS} ${ARROW_LIBRARIES_FOR_STATIC_TESTS} parquet_static) if(WIN32) # The benchmarks depend on some static Thrift symbols - set(PARQUET_BENCHMARK_LINK_OPTION STATIC_LINK_LIBS arrow_benchmark_main parquet_static) + set(PARQUET_BENCHMARK_LINK_OPTION STATIC_LINK_LIBS benchmark::benchmark_main + parquet_static) else() set(PARQUET_BENCHMARK_LINK_OPTION EXTRA_LINK_LIBS parquet_shared) endif() @@ -144,7 +145,7 @@ add_custom_command(OUTPUT ${THRIFT_OUTPUT_FILES} COMMAND ${THRIFT_COMPILER} --gen cpp -out ${THRIFT_OUTPUT_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/parquet.thrift - DEPENDS ${ABS_PARQUET_THRIFT} thriftstatic + DEPENDS ${ABS_PARQUET_THRIFT} Thrift::thrift COMMENT "Running thrift compiler on parquet.thrift" VERBATIM) @@ -184,12 +185,12 @@ if(NOT PARQUET_MINIMAL_DEPENDENCY) # These are libraries that we will link privately with parquet_shared (as they # do not need to be linked transitively by other linkers) - set(PARQUET_SHARED_PRIVATE_LINK_LIBS ${PARQUET_BOOST_LINK_LIBS} thriftstatic) + set(PARQUET_SHARED_PRIVATE_LINK_LIBS ${PARQUET_BOOST_LINK_LIBS} Thrift::thrift) # Link publicly with # parquet_static (because internal users need to # transitively link all dependencies) set(PARQUET_STATIC_LINK_LIBS ${PARQUET_STATIC_LINK_LIBS} ${PARQUET_BOOST_LINK_LIBS} - thriftstatic) + Thrift::thrift) # Although we don't link parquet_objlib against anything, we need it to depend # on these libs as we may generate their headers via ExternalProject_Add diff --git a/cpp/src/parquet/arrow/arrow-reader-writer-test.cc b/cpp/src/parquet/arrow/arrow-reader-writer-test.cc index e0678388c04..3995d2c2407 100644 --- a/cpp/src/parquet/arrow/arrow-reader-writer-test.cc +++ b/cpp/src/parquet/arrow/arrow-reader-writer-test.cc @@ -344,8 +344,8 @@ void AssertChunkedEqual(const ChunkedArray& expected, const ChunkedArray& actual auto c1 = actual.chunk(i); auto c2 = expected.chunk(i); if (!c1->Equals(*c2)) { - EXPECT_OK(::arrow::PrettyPrint(*c1, 0, &pp_result)); - EXPECT_OK(::arrow::PrettyPrint(*c2, 0, &pp_expected)); + ARROW_EXPECT_OK(::arrow::PrettyPrint(*c1, 0, &pp_result)); + ARROW_EXPECT_OK(::arrow::PrettyPrint(*c2, 0, &pp_expected)); FAIL() << "Chunk " << i << " Got: " << pp_result.str() << "\nExpected: " << pp_expected.str(); } @@ -358,7 +358,7 @@ void PrintColumn(const Column& col, std::stringstream* ss) { for (int i = 0; i < carr.num_chunks(); ++i) { auto c1 = carr.chunk(i); *ss << "Chunk " << i << std::endl; - EXPECT_OK(::arrow::PrettyPrint(*c1, 0, ss)); + ARROW_EXPECT_OK(::arrow::PrettyPrint(*c1, 0, ss)); *ss << std::endl; } } diff --git a/cpp/src/parquet/arrow/test-util.h b/cpp/src/parquet/arrow/test-util.h index aefe84ad297..b963c3e9765 100644 --- a/cpp/src/parquet/arrow/test-util.h +++ b/cpp/src/parquet/arrow/test-util.h @@ -471,10 +471,11 @@ void ExpectArrayT(void* expected, Array* result) { template <> void ExpectArrayT<::arrow::BooleanType>(void* expected, Array* result) { ::arrow::BooleanBuilder builder; - EXPECT_OK(builder.AppendValues(reinterpret_cast(expected), result->length())); + ARROW_EXPECT_OK( + builder.AppendValues(reinterpret_cast(expected), result->length())); std::shared_ptr expected_array; - EXPECT_OK(builder.Finish(&expected_array)); + ARROW_EXPECT_OK(builder.Finish(&expected_array)); EXPECT_TRUE(result->Equals(*expected_array)); } diff --git a/cpp/src/parquet/arrow/writer.cc b/cpp/src/parquet/arrow/writer.cc index fb43c924f62..f9092a3e93a 100644 --- a/cpp/src/parquet/arrow/writer.cc +++ b/cpp/src/parquet/arrow/writer.cc @@ -748,7 +748,7 @@ template <> Status ArrowColumnWriter::TypedWriteBatch( const Array& array, int64_t num_levels, const int16_t* def_levels, const int16_t* rep_levels) { - bool* buffer; + bool* buffer = nullptr; RETURN_NOT_OK(ctx_->GetScratchData(array.length(), &buffer)); const auto& data = static_cast(array); @@ -782,7 +782,7 @@ template <> Status ArrowColumnWriter::TypedWriteBatch( const Array& array, int64_t num_levels, const int16_t* def_levels, const int16_t* rep_levels) { - ByteArray* buffer; + ByteArray* buffer = nullptr; RETURN_NOT_OK(ctx_->GetScratchData(num_levels, &buffer)); const auto& data = static_cast(array); diff --git a/cpp/src/plasma/CMakeLists.txt b/cpp/src/plasma/CMakeLists.txt index da46503fe2c..07c67a615ce 100644 --- a/cpp/src/plasma/CMakeLists.txt +++ b/cpp/src/plasma/CMakeLists.txt @@ -31,8 +31,7 @@ find_package(Threads) set(PLASMA_SO_VERSION "${ARROW_SO_VERSION}") set(PLASMA_FULL_SO_VERSION "${ARROW_FULL_SO_VERSION}") -include_directories("${FLATBUFFERS_INCLUDE_DIR}" "${CMAKE_CURRENT_LIST_DIR}/" - "${CMAKE_CURRENT_LIST_DIR}/thirdparty/" +include_directories("${CMAKE_CURRENT_LIST_DIR}/" "${CMAKE_CURRENT_LIST_DIR}/thirdparty/" "${CMAKE_CURRENT_LIST_DIR}/../") set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-conversion") @@ -48,9 +47,7 @@ set(PLASMA_FBS_OUTPUT_FILES "${OUTPUT_DIR}/common_generated.h" add_custom_target(gen_plasma_fbs DEPENDS ${PLASMA_FBS_OUTPUT_FILES}) -if(FLATBUFFERS_VENDORED) - add_dependencies(gen_plasma_fbs flatbuffers_ep) -endif() +add_dependencies(gen_plasma_fbs flatbuffers::flatc) add_custom_command( OUTPUT ${PLASMA_FBS_OUTPUT_FILES} @@ -58,7 +55,7 @@ add_custom_command( # flatbuffers message Message, which can be used to store deserialized # messages in data structures. This is currently used for ObjectInfo for # example. - COMMAND ${FLATBUFFERS_COMPILER} -c -o ${OUTPUT_DIR} ${PLASMA_FBS_SRC} + COMMAND flatbuffers::flatc -c -o ${OUTPUT_DIR} ${PLASMA_FBS_SRC} --gen-object-api --scoped-enums DEPENDS ${PLASMA_FBS_SRC} @@ -140,6 +137,21 @@ if(ARROW_RPATH_ORIGIN) endif() set_target_properties(plasma_store_server PROPERTIES INSTALL_RPATH ${_lib_install_rpath}) +elseif(APPLE) + # With OSX and conda, we need to set the correct RPATH so that dependencies + # are found. The installed libraries with conda have an RPATH that matches + # for executables and libraries lying in $ENV{CONDA_PREFIX}/bin or + # $ENV{CONDA_PREFIX}/lib but our test libraries and executables are not + # installed there. + if(NOT "$ENV{CONDA_PREFIX}" STREQUAL "" AND APPLE) + set_target_properties(plasma_store_server + PROPERTIES BUILD_WITH_INSTALL_RPATH + TRUE + INSTALL_RPATH_USE_LINK_PATH + TRUE + INSTALL_RPATH + "$ENV{CONDA_PREFIX}/lib") + endif() endif() install(FILES common.h @@ -189,17 +201,17 @@ if(ARROW_PLASMA_JAVA_CLIENT) plasma_shared ${PLASMA_LINK_LIBS} "-undefined dynamic_lookup" - -Wl,-force_load,${FLATBUFFERS_STATIC_LIB} - ${FLATBUFFERS_STATIC_LIB} + -Wl,-force_load,flatbuffers::flatbuffers + flatbuffers::flatbuffers ${PTHREAD_LIBRARY}) else(APPLE) target_link_libraries(plasma_java plasma_shared ${PLASMA_LINK_LIBS} -Wl,--whole-archive - ${FLATBUFFERS_STATIC_LIB} + flatbuffers::flatbuffers -Wl,--no-whole-archive - ${FLATBUFFERS_STATIC_LIB} + flatbuffers::flatbuffers ${PTHREAD_LIBRARY}) endif(APPLE) endif() diff --git a/cpp/thirdparty/versions.txt b/cpp/thirdparty/versions.txt index 40f599c162e..e7ad54cf6d0 100644 --- a/cpp/thirdparty/versions.txt +++ b/cpp/thirdparty/versions.txt @@ -25,6 +25,7 @@ BOOST_VERSION=1.67.0 BROTLI_VERSION=v0.6.0 +BZIP2_VERSION=1.0.6 CARES_VERSION=1.15.0 DOUBLE_CONVERSION_VERSION=v3.1.1 FLATBUFFERS_VERSION=v1.10.0 @@ -37,10 +38,13 @@ JEMALLOC_VERSION=17c897976c60b0e6e4f4a365c751027244dada7a LZ4_VERSION=v1.8.3 ORC_VERSION=1.5.4 PROTOBUF_VERSION=v3.6.1 -RAPIDJSON_VERSION=v1.1.0 +# Because of https://github.com/Tencent/rapidjson/pull/1323, we require +# a pre-release version of RapidJSON to build with GCC 8 without +# warnings. +RAPIDJSON_VERSION=2bbd33b33217ff4a73434ebf10cdac41e2ef5e34 RE2_VERSION=2018-10-01 SNAPPY_VERSION=1.1.3 -THRIFT_VERSION=0.11.0 +THRIFT_VERSION=0.12.0 ZLIB_VERSION=1.2.8 ZSTD_VERSION=v1.3.7 diff --git a/docker-compose.yml b/docker-compose.yml index b740015d262..86dfba5d444 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -32,14 +32,38 @@ x-ubuntu-volumes: - .:/arrow:delegated - ubuntu-cache:/build:delegated +x-ubuntu-xenial-volumes: + &ubuntu-xenial-volumes + - .:/arrow:delegated + - ubuntu-xenial-cache:/build:delegated + +x-ubuntu-bionic-volumes: + &ubuntu-bionic-volumes + - .:/arrow:delegated + - ubuntu-bionic-cache:/build:delegated + x-alpine-volumes: &alpine-volumes - .:/arrow:delegated - alpine-cache:/build:delegated +x-fedora-volumes: + &fedora-volumes + - .:/arrow:delegated + - fedora-cache:/build:delegated + +x-debian-testing-volumes: + &debian-testing-volumes + - .:/arrow:delegated + - debian-testing-cache:/build:delegated + volumes: ubuntu-cache: + ubuntu-xenial-cache: + ubuntu-bionic-cache: alpine-cache: + fedora-cache: + debian-testing-cache: maven-cache: spark-cache: @@ -99,6 +123,58 @@ services: PARQUET_TEST_DATA: /arrow/cpp/submodules/parquet-testing/data volumes: *alpine-volumes + cpp-fedora: + # Usage: + # docker-compose build cpp-fedora + # docker-compose run cpp-fedora + image: arrow:cpp-fedora + shm_size: 2G + build: + context: . + dockerfile: cpp/Dockerfile.fedora + environment: + PARQUET_TEST_DATA: /arrow/cpp/submodules/parquet-testing/data + volumes: *fedora-volumes + + cpp-debian-testing: + # Usage: + # docker-compose build cpp-debian-testing + # docker-compose run cpp-debian-testing + image: arrow:cpp-debian-testing + shm_size: 2G + build: + context: . + dockerfile: cpp/Dockerfile.debian-testing + environment: + PARQUET_TEST_DATA: /arrow/cpp/submodules/parquet-testing/data + volumes: *debian-testing-volumes + + cpp-ubuntu-xenial: + # Usage: + # docker-compose build cpp-ubuntu-xenial + # docker-compose run cpp-ubuntu-xenial + image: arrow:cpp-ubuntu-xenial + shm_size: 2G + build: + context: . + dockerfile: cpp/Dockerfile.ubuntu-xenial + environment: + PARQUET_TEST_DATA: /arrow/cpp/submodules/parquet-testing/data + volumes: *ubuntu-xenial-volumes + + cpp-ubuntu-bionic: + # Usage: + # docker-compose build cpp-ubuntu-bionic + # docker-compose run cpp-ubuntu-bionic + image: arrow:cpp-ubuntu-bionic + shm_size: 2G + build: + context: . + dockerfile: cpp/Dockerfile.ubuntu-bionic + environment: + PARQUET_TEST_DATA: /arrow/cpp/submodules/parquet-testing/data + volumes: *ubuntu-bionic-volumes + go: # Usage: # docker-compose build go @@ -242,7 +318,7 @@ services: # $ docker-compose pull python-manylinux1 # an then run: # $ docker-compose run -e PYTHON_VERSION=3.7 python-manylinux1 - image: quay.io/xhochy/arrow_manylinux1_x86_64_base:ARROW-4778 + image: quay.io/xhochy/arrow_manylinux1_x86_64_base:build-on-fedora build: context: python/manylinux1 dockerfile: Dockerfile-x86_64_base diff --git a/python/manylinux1/Dockerfile-x86_64_base b/python/manylinux1/Dockerfile-x86_64_base index 8ba205ee375..fc9f7d38f42 100644 --- a/python/manylinux1/Dockerfile-x86_64_base +++ b/python/manylinux1/Dockerfile-x86_64_base @@ -85,3 +85,15 @@ RUN /build_llvm.sh ADD scripts/build_clang.sh / RUN /build_clang.sh + +ADD scripts/build_double_conversion.sh / +RUN /build_double_conversion.sh + +ADD scripts/build_rapidjson.sh / +RUN /build_rapidjson.sh + +ADD scripts/build_re2.sh / +RUN /build_re2.sh + +ADD scripts/build_gflags.sh / +RUN /build_gflags.sh diff --git a/python/manylinux1/build_arrow.sh b/python/manylinux1/build_arrow.sh index 7738e3dbf1e..dbb6accabc9 100755 --- a/python/manylinux1/build_arrow.sh +++ b/python/manylinux1/build_arrow.sh @@ -42,7 +42,7 @@ export PYARROW_WITH_PLASMA=1 export PYARROW_BUNDLE_ARROW_CPP=1 export PYARROW_BUNDLE_BOOST=1 export PYARROW_BOOST_NAMESPACE=arrow_boost -export PKG_CONFIG_PATH=/arrow-dist/lib/pkgconfig +export PKG_CONFIG_PATH=/usr/lib/pkgconfig:/arrow-dist/lib/pkgconfig export PYARROW_CMAKE_OPTIONS='-DTHRIFT_HOME=/usr -DBoost_NAMESPACE=arrow_boost -DBOOST_ROOT=/arrow_boost_dist' # Ensure the target directory exists @@ -56,7 +56,7 @@ PYTHON_INTERPRETER="${CPYTHON_PATH}/bin/python" PIP="${CPYTHON_PATH}/bin/pip" PATH="${PATH}:${CPYTHON_PATH}" -if [ ${PYTHON_VERSION} != "2.7" ]; then +if [ "${PYTHON_VERSION}" != "2.7" ]; then # Gandiva is not supported on Python 2.7 export PYARROW_WITH_GANDIVA=1 export BUILD_ARROW_GANDIVA=ON @@ -70,6 +70,8 @@ ARROW_BUILD_DIR=/tmp/build-PY${PYTHON_VERSION}-${UNICODE_WIDTH} mkdir -p "${ARROW_BUILD_DIR}" pushd "${ARROW_BUILD_DIR}" PATH="${CPYTHON_PATH}/bin:${PATH}" cmake -DCMAKE_BUILD_TYPE=Release \ + -DARROW_DEPENDENCY_SOURCE="SYSTEM" \ + -DZLIB_ROOT=/usr/local \ -DCMAKE_INSTALL_PREFIX=/arrow-dist \ -DCMAKE_INSTALL_LIBDIR=lib \ -DARROW_BUILD_TESTS=OFF \ diff --git a/python/manylinux1/scripts/build_double_conversion.sh b/python/manylinux1/scripts/build_double_conversion.sh new file mode 100755 index 00000000000..0e73c702bee --- /dev/null +++ b/python/manylinux1/scripts/build_double_conversion.sh @@ -0,0 +1,29 @@ +#!/bin/bash -ex +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +DC_VERSION=3.1.1 + +curl -sL https://github.com/google/double-conversion/archive/v${DC_VERSION}.tar.gz -o double-conversion-${DC_VERSION}.tar.gz +tar xf double-conversion-${DC_VERSION}.tar.gz +pushd double-conversion-3.1.1 + +cmake . -DCMAKE_INSTALL_PREFIX=/usr -DBUILD_SHARED_LIBS=OFF -DBUILD_TESTING=OFF -GNinja -DCMAKE_POSITION_INDEPENDENT_CODE=ON +ninja install + +popd +rm -rf double-conversion-${DC_VERSION}.tar.gz double-conversion-3.1.1 diff --git a/python/manylinux1/scripts/build_gflags.sh b/python/manylinux1/scripts/build_gflags.sh new file mode 100755 index 00000000000..7f40437a83f --- /dev/null +++ b/python/manylinux1/scripts/build_gflags.sh @@ -0,0 +1,38 @@ +#!/bin/bash -ex +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +export GFLAGS_VERSION="2.2.1" +export CFLAGS="-fPIC" +export CXXFLAGS="-fPIC" + +curl -sL "https://github.com/gflags/gflags/archive/v${GFLAGS_VERSION}.tar.gz" -o gflags-${GFLAGS_VERSION}.tar.gz +tar xf gflags-${GFLAGS_VERSION}.tar.gz +pushd gflags-${GFLAGS_VERSION} + +cmake . \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX=/usr \ + -DINSTALL_HEADERS=on \ + -DBUILD_SHARED_LIBS=off \ + -DBUILD_STATIC_LIBS=on \ + -DBUILD_TESTING=off \ + -GNinja + +ninja install +popd +rm -rf gflags-${GFLAGS_VERSION}.tar.gz gflags-${GFLAGS_VERSION} diff --git a/python/manylinux1/scripts/build_rapidjson.sh b/python/manylinux1/scripts/build_rapidjson.sh new file mode 100755 index 00000000000..3b2ee99c422 --- /dev/null +++ b/python/manylinux1/scripts/build_rapidjson.sh @@ -0,0 +1,37 @@ +#!/bin/bash -ex +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +export RAPIDJSON_VERSION="1.1.0" + +curl -sL "https://github.com/miloyip/rapidjson/archive/v${RAPIDJSON_VERSION}.tar.gz" -o rapidjson-${RAPIDJSON_VERSION}.tar.gz +tar xf rapidjson-${RAPIDJSON_VERSION}.tar.gz +pushd rapidjson-${RAPIDJSON_VERSION} +mkdir build +pushd build +cmake -GNinja \ + -DRAPIDJSON_HAS_STDSTRING=ON \ + -DCMAKE_INSTALL_PREFIX=/usr \ + -DRAPIDJSON_BUILD_TESTS=OFF \ + -DRAPIDJSON_BUILD_EXAMPLES=OFF \ + -DRAPIDJSON_BUILD_DOC=OFF \ + -DCMAKE_BUILD_TYPE=release \ + .. +ninja install +popd +popd +rm -rf rapidjson-${RAPIDJSON_VERSION}.tar.gz rapidjson-${RAPIDJSON_VERSION} diff --git a/python/manylinux1/scripts/build_re2.sh b/python/manylinux1/scripts/build_re2.sh new file mode 100755 index 00000000000..a0fe9e56802 --- /dev/null +++ b/python/manylinux1/scripts/build_re2.sh @@ -0,0 +1,32 @@ +#!/bin/bash -ex +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +export RE2_VERSION="2019-01-01" + +curl -sL "http://github.com/google/re2/archive/${RE2_VERSION}.tar.gz" -o re2-${RE2_VERSION}.tar.gz +tar xf re2-${RE2_VERSION}.tar.gz +pushd re2-${RE2_VERSION} + +export CXXFLAGS="-fPIC ${CXXFLAGS}" +export CFLAGS="-fPIC ${CFLAGS}" + +# Build shared libraries +make prefix=/usr install + +popd +rm -rf re2-${RE2_VERSION}.tar.gz re2-${RE2_VERSION} /usr/lib/libre2.so* diff --git a/python/manylinux1/scripts/build_snappy.sh b/python/manylinux1/scripts/build_snappy.sh index bc3afe62e8e..a749897ac0c 100755 --- a/python/manylinux1/scripts/build_snappy.sh +++ b/python/manylinux1/scripts/build_snappy.sh @@ -16,12 +16,11 @@ # specific language governing permissions and limitations # under the License. -export SNAPPY_VERSION="1.1.3" -curl -sL "https://github.com/google/snappy/releases/download/${SNAPPY_VERSION}/snappy-${SNAPPY_VERSION}.tar.gz" -o snappy-${SNAPPY_VERSION}.tar.gz +export SNAPPY_VERSION="1.1.7" +curl -sL "https://github.com/google/snappy/archive/${SNAPPY_VERSION}.tar.gz" -o snappy-${SNAPPY_VERSION}.tar.gz tar xf snappy-${SNAPPY_VERSION}.tar.gz pushd snappy-${SNAPPY_VERSION} -./configure --with-pic "--prefix=/usr" CXXFLAGS='-DNDEBUG -O2' -make -j5 -make install +CXXFLAGS='-DNDEBUG -O2' cmake -GNinja -DCMAKE_INSTALL_PREFIX=/usr -DBUILD_SHARED_LIBS=OFF -DCMAKE_POSITION_INDEPENDENT_CODE=1 -DSNAPPY_BUILD_TESTS=OFF . +ninja install popd rm -rf snappy-${SNAPPY_VERSION}.tar.gz snappy-${SNAPPY_VERSION} diff --git a/python/manylinux1/scripts/build_zstd.sh b/python/manylinux1/scripts/build_zstd.sh index ccfda3f399c..1add13faa61 100755 --- a/python/manylinux1/scripts/build_zstd.sh +++ b/python/manylinux1/scripts/build_zstd.sh @@ -16,15 +16,17 @@ # specific language governing permissions and limitations # under the License. -export ZSTD_VERSION="1.2.0" -export CFLAGS="${CFLAGS} -O3 -fPIC" -export PREFIX="/usr" -export LDFLAGS="${LDFLAGS} -Wl,-rpath,${PREFIX}/lib" +export ZSTD_VERSION="1.3.8" + curl -sL "https://github.com/facebook/zstd/archive/v${ZSTD_VERSION}.tar.gz" -o zstd-${ZSTD_VERSION}.tar.gz tar xf zstd-${ZSTD_VERSION}.tar.gz pushd zstd-${ZSTD_VERSION} +mkdir build_cmake +pushd build_cmake + +cmake -GNinja -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr ../build/cmake -DZSTD_BUILD_STATIC=ON -DZSTD_BUILD_SHARED=OFF -DCMAKE_POSITION_INDEPENDENT_CODE=1 +ninja install -make -j5 -make install PREFIX=$PREFIX +popd popd rm -rf zstd-${ZSTD_VERSION}.tar.gz zstd-${ZSTD_VERSION} diff --git a/python/requirements-wheel.txt b/python/requirements-wheel.txt index c44903efd36..98ee98e7c42 100644 --- a/python/requirements-wheel.txt +++ b/python/requirements-wheel.txt @@ -1,4 +1,5 @@ wheel==0.31.1 +setuptools_scm six>=1.0.0 numpy==1.14.5 futures; python_version < "3.2"